Page MenuHomeFreeBSD
Differential D15736

Implement fast path for malloc and free
AbandonedPublic
Actions

Authored by mjg on Jun 10 2018, 1:36 PM.
Tags
None
Referenced Files
Unknown Object (File)
Fri, Feb 28, 3:54 PM
Unknown Object (File)
Thu, Feb 27, 2:22 PM
Unknown Object (File)
Mon, Feb 24, 4:49 PM
Unknown Object (File)
Feb 22 2025, 3:57 AM
Unknown Object (File)
Feb 19 2025, 11:18 PM
Unknown Object (File)
Feb 8 2025, 7:35 AM
Unknown Object (File)
Jan 31 2025, 12:20 PM
Unknown Object (File)
Jan 30 2025, 6:56 AM
View All 40 Files
Subscribers
imp

Details

Reviewers
jeff
markj
Summary

Avoids func calls and branches in the common case.

malloc+free throughput per second: ~19323918 to ~27418181.

lock1 from will-it-scale (single-threaded): 2196902 -> 2345506

Test Plan

make universe. running with DEBUG_REDZONE and DEBUG_MEMGUARD

Diff Detail

Repository
rS FreeBSD src repository - subversion
Lint
Lint Skipped
Unit
Tests Skipped
Build Status
Buildable 17135

Event Timeline

mjg created this revision.Jun 10 2018, 1:36 PM
Herald added a subscriber: imp. · View Herald TranscriptJun 10 2018, 1:36 PM
Harbormaster completed remote builds in B17135: Diff 43543.Jun 10 2018, 1:36 PM
jeff added a comment.Jun 11 2018, 1:30 AM

Does this not include the malloc.h changes for M_ZERO?

Can we do constant size at the same time?

sys/kern/kern_malloc.c
654–660

This should be a uma_int.h inline.

865–869

Should also be an inline. Possibly with some of the code above.

markj added a reviewer: markj.Sep 6 2018, 3:18 PM
mjg updated this revision to Diff 48391.Sep 24 2018, 12:22 PM
  • rebased
mjg added a comment.Sep 25 2018, 10:40 AM

The M_ZERO stuff went in quite some time ago.

I plan to deal with static one lookup later.

I don't think the inlined cache bump matters. But if you insist i can move it to a dedicated routine.

mjg planned changes to this revision.Aug 31 2019, 9:50 AM

Apart from addressing this review request this needs some reworking due to other changes in uma.

mjg abandoned this revision.Mar 12 2024, 10:30 PM

Revision Contents
Changeset List

PathSize
sys/
kern/
208 lines
vm/
9 lines
8 lines

Diff 43543

View Options

sys/kern/kern_malloc.c

Show First 20 Lines • Show All 587 Lines • ▼ Show 20 Linesstruct ifmultiaddr {
struct ifnet *ifma_ifp; /* back-pointer to interface */ struct ifnet *ifma_ifp; /* back-pointer to interface */
u_int ifma_refcount; /* reference count */ u_int ifma_refcount; /* reference count */
int ifma_flags; int ifma_flags;
void *ifma_protospec; /* protocol-specific state, if any */ void *ifma_protospec; /* protocol-specific state, if any */
struct ifmultiaddr *ifma_llifma; /* pointer to ifma for ifma_lladdr */ struct ifmultiaddr *ifma_llifma; /* pointer to ifma for ifma_lladdr */
struct epoch_context ifma_epoch_ctx; struct epoch_context ifma_epoch_ctx;
}; };
extern struct rwlock ifnet_rwlock;
extern struct sx ifnet_sxlock; extern struct sx ifnet_sxlock;
#define IFNET_WLOCK() do { \ #define IFNET_WLOCK() do { \
sx_xlock(&ifnet_sxlock); \ sx_xlock(&ifnet_sxlock); \
rw_wlock(&ifnet_rwlock); \
} while (0) } while (0)
#define IFNET_WUNLOCK() do { \ #define IFNET_WUNLOCK() do { \
rw_wunlock(&ifnet_rwlock); \
sx_xunlock(&ifnet_sxlock); \ sx_xunlock(&ifnet_sxlock); \
} while (0) } while (0)
/*
* To assert the ifnet lock, you must know not only whether it's for read or
* write, but also whether it was acquired with sleep support or not.
*/
#define IFNET_RLOCK_ASSERT() sx_assert(&ifnet_sxlock, SA_SLOCKED) #define IFNET_RLOCK_ASSERT() sx_assert(&ifnet_sxlock, SA_SLOCKED)
kevansUnsubmitted
Done
Inline Actions

This comment should just go away entirely, probably... you only need to know whether it's for read or write now.

kevans: This comment should just go away entirely, probably... you only need to know whether it's for…
#define IFNET_WLOCK_ASSERT() do { \ #define IFNET_WLOCK_ASSERT() do { \
sx_assert(&ifnet_sxlock, SA_XLOCKED); \ sx_assert(&ifnet_sxlock, SA_XLOCKED); \
rw_assert(&ifnet_rwlock, RA_WLOCKED); \
} while (0) } while (0)
#define IFNET_RLOCK() sx_slock(&ifnet_sxlock) #define IFNET_RLOCK() sx_slock(&ifnet_sxlock)
#define IFNET_RUNLOCK() sx_sunlock(&ifnet_sxlock) #define IFNET_RUNLOCK() sx_sunlock(&ifnet_sxlock)
/* /*
* Look up an ifnet given its index; the _ref variant also acquires a * Look up an ifnet given its index; the _ref variant also acquires a
* reference that must be freed using if_rele(). It is almost always a bug * reference that must be freed using if_rele(). It is almost always a bug
▲ Show 20 Lines • Show All 189 Lines • Show Last 20 Lines
View Options

sys/vm/uma_core.c

Show First 20 Lines • Show All 587 Lines • ▼ Show 20 Linesstruct ifmultiaddr {
struct ifnet *ifma_ifp; /* back-pointer to interface */ struct ifnet *ifma_ifp; /* back-pointer to interface */
u_int ifma_refcount; /* reference count */ u_int ifma_refcount; /* reference count */
int ifma_flags; int ifma_flags;
void *ifma_protospec; /* protocol-specific state, if any */ void *ifma_protospec; /* protocol-specific state, if any */
struct ifmultiaddr *ifma_llifma; /* pointer to ifma for ifma_lladdr */ struct ifmultiaddr *ifma_llifma; /* pointer to ifma for ifma_lladdr */
struct epoch_context ifma_epoch_ctx; struct epoch_context ifma_epoch_ctx;
}; };
extern struct rwlock ifnet_rwlock;
extern struct sx ifnet_sxlock; extern struct sx ifnet_sxlock;
#define IFNET_WLOCK() do { \ #define IFNET_WLOCK() do { \
sx_xlock(&ifnet_sxlock); \ sx_xlock(&ifnet_sxlock); \
rw_wlock(&ifnet_rwlock); \
} while (0) } while (0)
#define IFNET_WUNLOCK() do { \ #define IFNET_WUNLOCK() do { \
rw_wunlock(&ifnet_rwlock); \
sx_xunlock(&ifnet_sxlock); \ sx_xunlock(&ifnet_sxlock); \
} while (0) } while (0)
/*
* To assert the ifnet lock, you must know not only whether it's for read or
* write, but also whether it was acquired with sleep support or not.
*/
#define IFNET_RLOCK_ASSERT() sx_assert(&ifnet_sxlock, SA_SLOCKED) #define IFNET_RLOCK_ASSERT() sx_assert(&ifnet_sxlock, SA_SLOCKED)
kevansUnsubmitted
Done
Inline Actions

This comment should just go away entirely, probably... you only need to know whether it's for read or write now.

kevans: This comment should just go away entirely, probably... you only need to know whether it's for…
#define IFNET_WLOCK_ASSERT() do { \ #define IFNET_WLOCK_ASSERT() do { \
sx_assert(&ifnet_sxlock, SA_XLOCKED); \ sx_assert(&ifnet_sxlock, SA_XLOCKED); \
rw_assert(&ifnet_rwlock, RA_WLOCKED); \
} while (0) } while (0)
#define IFNET_RLOCK() sx_slock(&ifnet_sxlock) #define IFNET_RLOCK() sx_slock(&ifnet_sxlock)
#define IFNET_RUNLOCK() sx_sunlock(&ifnet_sxlock) #define IFNET_RUNLOCK() sx_sunlock(&ifnet_sxlock)
/* /*
* Look up an ifnet given its index; the _ref variant also acquires a * Look up an ifnet given its index; the _ref variant also acquires a
* reference that must be freed using if_rele(). It is almost always a bug * reference that must be freed using if_rele(). It is almost always a bug
▲ Show 20 Lines • Show All 189 Lines • Show Last 20 Lines
View Options

sys/vm/uma_dbg.h

Show First 20 Lines • Show All 587 Lines • ▼ Show 20 Linesstruct ifmultiaddr {
struct ifnet *ifma_ifp; /* back-pointer to interface */ struct ifnet *ifma_ifp; /* back-pointer to interface */
u_int ifma_refcount; /* reference count */ u_int ifma_refcount; /* reference count */
int ifma_flags; int ifma_flags;
void *ifma_protospec; /* protocol-specific state, if any */ void *ifma_protospec; /* protocol-specific state, if any */
struct ifmultiaddr *ifma_llifma; /* pointer to ifma for ifma_lladdr */ struct ifmultiaddr *ifma_llifma; /* pointer to ifma for ifma_lladdr */
struct epoch_context ifma_epoch_ctx; struct epoch_context ifma_epoch_ctx;
}; };
extern struct rwlock ifnet_rwlock;
extern struct sx ifnet_sxlock; extern struct sx ifnet_sxlock;
#define IFNET_WLOCK() do { \ #define IFNET_WLOCK() do { \
sx_xlock(&ifnet_sxlock); \ sx_xlock(&ifnet_sxlock); \
rw_wlock(&ifnet_rwlock); \
} while (0) } while (0)
#define IFNET_WUNLOCK() do { \ #define IFNET_WUNLOCK() do { \
rw_wunlock(&ifnet_rwlock); \
sx_xunlock(&ifnet_sxlock); \ sx_xunlock(&ifnet_sxlock); \
} while (0) } while (0)
/*
* To assert the ifnet lock, you must know not only whether it's for read or
* write, but also whether it was acquired with sleep support or not.
*/
#define IFNET_RLOCK_ASSERT() sx_assert(&ifnet_sxlock, SA_SLOCKED) #define IFNET_RLOCK_ASSERT() sx_assert(&ifnet_sxlock, SA_SLOCKED)
kevansUnsubmitted
Done
Inline Actions

This comment should just go away entirely, probably... you only need to know whether it's for read or write now.

kevans: This comment should just go away entirely, probably... you only need to know whether it's for…
#define IFNET_WLOCK_ASSERT() do { \ #define IFNET_WLOCK_ASSERT() do { \
sx_assert(&ifnet_sxlock, SA_XLOCKED); \ sx_assert(&ifnet_sxlock, SA_XLOCKED); \
rw_assert(&ifnet_rwlock, RA_WLOCKED); \
} while (0) } while (0)
#define IFNET_RLOCK() sx_slock(&ifnet_sxlock) #define IFNET_RLOCK() sx_slock(&ifnet_sxlock)
#define IFNET_RUNLOCK() sx_sunlock(&ifnet_sxlock) #define IFNET_RUNLOCK() sx_sunlock(&ifnet_sxlock)
/* /*
* Look up an ifnet given its index; the _ref variant also acquires a * Look up an ifnet given its index; the _ref variant also acquires a
* reference that must be freed using if_rele(). It is almost always a bug * reference that must be freed using if_rele(). It is almost always a bug
▲ Show 20 Lines • Show All 189 Lines • Show Last 20 Lines