Changeset View
Changeset View
Standalone View
Standalone View
sys/netinet/in_pcb.h
Show First 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | |||||
#include <sys/_mutex.h> | #include <sys/_mutex.h> | ||||
#include <sys/_rwlock.h> | #include <sys/_rwlock.h> | ||||
#include <net/route.h> | #include <net/route.h> | ||||
#ifdef _KERNEL | #ifdef _KERNEL | ||||
#include <sys/lock.h> | #include <sys/lock.h> | ||||
#include <sys/rwlock.h> | #include <sys/rwlock.h> | ||||
#include <net/vnet.h> | #include <net/vnet.h> | ||||
#include <net/if.h> | |||||
#include <net/if_var.h> | |||||
#include <vm/uma.h> | #include <vm/uma.h> | ||||
#endif | #endif | ||||
#include <sys/ck.h> | #include <sys/ck.h> | ||||
#define in6pcb inpcb /* for KAME src sync over BSD*'s */ | #define in6pcb inpcb /* for KAME src sync over BSD*'s */ | ||||
#define in6p_sp inp_sp /* for KAME src sync over BSD*'s */ | #define in6p_sp inp_sp /* for KAME src sync over BSD*'s */ | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 90 Lines • ▼ Show 20 Lines | |||||
* This ensures that the relatively rare global list iterations safely walk a | * This ensures that the relatively rare global list iterations safely walk a | ||||
* stable snapshot of connections while allowing more common list modifications | * stable snapshot of connections while allowing more common list modifications | ||||
* to safely grab the pcblist lock just while adding or removing a connection | * to safely grab the pcblist lock just while adding or removing a connection | ||||
* from the global list. | * from the global list. | ||||
* | * | ||||
* Key: | * Key: | ||||
* (b) - Protected by the hpts lock. | * (b) - Protected by the hpts lock. | ||||
* (c) - Constant after initialization | * (c) - Constant after initialization | ||||
* (e) - Protected by the net_epoch_prempt epoch | |||||
* (g) - Protected by the pcbgroup lock | * (g) - Protected by the pcbgroup lock | ||||
* (i) - Protected by the inpcb lock | * (i) - Protected by the inpcb lock | ||||
* (p) - Protected by the pcbinfo lock for the inpcb | * (p) - Protected by the pcbinfo lock for the inpcb | ||||
* (l) - Protected by the pcblist lock for the inpcb | * (l) - Protected by the pcblist lock for the inpcb | ||||
* (h) - Protected by the pcbhash lock for the inpcb | * (h) - Protected by the pcbhash lock for the inpcb | ||||
* (s) - Protected by another subsystem's locks | * (s) - Protected by another subsystem's locks | ||||
* (x) - Undefined locking | * (x) - Undefined locking | ||||
* | * | ||||
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines | |||||
* read-lock usage during modification, this model can be applied to other | * read-lock usage during modification, this model can be applied to other | ||||
* protocols (especially SCTP). | * protocols (especially SCTP). | ||||
*/ | */ | ||||
struct icmp6_filter; | struct icmp6_filter; | ||||
struct inpcbpolicy; | struct inpcbpolicy; | ||||
struct m_snd_tag; | struct m_snd_tag; | ||||
struct inpcb { | struct inpcb { | ||||
/* Cache line #1 (amd64) */ | /* Cache line #1 (amd64) */ | ||||
CK_LIST_ENTRY(inpcb) inp_hash; /* (h/i) hash list */ | CK_LIST_ENTRY(inpcb) inp_hash; /* [w](h/i) [r](e/i) hash list */ | ||||
CK_LIST_ENTRY(inpcb) inp_pcbgrouphash; /* (g/i) hash list */ | CK_LIST_ENTRY(inpcb) inp_pcbgrouphash; /* (g/i) hash list */ | ||||
struct rwlock inp_lock; | struct rwlock inp_lock; | ||||
/* Cache line #2 (amd64) */ | /* Cache line #2 (amd64) */ | ||||
#define inp_start_zero inp_hpts | #define inp_start_zero inp_hpts | ||||
#define inp_zero_size (sizeof(struct inpcb) - \ | #define inp_zero_size (sizeof(struct inpcb) - \ | ||||
offsetof(struct inpcb, inp_start_zero)) | offsetof(struct inpcb, inp_start_zero)) | ||||
TAILQ_ENTRY(inpcb) inp_hpts; /* pacing out queue next lock(b) */ | TAILQ_ENTRY(inpcb) inp_hpts; /* pacing out queue next lock(b) */ | ||||
▲ Show 20 Lines • Show All 75 Lines • ▼ Show 20 Lines | #endif | ||||
struct inpcbport *inp_phd; /* (i/h) head of this list */ | struct inpcbport *inp_phd; /* (i/h) head of this list */ | ||||
inp_gen_t inp_gencnt; /* (c) generation count */ | inp_gen_t inp_gencnt; /* (c) generation count */ | ||||
struct llentry *inp_lle; /* cached L2 information */ | struct llentry *inp_lle; /* cached L2 information */ | ||||
rt_gen_t inp_rt_cookie; /* generation for route entry */ | rt_gen_t inp_rt_cookie; /* generation for route entry */ | ||||
union { /* cached L3 information */ | union { /* cached L3 information */ | ||||
struct route inp_route; | struct route inp_route; | ||||
struct route_in6 inp_route6; | struct route_in6 inp_route6; | ||||
}; | }; | ||||
CK_LIST_ENTRY(inpcb) inp_list; /* (p/l) list for all PCBs for proto */ | CK_LIST_ENTRY(inpcb) inp_list; /* (p/l) list for all PCBs for proto */ | ||||
jtl: While here, please update/fix the locking annotations on these lines. (For example, the [w] and… | |||||
/* (p[w]) for list iteration */ | /* (e[r]) for list iteration */ | ||||
/* (p[r]/l) for addition/removal */ | /* (p[w]/l) for addition/removal */ | ||||
struct epoch_context inp_epoch_ctx; | struct epoch_context inp_epoch_ctx; | ||||
}; | }; | ||||
#endif /* _KERNEL */ | #endif /* _KERNEL */ | ||||
#define inp_fport inp_inc.inc_fport | #define inp_fport inp_inc.inc_fport | ||||
#define inp_lport inp_inc.inc_lport | #define inp_lport inp_inc.inc_lport | ||||
#define inp_faddr inp_inc.inc_faddr | #define inp_faddr inp_inc.inc_faddr | ||||
#define inp_laddr inp_inc.inc_laddr | #define inp_laddr inp_inc.inc_laddr | ||||
▲ Show 20 Lines • Show All 94 Lines • ▼ Show 20 Lines | |||||
* ipi_lock (before) | * ipi_lock (before) | ||||
* inpcb locks (before) | * inpcb locks (before) | ||||
* ipi_list locks (before) | * ipi_list locks (before) | ||||
* {ipi_hash_lock, pcbgroup locks} | * {ipi_hash_lock, pcbgroup locks} | ||||
* | * | ||||
* Locking key: | * Locking key: | ||||
* | * | ||||
* (c) Constant or nearly constant after initialisation | * (c) Constant or nearly constant after initialisation | ||||
* (e) - Protected by the net_epoch_prempt epoch | |||||
* (g) Locked by ipi_lock | * (g) Locked by ipi_lock | ||||
* (l) Locked by ipi_list_lock | * (l) Locked by ipi_list_lock | ||||
* (h) Read using either ipi_hash_lock or inpcb lock; write requires both | * (h) Read using either net_epoch_preempt or inpcb lock; write requires both ipi_hash_lock and inpcb lock | ||||
* (p) Protected by one or more pcbgroup locks | * (p) Protected by one or more pcbgroup locks | ||||
* (x) Synchronisation properties poorly defined | * (x) Synchronisation properties poorly defined | ||||
*/ | */ | ||||
struct inpcbinfo { | struct inpcbinfo { | ||||
/* | /* | ||||
* Global lock protecting full inpcb list traversal | * Global lock protecting inpcb modification | ||||
Not Done Inline ActionsIts not really "inpcb modification", but rather "inpcb list modification", right? jtl: Its not really "inpcb modification", but rather "inpcb list modification", right? | |||||
Not Done Inline ActionsCorrect. mmacy: Correct. | |||||
Not Done Inline ActionsOK. Then, it seems like this commend should be updated when you commit the change. jtl: OK. Then, it seems like this commend should be updated when you commit the change. | |||||
*/ | */ | ||||
struct rwlock ipi_lock; | struct mtx ipi_lock; | ||||
/* | /* | ||||
* Global list of inpcbs on the protocol. | * Global list of inpcbs on the protocol. | ||||
*/ | */ | ||||
struct inpcbhead *ipi_listhead; /* (g/l) */ | struct inpcbhead *ipi_listhead; /* [r](e) [w](g/l) */ | ||||
u_int ipi_count; /* (l) */ | u_int ipi_count; /* (l) */ | ||||
/* | /* | ||||
* Generation count -- incremented each time a connection is allocated | * Generation count -- incremented each time a connection is allocated | ||||
* or freed. | * or freed. | ||||
*/ | */ | ||||
u_quad_t ipi_gencnt; /* (l) */ | u_quad_t ipi_gencnt; /* (l) */ | ||||
Show All 14 Lines | struct inpcbinfo { | ||||
* constant, but pcbgroup structures themselves are protected by | * constant, but pcbgroup structures themselves are protected by | ||||
* per-pcbgroup locks. | * per-pcbgroup locks. | ||||
*/ | */ | ||||
struct inpcbgroup *ipi_pcbgroups; /* (c) */ | struct inpcbgroup *ipi_pcbgroups; /* (c) */ | ||||
u_int ipi_npcbgroups; /* (c) */ | u_int ipi_npcbgroups; /* (c) */ | ||||
u_int ipi_hashfields; /* (c) */ | u_int ipi_hashfields; /* (c) */ | ||||
/* | /* | ||||
* Global lock protecting non-pcbgroup hash lookup tables. | * Global lock protecting modification non-pcbgroup hash lookup tables. | ||||
*/ | */ | ||||
struct rwlock ipi_hash_lock; | struct mtx ipi_hash_lock; | ||||
/* | /* | ||||
* Global hash of inpcbs, hashed by local and foreign addresses and | * Global hash of inpcbs, hashed by local and foreign addresses and | ||||
* port numbers. | * port numbers. | ||||
*/ | */ | ||||
struct inpcbhead *ipi_hashbase; /* (h) */ | struct inpcbhead *ipi_hashbase; /* (h) */ | ||||
u_long ipi_hashmask; /* (h) */ | u_long ipi_hashmask; /* (h) */ | ||||
▲ Show 20 Lines • Show All 124 Lines • ▼ Show 20 Lines | |||||
struct tcpcb * | struct tcpcb * | ||||
inp_inpcbtotcpcb(struct inpcb *inp); | inp_inpcbtotcpcb(struct inpcb *inp); | ||||
void inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp, | void inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp, | ||||
uint32_t *faddr, uint16_t *fp); | uint32_t *faddr, uint16_t *fp); | ||||
int inp_so_options(const struct inpcb *inp); | int inp_so_options(const struct inpcb *inp); | ||||
#endif /* _KERNEL */ | #endif /* _KERNEL */ | ||||
#define INP_INFO_LOCK_INIT(ipi, d) \ | #define INP_INFO_LOCK_INIT(ipi, d) \ | ||||
rw_init_flags(&(ipi)->ipi_lock, (d), RW_RECURSE) | mtx_init(&(ipi)->ipi_lock, (d), NULL, MTX_DEF| MTX_RECURSE) | ||||
Not Done Inline ActionsJust off the top of my head, this section of lock macros doesn't look right. For example, INP_INFO_TRY_UPGRADE can't try to upgrade an rw lock that it won't actually own in read mode (since INP_INFO_RLOCK doesn't actually acquire a read lock). Also, now that we will only hold a lock when writing, it seems like this should all switch from rw locks to mutexes. jtl: Just off the top of my head, this section of lock macros doesn't look right.
For example… | |||||
#define INP_INFO_LOCK_DESTROY(ipi) rw_destroy(&(ipi)->ipi_lock) | #define INP_INFO_LOCK_DESTROY(ipi) mtx_destroy(&(ipi)->ipi_lock) | ||||
#define INP_INFO_RLOCK(ipi) rw_rlock(&(ipi)->ipi_lock) | #define INP_INFO_RLOCK(ipi) NET_EPOCH_ENTER() | ||||
#define INP_INFO_WLOCK(ipi) rw_wlock(&(ipi)->ipi_lock) | #define INP_INFO_WLOCK(ipi) mtx_lock(&(ipi)->ipi_lock) | ||||
#define INP_INFO_TRY_RLOCK(ipi) rw_try_rlock(&(ipi)->ipi_lock) | #define INP_INFO_TRY_WLOCK(ipi) mtx_trylock(&(ipi)->ipi_lock) | ||||
#define INP_INFO_TRY_WLOCK(ipi) rw_try_wlock(&(ipi)->ipi_lock) | #define INP_INFO_WLOCKED(ipi) mtx_owned(&(ipi)->ipi_lock) | ||||
#define INP_INFO_TRY_UPGRADE(ipi) rw_try_upgrade(&(ipi)->ipi_lock) | #define INP_INFO_RUNLOCK(ipi) NET_EPOCH_EXIT() | ||||
#define INP_INFO_WLOCKED(ipi) rw_wowned(&(ipi)->ipi_lock) | #define INP_INFO_WUNLOCK(ipi) mtx_unlock(&(ipi)->ipi_lock) | ||||
#define INP_INFO_RUNLOCK(ipi) rw_runlock(&(ipi)->ipi_lock) | #define INP_INFO_LOCK_ASSERT(ipi) MPASS(in_epoch() || mtx_owned(&(ipi)->ipi_lock)) | ||||
#define INP_INFO_WUNLOCK(ipi) rw_wunlock(&(ipi)->ipi_lock) | #define INP_INFO_RLOCK_ASSERT(ipi) MPASS(in_epoch()) | ||||
#define INP_INFO_LOCK_ASSERT(ipi) rw_assert(&(ipi)->ipi_lock, RA_LOCKED) | #define INP_INFO_WLOCK_ASSERT(ipi) mtx_assert(&(ipi)->ipi_lock, MA_OWNED) | ||||
#define INP_INFO_RLOCK_ASSERT(ipi) rw_assert(&(ipi)->ipi_lock, RA_RLOCKED) | #define INP_INFO_UNLOCK_ASSERT(ipi) MPASS(!in_epoch() && !mtx_owned(&(ipi)->ipi_lock)) | ||||
#define INP_INFO_WLOCK_ASSERT(ipi) rw_assert(&(ipi)->ipi_lock, RA_WLOCKED) | |||||
#define INP_INFO_UNLOCK_ASSERT(ipi) rw_assert(&(ipi)->ipi_lock, RA_UNLOCKED) | |||||
#define INP_LIST_LOCK_INIT(ipi, d) \ | #define INP_LIST_LOCK_INIT(ipi, d) \ | ||||
Not Done Inline ActionsDo the INP_LIST macros need similar treatment to the INP_INFO macros? jtl: Do the INP_LIST macros need similar treatment to the INP_INFO macros? | |||||
Not Done Inline ActionsNo, I'm keeping them unchanged. I don't think that the code it protects is performance critical. mmacy: No, I'm keeping them unchanged. I don't think that the code it protects is performance critical. | |||||
rw_init_flags(&(ipi)->ipi_list_lock, (d), 0) | rw_init_flags(&(ipi)->ipi_list_lock, (d), 0) | ||||
#define INP_LIST_LOCK_DESTROY(ipi) rw_destroy(&(ipi)->ipi_list_lock) | #define INP_LIST_LOCK_DESTROY(ipi) rw_destroy(&(ipi)->ipi_list_lock) | ||||
#define INP_LIST_RLOCK(ipi) rw_rlock(&(ipi)->ipi_list_lock) | #define INP_LIST_RLOCK(ipi) rw_rlock(&(ipi)->ipi_list_lock) | ||||
#define INP_LIST_WLOCK(ipi) rw_wlock(&(ipi)->ipi_list_lock) | #define INP_LIST_WLOCK(ipi) rw_wlock(&(ipi)->ipi_list_lock) | ||||
#define INP_LIST_TRY_RLOCK(ipi) rw_try_rlock(&(ipi)->ipi_list_lock) | #define INP_LIST_TRY_RLOCK(ipi) rw_try_rlock(&(ipi)->ipi_list_lock) | ||||
#define INP_LIST_TRY_WLOCK(ipi) rw_try_wlock(&(ipi)->ipi_list_lock) | #define INP_LIST_TRY_WLOCK(ipi) rw_try_wlock(&(ipi)->ipi_list_lock) | ||||
#define INP_LIST_TRY_UPGRADE(ipi) rw_try_upgrade(&(ipi)->ipi_list_lock) | #define INP_LIST_TRY_UPGRADE(ipi) rw_try_upgrade(&(ipi)->ipi_list_lock) | ||||
#define INP_LIST_RUNLOCK(ipi) rw_runlock(&(ipi)->ipi_list_lock) | #define INP_LIST_RUNLOCK(ipi) rw_runlock(&(ipi)->ipi_list_lock) | ||||
#define INP_LIST_WUNLOCK(ipi) rw_wunlock(&(ipi)->ipi_list_lock) | #define INP_LIST_WUNLOCK(ipi) rw_wunlock(&(ipi)->ipi_list_lock) | ||||
#define INP_LIST_LOCK_ASSERT(ipi) \ | #define INP_LIST_LOCK_ASSERT(ipi) \ | ||||
rw_assert(&(ipi)->ipi_list_lock, RA_LOCKED) | rw_assert(&(ipi)->ipi_list_lock, RA_LOCKED) | ||||
#define INP_LIST_RLOCK_ASSERT(ipi) \ | #define INP_LIST_RLOCK_ASSERT(ipi) \ | ||||
rw_assert(&(ipi)->ipi_list_lock, RA_RLOCKED) | rw_assert(&(ipi)->ipi_list_lock, RA_RLOCKED) | ||||
#define INP_LIST_WLOCK_ASSERT(ipi) \ | #define INP_LIST_WLOCK_ASSERT(ipi) \ | ||||
rw_assert(&(ipi)->ipi_list_lock, RA_WLOCKED) | rw_assert(&(ipi)->ipi_list_lock, RA_WLOCKED) | ||||
#define INP_LIST_UNLOCK_ASSERT(ipi) \ | #define INP_LIST_UNLOCK_ASSERT(ipi) \ | ||||
rw_assert(&(ipi)->ipi_list_lock, RA_UNLOCKED) | rw_assert(&(ipi)->ipi_list_lock, RA_UNLOCKED) | ||||
#define INP_HASH_LOCK_INIT(ipi, d) \ | #define INP_HASH_LOCK_INIT(ipi, d) mtx_init(&(ipi)->ipi_hash_lock, (d), NULL, MTX_DEF) | ||||
rw_init_flags(&(ipi)->ipi_hash_lock, (d), 0) | #define INP_HASH_LOCK_DESTROY(ipi) mtx_destroy(&(ipi)->ipi_hash_lock) | ||||
#define INP_HASH_LOCK_DESTROY(ipi) rw_destroy(&(ipi)->ipi_hash_lock) | #define INP_HASH_RLOCK(ipi) NET_EPOCH_ENTER() | ||||
#define INP_HASH_RLOCK(ipi) rw_rlock(&(ipi)->ipi_hash_lock) | #define INP_HASH_WLOCK(ipi) mtx_lock(&(ipi)->ipi_hash_lock) | ||||
#define INP_HASH_WLOCK(ipi) rw_wlock(&(ipi)->ipi_hash_lock) | #define INP_HASH_RUNLOCK(ipi) NET_EPOCH_EXIT() | ||||
#define INP_HASH_RUNLOCK(ipi) rw_runlock(&(ipi)->ipi_hash_lock) | #define INP_HASH_WUNLOCK(ipi) mtx_unlock(&(ipi)->ipi_hash_lock) | ||||
#define INP_HASH_WUNLOCK(ipi) rw_wunlock(&(ipi)->ipi_hash_lock) | #define INP_HASH_LOCK_ASSERT(ipi) MPASS(in_epoch() || mtx_owned(&(ipi)->ipi_hash_lock)) | ||||
#define INP_HASH_LOCK_ASSERT(ipi) rw_assert(&(ipi)->ipi_hash_lock, \ | #define INP_HASH_WLOCK_ASSERT(ipi) mtx_assert(&(ipi)->ipi_hash_lock, MA_OWNED); | ||||
RA_LOCKED) | |||||
#define INP_HASH_WLOCK_ASSERT(ipi) rw_assert(&(ipi)->ipi_hash_lock, \ | |||||
RA_WLOCKED) | |||||
#define INP_GROUP_LOCK_INIT(ipg, d) mtx_init(&(ipg)->ipg_lock, (d), NULL, \ | #define INP_GROUP_LOCK_INIT(ipg, d) mtx_init(&(ipg)->ipg_lock, (d), NULL, \ | ||||
MTX_DEF | MTX_DUPOK) | MTX_DEF | MTX_DUPOK) | ||||
#define INP_GROUP_LOCK_DESTROY(ipg) mtx_destroy(&(ipg)->ipg_lock) | #define INP_GROUP_LOCK_DESTROY(ipg) mtx_destroy(&(ipg)->ipg_lock) | ||||
#define INP_GROUP_LOCK(ipg) mtx_lock(&(ipg)->ipg_lock) | #define INP_GROUP_LOCK(ipg) mtx_lock(&(ipg)->ipg_lock) | ||||
#define INP_GROUP_LOCK_ASSERT(ipg) mtx_assert(&(ipg)->ipg_lock, MA_OWNED) | #define INP_GROUP_LOCK_ASSERT(ipg) mtx_assert(&(ipg)->ipg_lock, MA_OWNED) | ||||
#define INP_GROUP_UNLOCK(ipg) mtx_unlock(&(ipg)->ipg_lock) | #define INP_GROUP_UNLOCK(ipg) mtx_unlock(&(ipg)->ipg_lock) | ||||
▲ Show 20 Lines • Show All 211 Lines • Show Last 20 Lines |
While here, please update/fix the locking annotations on these lines. (For example, the [w] and [r] are clearly backwards.)