Eliminate lock recursion from turnstiles.
ClosedPublic
Actions

Authored by jeff on Dec 10 2019, 6:16 AM.

Details

Reviewers

jhb
kib
mav

Commits

rS355782: schedlock 3/4

Summary

This avoids the one place we recursively lock turnstiles. In highly contended workloads the recursive locking actually imposes a performance penalty. We trade failed atomic cmpsets and multiple modifications to the lock cacheline for a few pointer comparisons.

Diff Detail

Repository

rS FreeBSD src repository - subversion

Lint

Lint Not Applicable

Unit

Tests Not Applicable

Event Timeline

jeff created this revision.Dec 10 2019, 6:16 AM

Harbormaster completed remote builds in B28055: Diff 65449.Dec 10 2019, 6:16 AM

jeff edited the summary of this revision. (Show Details)Dec 10 2019, 6:21 AM

jeff added reviewers: jhb, kib, mav.

jeff set the repository for this revision to rS FreeBSD src repository - subversion.

Herald added a subscriber: imp. · View Herald TranscriptDec 10 2019, 6:21 AM

kib accepted this revision.Dec 10 2019, 2:42 PM

This revision is now accepted and ready to land.Dec 10 2019, 2:42 PM

Closed by commit rS355782: schedlock 3/4 (authored by jeff). · Explain WhyDec 15 2019, 9:20 PM

This revision was automatically updated to reflect the committed changes.

jeff added a commit: rS355782: schedlock 3/4.

Revision Contents
Changeset List

Path

Size

head/

sys/

kern/

subr_turnstile.c

50 lines

Diff 65690

View Options

head/sys/kern/subr_turnstile.c

Show First 20 Lines • Show All 110 Lines • ▼ Show 20 Lines	do_buff_decode(u_int8_t *databuf, size_t len,
int done = 0;		int done = 0;
static u_char mask[] = {0, 0x01, 0x03, 0x07, 0x0f,		static u_char mask[] = {0, 0x01, 0x03, 0x07, 0x0f,
0x1f, 0x3f, 0x7f, 0xff};		0x1f, 0x3f, 0x7f, 0xff};
int value;		int value;
u_char *base = databuf;		u_char *base = databuf;
char *intendp;		char *intendp;
char letter;		char letter;
char field_name[80];		char field_name[80];

# define ARG_PUT(ARG) \		# define ARG_PUT(ARG) \
		cemUnsubmitted Done Inline Actions 99% of the time assert(3) will just compile out in userspace. Does this library have any other use of assert(3) (i.e. does it expect anybody to actually compile with -DDEBUG or whatever)? cem: 99% of the time assert(3) will just compile out in userspace. Does this library have any other…
		ngieAuthorUnsubmitted Done Inline Actions It doesn't. I'll remove the asserts since they're inconsistent, but will add this to a list of items that need to be fixed in libcam, from an API perspective. ngie: It doesn't. I'll remove the asserts since they're inconsistent, but will add this to a list of…
		markjUnsubmitted Done Inline Actions assert(3) is opt-out. You need to compile with -DNDEBUG to disable them. We only do that in bsd.{lib,prog}.mk if WITHOUT_ASSERT_DEBUG is set. So they're on by default AFAIK. markj: assert(3) is opt-out. You need to compile with -DNDEBUG to disable them. We only do that in bsd.
		cemUnsubmitted Done Inline Actions For some (incorrect) reason I thought -DNDEBUG got inserted automatically at -O1 and higher. It looks like that is not the case for compilers. However, toolchains (autoconf, CMake, FreeBSD build process) may insert the flag automatically for optimized builds. cem: For some (incorrect) reason I thought -DNDEBUG got inserted automatically at -O1 and higher.
do \		do \
{ \		{ \
if (!suppress) \		if (!suppress) \
{ \		{ \
if (arg_put) \		if (arg_put) \
(*arg_put)(puthook, (letter == 't' ? \		(*arg_put)(puthook, (letter == 't' ? \
'b' : letter), \		'b' : letter), \
(void *)((long)(ARG)), width, \		(void *)((long)(ARG)), width, \
field_name); \		field_name); \
else \		else \
(va_arg(ap, int )) = (ARG); \		(va_arg(ap, int )) = (ARG); \
		cemUnsubmitted Done Inline Actions Wowza. cem: Wowza.
		ngieAuthorUnsubmitted Done Inline Actions Yup.. ngie: Yup..
assigned++; \		assigned++; \
} \		} \
field_name[0] = 0; \		field_name[0] = 0; \
suppress = 0; \		suppress = 0; \
} while (0)		} while (0)

u_char bits = 0; /* For bit fields */		u_char bits = 0; /* For bit fields */
int shift = 0; /* Bits already shifted out */		int shift = 0; /* Bits already shifted out */
▲ Show 20 Lines • Show All 517 Lines • ▼ Show 20 Lines	#endif

return encoded;		return encoded;
}		}

int		int
csio_decode(struct ccb_scsiio csio, const char fmt, ...)		csio_decode(struct ccb_scsiio csio, const char fmt, ...)
{		{
va_list ap;		va_list ap;
		int retval;

va_start(ap, fmt);		va_start(ap, fmt);

return(do_buff_decode(csio->data_ptr, (size_t)csio->dxfer_len,		retval = do_buff_decode(csio->data_ptr, (size_t)csio->dxfer_len, 0, 0,
0, 0, fmt, ap));		fmt, ap);

		va_end(ap);

		return (retval);
}		}

int		int
csio_decode_visit(struct ccb_scsiio csio, const char fmt,		csio_decode_visit(struct ccb_scsiio csio, const char fmt,
void (arg_put)(void , int, void , int, char ),		void (arg_put)(void , int, void , int, char ),
void *puthook)		void *puthook)
{		{
va_list ap;		va_list ap;
		int retval;

/*		/*
* We need some way to output things; we can't do it without		* We need some way to output things; we can't do it without
* the arg_put function.		* the arg_put function.
*/		*/
if (arg_put == NULL)		if (arg_put == NULL)
return(-1);		return (-1);

bzero(&ap, sizeof(ap));		bzero(&ap, sizeof(ap));
		cemUnsubmitted Done Inline Actions This is probably not a portable assumption. cem: This is probably not a portable assumption.
		ngieAuthorUnsubmitted Done Inline Actions Agreed. ngie: Agreed.

return(do_buff_decode(csio->data_ptr, (size_t)csio->dxfer_len,		retval = do_buff_decode(csio->data_ptr, (size_t)csio->dxfer_len,
arg_put, puthook, fmt, ap));		arg_put, puthook, fmt, ap);

		va_end(ap);
		cemUnsubmitted Done Inline Actions No va_start here. cem: No va_start here.

		return (retval);
}		}

int		int
buff_decode(u_int8_t buff, size_t len, const char fmt, ...)		buff_decode(u_int8_t buff, size_t len, const char fmt, ...)
{		{
va_list ap;		va_list ap;
		int retval;

va_start(ap, fmt);		va_start(ap, fmt);

return(do_buff_decode(buff, len, 0, 0, fmt, ap));		retval = do_buff_decode(buff, len, 0, 0, fmt, ap);

		va_end(ap);

		return (retval);
}		}

int		int
buff_decode_visit(u_int8_t buff, size_t len, const char fmt,		buff_decode_visit(u_int8_t buff, size_t len, const char fmt,
void (arg_put)(void , int, void , int, char ),		void (arg_put)(void , int, void , int, char ),
void *puthook)		void *puthook)
{		{
va_list ap;		va_list ap;
Show All 19 Lines	csio_build(struct ccb_scsiio csio, u_int8_t data_ptr, u_int32_t dxfer_len,
u_int32_t flags, int retry_count, int timeout, const char *cmd_spec,		u_int32_t flags, int retry_count, int timeout, const char *cmd_spec,
...)		...)
{		{
size_t cmdlen;		size_t cmdlen;
int retval;		int retval;
va_list ap;		va_list ap;

if (csio == NULL)		if (csio == NULL)
return(0);		return (0);

bzero(csio, sizeof(struct ccb_scsiio));		bzero(csio, sizeof(struct ccb_scsiio));

va_start(ap, cmd_spec);		va_start(ap, cmd_spec);

if ((retval = do_encode(csio->cdb_io.cdb_bytes, SCSI_MAX_CDBLEN,		if ((retval = do_encode(csio->cdb_io.cdb_bytes, SCSI_MAX_CDBLEN,
&cmdlen, NULL, NULL, cmd_spec, ap)) == -1)		&cmdlen, NULL, NULL, cmd_spec, ap)) == -1)
return(retval);		goto done;

cam_fill_csio(csio,		cam_fill_csio(csio,
/* retries */ retry_count,		/* retries */ retry_count,
/* cbfcnp */ NULL,		/* cbfcnp */ NULL,
/* flags */ flags,		/* flags */ flags,
/* tag_action */ MSG_SIMPLE_Q_TAG,		/* tag_action */ MSG_SIMPLE_Q_TAG,
/* data_ptr */ data_ptr,		/* data_ptr */ data_ptr,
/* dxfer_len */ dxfer_len,		/* dxfer_len */ dxfer_len,
/* sense_len */ SSD_FULL_SIZE,		/* sense_len */ SSD_FULL_SIZE,
/* cdb_len */ cmdlen,		/* cdb_len */ cmdlen,
/* timeout */ timeout ? timeout : 5000);		/* timeout */ timeout ? timeout : 5000);

		done:
		va_end(ap);

return(retval);		return (retval);
}		}

int		int
csio_build_visit(struct ccb_scsiio csio, u_int8_t data_ptr,		csio_build_visit(struct ccb_scsiio csio, u_int8_t data_ptr,
u_int32_t dxfer_len, u_int32_t flags, int retry_count,		u_int32_t dxfer_len, u_int32_t flags, int retry_count,
int timeout, const char *cmd_spec,		int timeout, const char *cmd_spec,
int (arg_get)(void hook, char field_name), void gethook)		int (arg_get)(void hook, char field_name), void gethook)
{		{
Show All 32 Lines	csio_build_visit(struct ccb_scsiio csio, u_int8_t data_ptr,

return(retval);		return(retval);
}		}

int		int
csio_encode(struct ccb_scsiio csio, const char fmt, ...)		csio_encode(struct ccb_scsiio csio, const char fmt, ...)
{		{
va_list ap;		va_list ap;
		int retval;

if (csio == NULL)		if (csio == NULL)
return(0);		return (0);

va_start(ap, fmt);		va_start(ap, fmt);

return(do_encode(csio->data_ptr, csio->dxfer_len, 0, 0, 0, fmt, ap));		retval = do_encode(csio->data_ptr, csio->dxfer_len, 0, 0, 0, fmt, ap);

		va_end(ap);

		return (retval);
}		}

int		int
buff_encode_visit(u_int8_t buff, size_t len, const char fmt,		buff_encode_visit(u_int8_t buff, size_t len, const char fmt,
int (arg_get)(void hook, char field_name), void gethook)		int (arg_get)(void hook, char field_name), void gethook)
{		{
va_list ap;		va_list ap;

Show All 30 Lines

Eliminate lock recursion from turnstiles.ClosedPublicActions