Differential D29467 Diff 86438 sys/compat/linuxkpi/common/src/linux_rcu.c

Changeset View

Standalone View

sys/compat/linuxkpi/common/src/linux_rcu.c

/*-		/*-
* Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io)		* Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io)
* Copyright (c) 2017-2020 Hans Petter Selasky (hselasky@freebsd.org)		* Copyright (c) 2017-2021 Hans Petter Selasky (hselasky@freebsd.org)
* All rights reserved.		* All rights reserved.
*		*
* Redistribution and use in source and binary forms, with or without		* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions		* modification, are permitted provided that the following conditions
* are met:		* are met:
* 1. Redistributions of source code must retain the above copyright		* 1. Redistributions of source code must retain the above copyright
* notice unmodified, this list of conditions, and the following		* notice unmodified, this list of conditions, and the following
* disclaimer.		* disclaimer.
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines
* Verify that "struct rcu_head" is big enough to hold "struct		* Verify that "struct rcu_head" is big enough to hold "struct
* callback_head". This has been done to avoid having to add special		* callback_head". This has been done to avoid having to add special
* compile flags for including ck_epoch.h to all clients of the		* compile flags for including ck_epoch.h to all clients of the
* LinuxKPI.		* LinuxKPI.
*/		*/
CTASSERT(sizeof(struct rcu_head) == sizeof(struct callback_head));		CTASSERT(sizeof(struct rcu_head) == sizeof(struct callback_head));

/*		/*
		* Verify that "rcu_section[0]" has the same size as
		* "ck_epoch_section_t". This has been done to avoid having to add
		* special compile flags for including ck_epoch.h to all clients of
		* the LinuxKPI.
		*/
		CTASSERT(sizeof(((struct task_struct *)0)->rcu_section[0] ==
		kibUnsubmitted Done Inline Actions New code should prefer _Static_assert() over CTASSERT(). kib: New code should prefer _Static_assert() over CTASSERT().
		hselaskyAuthorUnsubmitted Done Inline Actions The style in LinuxKPI is CTASSERT() for now. hselasky: The style in LinuxKPI is CTASSERT() for now.
		sizeof(ck_epoch_section_t)));

		/*
* Verify that "epoch_record" is at beginning of "struct		* Verify that "epoch_record" is at beginning of "struct
* linux_epoch_record":		* linux_epoch_record":
*/		*/
CTASSERT(offsetof(struct linux_epoch_record, epoch_record) == 0);		CTASSERT(offsetof(struct linux_epoch_record, epoch_record) == 0);

CTASSERT(TS_RCU_TYPE_MAX == RCU_TYPE_MAX);		CTASSERT(TS_RCU_TYPE_MAX == RCU_TYPE_MAX);

static ck_epoch_t linux_epoch[RCU_TYPE_MAX];		static ck_epoch_t linux_epoch[RCU_TYPE_MAX];
▲ Show 20 Lines • Show All 87 Lines • ▼ Show 20 Lines	linux_rcu_read_lock(unsigned type)
struct linux_epoch_record *record;		struct linux_epoch_record *record;
struct task_struct *ts;		struct task_struct *ts;

MPASS(type < RCU_TYPE_MAX);		MPASS(type < RCU_TYPE_MAX);

if (RCU_SKIP())		if (RCU_SKIP())
return;		return;

		ts = current;

		/* assert valid refcount */
		kibUnsubmitted Done Inline Actions You should assert that the increment does not overflow. kib: You should assert that the increment does not overflow.
		MPASS(ts->rcu_recurse[type] != INT_MAX);

		if (++(ts->rcu_recurse[type]) != 1)
		return;

/*		/*
* Pin thread to current CPU so that the unlock code gets the		* Pin thread to current CPU so that the unlock code gets the
* same per-CPU epoch record:		* same per-CPU epoch record:
*/		*/
sched_pin();		sched_pin();

record = &DPCPU_GET(linux_epoch_record[type]);		record = &DPCPU_GET(linux_epoch_record[type]);
ts = current;

/*		/*
* Use a critical section to prevent recursion inside		* Use a critical section to prevent recursion inside
* ck_epoch_begin(). Else this function supports recursion.		* ck_epoch_begin(). Else this function supports recursion.
*/		*/
critical_enter();		critical_enter();
ck_epoch_begin(&record->epoch_record, NULL);		ck_epoch_begin(&record->epoch_record,
ts->rcu_recurse[type]++;		(ck_epoch_section_t *)&ts->rcu_section[type]);
if (ts->rcu_recurse[type] == 1)
TAILQ_INSERT_TAIL(&record->ts_head, ts, rcu_entry[type]);		TAILQ_INSERT_TAIL(&record->ts_head, ts, rcu_entry[type]);
		markjUnsubmitted Done Inline Actions Is this line too long? markj: Is this line too long?
critical_exit();		critical_exit();
}		}

void		void
linux_rcu_read_unlock(unsigned type)		linux_rcu_read_unlock(unsigned type)
{		{
struct linux_epoch_record *record;		struct linux_epoch_record *record;
struct task_struct *ts;		struct task_struct *ts;

MPASS(type < RCU_TYPE_MAX);		MPASS(type < RCU_TYPE_MAX);

if (RCU_SKIP())		if (RCU_SKIP())
return;		return;

record = &DPCPU_GET(linux_epoch_record[type]);
ts = current;		ts = current;

		/* assert valid refcount */
		kibUnsubmitted Done Inline Actions You need to assert that rcu_recurse[type] > 0 before decrement. kib: You need to assert that rcu_recurse[type] > 0 before decrement.
		MPASS(ts->rcu_recurse[type] > 0);

		if (--(ts->rcu_recurse[type]) != 0)
		return;

		record = &DPCPU_GET(linux_epoch_record[type]);

/*		/*
* Use a critical section to prevent recursion inside		* Use a critical section to prevent recursion inside
* ck_epoch_end(). Else this function supports recursion.		* ck_epoch_end(). Else this function supports recursion.
*/		*/
critical_enter();		critical_enter();
ck_epoch_end(&record->epoch_record, NULL);		ck_epoch_end(&record->epoch_record,
ts->rcu_recurse[type]--;		(ck_epoch_section_t *)&ts->rcu_section[type]);
		kibUnsubmitted Done Inline Actions This container_of() was quite puzzling to me until I realized that it is in fact a type cast. See my note near rcu_section[]. kib: This container_of() was quite puzzling to me until I realized that it is in fact a type cast.
		hselaskyAuthorUnsubmitted Done Inline Actions Changed into a regular cast. The problem is that EPOCH header files do not live under sys/ and require special C-flags currently. hselasky: Changed into a regular cast. The problem is that EPOCH header files do not live under sys/ and…
if (ts->rcu_recurse[type] == 0)
TAILQ_REMOVE(&record->ts_head, ts, rcu_entry[type]);		TAILQ_REMOVE(&record->ts_head, ts, rcu_entry[type]);
critical_exit();		critical_exit();

sched_unpin();		sched_unpin();
}		}

static void		static void
linux_synchronize_rcu_cb(ck_epoch_t epoch __unused, ck_epoch_record_t epoch_record, void *arg __unused)		linux_synchronize_rcu_cb(ck_epoch_t epoch __unused, ck_epoch_record_t epoch_record, void *arg __unused)
{		{
▲ Show 20 Lines • Show All 178 Lines • Show Last 20 Lines