Differential D23570 Diff 68159 sys/amd64/include/counter.h

Changeset View

Standalone View

sys/amd64/include/counter.h

	Show All 27 Lines
	* $FreeBSD$			* $FreeBSD$
	*/			*/

	#ifndef __MACHINE_COUNTER_H__			#ifndef __MACHINE_COUNTER_H__
	#define __MACHINE_COUNTER_H__			#define __MACHINE_COUNTER_H__

	#include <sys/pcpu.h>			#include <sys/pcpu.h>

	#define EARLY_COUNTER &temp_bsp_pcpu.pc_early_dummy_counter			#define EARLY_COUNTER (void *)__offsetof(struct pcpu, pc_early_dummy_counter)

	#define counter_enter() do {} while (0)			#define counter_enter() do {} while (0)
	#define counter_exit() do {} while (0)			#define counter_exit() do {} while (0)

	#ifdef IN_SUBR_COUNTER_C			#ifdef IN_SUBR_COUNTER_C
	static inline uint64_t			static inline uint64_t
	counter_u64_read_one(counter_u64_t c, int cpu)			counter_u64_read_one(counter_u64_t c, int cpu)
	{			{

				MPASS(c != EARLY_COUNTER);
	return (*zpcpu_get_cpu(c, cpu));			return (*zpcpu_get_cpu(c, cpu));
	}			}

	static inline uint64_t			static inline uint64_t
	counter_u64_fetch_inline(uint64_t *c)			counter_u64_fetch_inline(uint64_t *c)
	{			{
	uint64_t r;			uint64_t r;
	int cpu;			int cpu;

	r = 0;			r = 0;
	CPU_FOREACH(cpu)			CPU_FOREACH(cpu)
	r += counter_u64_read_one(c, cpu);			r += counter_u64_read_one(c, cpu);

	return (r);			return (r);
	}			}

	static void			static void
	counter_u64_zero_one_cpu(void *arg)			counter_u64_zero_one_cpu(void *arg)
	{			{
	counter_u64_t c;			counter_u64_t c;

	c = arg;			c = arg;
				MPASS(c != EARLY_COUNTER);
	*(zpcpu_get(c)) = 0;			*(zpcpu_get(c)) = 0;
	}			}

	static inline void			static inline void
	counter_u64_zero_inline(counter_u64_t c)			counter_u64_zero_inline(counter_u64_t c)
	{			{

	smp_rendezvous(smp_no_rendezvous_barrier, counter_u64_zero_one_cpu,			smp_rendezvous(smp_no_rendezvous_barrier, counter_u64_zero_one_cpu,
	smp_no_rendezvous_barrier, c);			smp_no_rendezvous_barrier, c);
	}			}
	#endif			#endif

	#define counter_u64_add_protected(c, i) counter_u64_add(c, i)			#define counter_u64_add_protected(c, i) counter_u64_add(c, i)

	static inline void			static inline void
	counter_u64_add(counter_u64_t c, int64_t inc)			counter_u64_add(counter_u64_t c, int64_t inc)
	{			{

	KASSERT(IS_BSP() \|\| c != EARLY_COUNTER, ("EARLY_COUNTER used on AP"));			KASSERT(IS_BSP() \|\| c != EARLY_COUNTER, ("EARLY_COUNTER used on AP"));
	__asm __volatile("addq\t%1,%%gs:(%0)"			__asm __volatile("addq\t%1,%%gs:(%0)"
	:			:
	: "r" ((char )c - (char )&__pcpu[0]), "ri" (inc)			: "r" (c), "ri" (inc)
	: "memory", "cc");			: "memory", "cc");
	}			}

	#endif /* ! __MACHINE_COUNTER_H__ */			#endif /* ! __MACHINE_COUNTER_H__ */