Differential D1892 Diff 4110 sys/x86/iommu/intel_dmar.h

Changeset View

Standalone View

sys/x86/iommu/intel_dmar.h

/*-		/*-
* Copyright (c) 2013 The FreeBSD Foundation		* Copyright (c) 2013-2015 The FreeBSD Foundation
* All rights reserved.		* All rights reserved.
*		*
* This software was developed by Konstantin Belousov <kib@FreeBSD.org>		* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
* under sponsorship from the FreeBSD Foundation.		* under sponsorship from the FreeBSD Foundation.
*		*
* Redistribution and use in source and binary forms, with or without		* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions		* modification, are permitted provided that the following conditions
* are met:		* are met:
▲ Show 20 Lines • Show All 169 Lines • ▼ Show 20 Lines	struct dmar_unit {
volatile uint32_t inv_waitd_seq_hw; /* hw writes there on wait		volatile uint32_t inv_waitd_seq_hw; /* hw writes there on wait
descr completion */		descr completion */
uint64_t inv_waitd_seq_hw_phys;		uint64_t inv_waitd_seq_hw_phys;
uint32_t inv_waitd_seq; /* next sequence number to use for wait descr */		uint32_t inv_waitd_seq; /* next sequence number to use for wait descr */
u_int inv_waitd_gen; /* seq number generation AKA seq overflows */		u_int inv_waitd_gen; /* seq number generation AKA seq overflows */
u_int inv_seq_waiters; /* count of waiters for seq */		u_int inv_seq_waiters; /* count of waiters for seq */
u_int inv_queue_full; /* informational counter */		u_int inv_queue_full; /* informational counter */

		/* IR */
		int ir_enabled;
		vm_paddr_t irt_phys;
		dmar_irte_t *irt;
		u_int irte_cnt;
		vmem_t *irtids;

/* Delayed freeing of map entries queue processing */		/* Delayed freeing of map entries queue processing */
struct dmar_map_entries_tailq tlb_flush_entries;		struct dmar_map_entries_tailq tlb_flush_entries;
struct task qi_task;		struct task qi_task;
struct taskqueue *qi_taskqueue;		struct taskqueue *qi_taskqueue;

/* Busdma delayed map load */		/* Busdma delayed map load */
struct task dmamap_load_task;		struct task dmamap_load_task;
TAILQ_HEAD(, bus_dmamap_dmar) delayed_maps;		TAILQ_HEAD(, bus_dmamap_dmar) delayed_maps;
struct taskqueue *delayed_taskqueue;		struct taskqueue *delayed_taskqueue;

		int dma_enabled;
};		};

#define DMAR_LOCK(dmar) mtx_lock(&(dmar)->lock)		#define DMAR_LOCK(dmar) mtx_lock(&(dmar)->lock)
#define DMAR_UNLOCK(dmar) mtx_unlock(&(dmar)->lock)		#define DMAR_UNLOCK(dmar) mtx_unlock(&(dmar)->lock)
#define DMAR_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->lock, MA_OWNED)		#define DMAR_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->lock, MA_OWNED)

#define DMAR_FAULT_LOCK(dmar) mtx_lock_spin(&(dmar)->fault_lock)		#define DMAR_FAULT_LOCK(dmar) mtx_lock_spin(&(dmar)->fault_lock)
#define DMAR_FAULT_UNLOCK(dmar) mtx_unlock_spin(&(dmar)->fault_lock)		#define DMAR_FAULT_UNLOCK(dmar) mtx_unlock_spin(&(dmar)->fault_lock)
#define DMAR_FAULT_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->fault_lock, MA_OWNED)		#define DMAR_FAULT_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->fault_lock, MA_OWNED)

#define DMAR_IS_COHERENT(dmar) (((dmar)->hw_ecap & DMAR_ECAP_C) != 0)		#define DMAR_IS_COHERENT(dmar) (((dmar)->hw_ecap & DMAR_ECAP_C) != 0)
#define DMAR_HAS_QI(dmar) (((dmar)->hw_ecap & DMAR_ECAP_QI) != 0)		#define DMAR_HAS_QI(dmar) (((dmar)->hw_ecap & DMAR_ECAP_QI) != 0)
		#define DMAR_X2APIC(dmar) \
		(x2apic_mode && ((dmar)->hw_ecap & DMAR_ECAP_EIM) != 0)

/* Barrier ids */		/* Barrier ids */
#define DMAR_BARRIER_RMRR 0		#define DMAR_BARRIER_RMRR 0
#define DMAR_BARRIER_USEQ 1		#define DMAR_BARRIER_USEQ 1

struct dmar_unit *dmar_find(device_t dev);		struct dmar_unit *dmar_find(device_t dev);
		struct dmar_unit dmar_find_hpet(device_t dev, uint16_t rid);
		struct dmar_unit dmar_find_ioapic(u_int apic_id, uint16_t rid);

u_int dmar_nd2mask(u_int nd);		u_int dmar_nd2mask(u_int nd);
bool dmar_pglvl_supported(struct dmar_unit *unit, int pglvl);		bool dmar_pglvl_supported(struct dmar_unit *unit, int pglvl);
int ctx_set_agaw(struct dmar_ctx *ctx, int mgaw);		int ctx_set_agaw(struct dmar_ctx *ctx, int mgaw);
int dmar_maxaddr2mgaw(struct dmar_unit* unit, dmar_gaddr_t maxaddr,		int dmar_maxaddr2mgaw(struct dmar_unit* unit, dmar_gaddr_t maxaddr,
bool allow_less);		bool allow_less);
vm_pindex_t pglvl_max_pages(int pglvl);		vm_pindex_t pglvl_max_pages(int pglvl);
int ctx_is_sp_lvl(struct dmar_ctx *ctx, int lvl);		int ctx_is_sp_lvl(struct dmar_ctx *ctx, int lvl);
Show All 10 Lines
int dmar_inv_ctx_glob(struct dmar_unit *unit);		int dmar_inv_ctx_glob(struct dmar_unit *unit);
int dmar_inv_iotlb_glob(struct dmar_unit *unit);		int dmar_inv_iotlb_glob(struct dmar_unit *unit);
int dmar_flush_write_bufs(struct dmar_unit *unit);		int dmar_flush_write_bufs(struct dmar_unit *unit);
void dmar_flush_pte_to_ram(struct dmar_unit unit, dmar_pte_t dst);		void dmar_flush_pte_to_ram(struct dmar_unit unit, dmar_pte_t dst);
void dmar_flush_ctx_to_ram(struct dmar_unit unit, dmar_ctx_entry_t dst);		void dmar_flush_ctx_to_ram(struct dmar_unit unit, dmar_ctx_entry_t dst);
void dmar_flush_root_to_ram(struct dmar_unit unit, dmar_root_entry_t dst);		void dmar_flush_root_to_ram(struct dmar_unit unit, dmar_root_entry_t dst);
int dmar_enable_translation(struct dmar_unit *unit);		int dmar_enable_translation(struct dmar_unit *unit);
int dmar_disable_translation(struct dmar_unit *unit);		int dmar_disable_translation(struct dmar_unit *unit);
		int dmar_load_irt_ptr(struct dmar_unit *unit);
		int dmar_enable_ir(struct dmar_unit *unit);
		int dmar_disable_ir(struct dmar_unit *unit);
bool dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id);		bool dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id);
void dmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id);		void dmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id);

int dmar_fault_intr(void *arg);		int dmar_fault_intr(void *arg);
void dmar_enable_fault_intr(struct dmar_unit *unit);		void dmar_enable_fault_intr(struct dmar_unit *unit);
void dmar_disable_fault_intr(struct dmar_unit *unit);		void dmar_disable_fault_intr(struct dmar_unit *unit);
int dmar_init_fault_log(struct dmar_unit *unit);		int dmar_init_fault_log(struct dmar_unit *unit);
void dmar_fini_fault_log(struct dmar_unit *unit);		void dmar_fini_fault_log(struct dmar_unit *unit);

int dmar_qi_intr(void *arg);		int dmar_qi_intr(void *arg);
void dmar_enable_qi_intr(struct dmar_unit *unit);		void dmar_enable_qi_intr(struct dmar_unit *unit);
void dmar_disable_qi_intr(struct dmar_unit *unit);		void dmar_disable_qi_intr(struct dmar_unit *unit);
int dmar_init_qi(struct dmar_unit *unit);		int dmar_init_qi(struct dmar_unit *unit);
void dmar_fini_qi(struct dmar_unit *unit);		void dmar_fini_qi(struct dmar_unit *unit);
void dmar_qi_invalidate_locked(struct dmar_ctx *ctx, dmar_gaddr_t start,		void dmar_qi_invalidate_locked(struct dmar_ctx *ctx, dmar_gaddr_t start,
dmar_gaddr_t size, struct dmar_qi_genseq *pseq);		dmar_gaddr_t size, struct dmar_qi_genseq *pseq);
void dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit);		void dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit);
void dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit);		void dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit);
		void dmar_qi_invalidate_iec_glob(struct dmar_unit *unit);
		void dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt);

vm_object_t ctx_get_idmap_pgtbl(struct dmar_ctx *ctx, dmar_gaddr_t maxaddr);		vm_object_t ctx_get_idmap_pgtbl(struct dmar_ctx *ctx, dmar_gaddr_t maxaddr);
void put_idmap_pgtbl(vm_object_t obj);		void put_idmap_pgtbl(vm_object_t obj);
int ctx_map_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,		int ctx_map_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
vm_page_t *ma, uint64_t pflags, int flags);		vm_page_t *ma, uint64_t pflags, int flags);
int ctx_unmap_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,		int ctx_unmap_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
int flags);		int flags);
void ctx_flush_iotlb_sync(struct dmar_ctx *ctx, dmar_gaddr_t base,		void ctx_flush_iotlb_sync(struct dmar_ctx *ctx, dmar_gaddr_t base,
Show All 10 Lines
struct dmar_ctx dmar_find_ctx_locked(struct dmar_unit dmar, uint16_t rid);		struct dmar_ctx dmar_find_ctx_locked(struct dmar_unit dmar, uint16_t rid);
void dmar_ctx_unload_entry(struct dmar_map_entry *entry, bool free);		void dmar_ctx_unload_entry(struct dmar_map_entry *entry, bool free);
void dmar_ctx_unload(struct dmar_ctx *ctx,		void dmar_ctx_unload(struct dmar_ctx *ctx,
struct dmar_map_entries_tailq *entries, bool cansleep);		struct dmar_map_entries_tailq *entries, bool cansleep);
void dmar_ctx_free_entry(struct dmar_map_entry *entry, bool free);		void dmar_ctx_free_entry(struct dmar_map_entry *entry, bool free);

int dmar_init_busdma(struct dmar_unit *unit);		int dmar_init_busdma(struct dmar_unit *unit);
void dmar_fini_busdma(struct dmar_unit *unit);		void dmar_fini_busdma(struct dmar_unit *unit);
		device_t dmar_get_requester(device_t dev, uint16_t *rid);

void dmar_gas_init_ctx(struct dmar_ctx *ctx);		void dmar_gas_init_ctx(struct dmar_ctx *ctx);
void dmar_gas_fini_ctx(struct dmar_ctx *ctx);		void dmar_gas_fini_ctx(struct dmar_ctx *ctx);
struct dmar_map_entry dmar_gas_alloc_entry(struct dmar_ctx ctx, u_int flags);		struct dmar_map_entry dmar_gas_alloc_entry(struct dmar_ctx ctx, u_int flags);
void dmar_gas_free_entry(struct dmar_ctx ctx, struct dmar_map_entry entry);		void dmar_gas_free_entry(struct dmar_ctx ctx, struct dmar_map_entry entry);
void dmar_gas_free_space(struct dmar_ctx ctx, struct dmar_map_entry entry);		void dmar_gas_free_space(struct dmar_ctx ctx, struct dmar_map_entry entry);
int dmar_gas_map(struct dmar_ctx ctx, const struct bus_dma_tag_common common,		int dmar_gas_map(struct dmar_ctx ctx, const struct bus_dma_tag_common common,
dmar_gaddr_t size, u_int eflags, u_int flags, vm_page_t *ma,		dmar_gaddr_t size, u_int eflags, u_int flags, vm_page_t *ma,
struct dmar_map_entry **res);		struct dmar_map_entry **res);
void dmar_gas_free_region(struct dmar_ctx ctx, struct dmar_map_entry entry);		void dmar_gas_free_region(struct dmar_ctx ctx, struct dmar_map_entry entry);
int dmar_gas_map_region(struct dmar_ctx ctx, struct dmar_map_entry entry,		int dmar_gas_map_region(struct dmar_ctx ctx, struct dmar_map_entry entry,
u_int eflags, u_int flags, vm_page_t *ma);		u_int eflags, u_int flags, vm_page_t *ma);
int dmar_gas_reserve_region(struct dmar_ctx *ctx, dmar_gaddr_t start,		int dmar_gas_reserve_region(struct dmar_ctx *ctx, dmar_gaddr_t start,
dmar_gaddr_t end);		dmar_gaddr_t end);

void dmar_ctx_parse_rmrr(struct dmar_ctx *ctx, device_t dev,		void dmar_ctx_parse_rmrr(struct dmar_ctx *ctx, device_t dev,
struct dmar_map_entries_tailq *rmrr_entries);		struct dmar_map_entries_tailq *rmrr_entries);
int dmar_instantiate_rmrr_ctxs(struct dmar_unit *dmar);		int dmar_instantiate_rmrr_ctxs(struct dmar_unit *dmar);

void dmar_quirks_post_ident(struct dmar_unit *dmar);		void dmar_quirks_post_ident(struct dmar_unit *dmar);
void dmar_quirks_pre_use(struct dmar_unit *dmar);		void dmar_quirks_pre_use(struct dmar_unit *dmar);

		int dmar_init_irt(struct dmar_unit *unit);
		void dmar_fini_irt(struct dmar_unit *unit);

#define DMAR_GM_CANWAIT 0x0001		#define DMAR_GM_CANWAIT 0x0001
#define DMAR_GM_CANSPLIT 0x0002		#define DMAR_GM_CANSPLIT 0x0002

#define DMAR_PGF_WAITOK 0x0001		#define DMAR_PGF_WAITOK 0x0001
#define DMAR_PGF_ZERO 0x0002		#define DMAR_PGF_ZERO 0x0002
#define DMAR_PGF_ALLOC 0x0004		#define DMAR_PGF_ALLOC 0x0004
#define DMAR_PGF_NOALLOC 0x0008		#define DMAR_PGF_NOALLOC 0x0008
#define DMAR_PGF_OBJL 0x0010		#define DMAR_PGF_OBJL 0x0010
▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines
}		}

/*		/*
* dmar_pte_store and dmar_pte_clear ensure that on i386, 32bit writes		* dmar_pte_store and dmar_pte_clear ensure that on i386, 32bit writes
* are issued in the correct order. For store, the lower word,		* are issued in the correct order. For store, the lower word,
* containing the P or R and W bits, is set only after the high word		* containing the P or R and W bits, is set only after the high word
* is written. For clear, the P bit is cleared first, then the high		* is written. For clear, the P bit is cleared first, then the high
* word is cleared.		* word is cleared.
		*
		* dmar_pte_update updates the pte. For amd64, the update is atomic.
		* For i386, it first disables the entry by clearing the word
		* containing the P bit, and then defer to dmar_pte_store. The locked
		* cmpxchg8b is probably available on any machine having DMAR support,
		* but interrupt translation table may be mapped uncached.
*/		*/
static inline void		static inline void
dmar_pte_store(volatile uint64_t *dst, uint64_t val)		dmar_pte_store1(volatile uint64_t *dst, uint64_t val)
{		{

KASSERT(*dst == 0, ("used pte %p oldval %jx newval %jx",
dst, (uintmax_t)*dst, (uintmax_t)val));
#ifdef __i386__		#ifdef __i386__
volatile uint32_t *p;		volatile uint32_t *p;
uint32_t hi, lo;		uint32_t hi, lo;

hi = val >> 32;		hi = val >> 32;
lo = val;		lo = val;
p = (volatile uint32_t *)dst;		p = (volatile uint32_t *)dst;
*(p + 1) = hi;		*(p + 1) = hi;
*p = lo;		*p = lo;
#else		#else
*dst = val;		*dst = val;
#endif		#endif
		}

		static inline void
		dmar_pte_store(volatile uint64_t *dst, uint64_t val)
		{

		KASSERT(*dst == 0, ("used pte %p oldval %jx newval %jx",
		dst, (uintmax_t)*dst, (uintmax_t)val));
		dmar_pte_store1(dst, val);
		}

		static inline void
		dmar_pte_update(volatile uint64_t *dst, uint64_t val)
		{

		#ifdef __i386__
		volatile uint32_t *p;

		p = (volatile uint32_t *)dst;
		*p = 0;
		#endif
		dmar_pte_store1(dst, val);
}		}

static inline void		static inline void
dmar_pte_clear(volatile uint64_t *dst)		dmar_pte_clear(volatile uint64_t *dst)
{		{
#ifdef __i386__		#ifdef __i386__
volatile uint32_t *p;		volatile uint32_t *p;

Show All 32 Lines