Differential D26683 Diff 78635 head/sys/dev/ntb/ntb_hw/ntb_hw_intel.c

Changeset View

Standalone View

head/sys/dev/ntb/ntb_hw/ntb_hw_intel.c

Show First 20 Lines • Show All 62 Lines • ▼ Show 20 Lines
#include <machine/resource.h>		#include <machine/resource.h>
#include <dev/pci/pcireg.h>		#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>		#include <dev/pci/pcivar.h>
#include <dev/iommu/iommu.h>		#include <dev/iommu/iommu.h>

#include "ntb_hw_intel.h"		#include "ntb_hw_intel.h"
#include "../ntb.h"		#include "../ntb.h"

#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT)		#define MAX_MSIX_INTERRUPTS \
		MAX(MAX(XEON_DB_COUNT, ATOM_DB_COUNT), XEON_GEN3_DB_COUNT)

#define NTB_HB_TIMEOUT 1 /* second */		#define NTB_HB_TIMEOUT 1 /* second */
#define ATOM_LINK_RECOVERY_TIME 500 /* ms */		#define ATOM_LINK_RECOVERY_TIME 500 /* ms */
#define BAR_HIGH_MASK (~((1ull << 12) - 1))		#define BAR_HIGH_MASK (~((1ull << 12) - 1))

#define NTB_MSIX_VER_GUARD 0xaabbccdd		#define NTB_MSIX_VER_GUARD 0xaabbccdd
#define NTB_MSIX_RECEIVED 0xe0f0e0f0		#define NTB_MSIX_RECEIVED 0xe0f0e0f0

/*		/*
* PCI constants could be somewhere more generic, but aren't defined/used in		* PCI constants could be somewhere more generic, but aren't defined/used in
* pci.c.		* pci.c.
*/		*/
#define PCI_MSIX_ENTRY_SIZE 16		#define PCI_MSIX_ENTRY_SIZE 16
#define PCI_MSIX_ENTRY_LOWER_ADDR 0		#define PCI_MSIX_ENTRY_LOWER_ADDR 0
#define PCI_MSIX_ENTRY_UPPER_ADDR 4		#define PCI_MSIX_ENTRY_UPPER_ADDR 4
#define PCI_MSIX_ENTRY_DATA 8		#define PCI_MSIX_ENTRY_DATA 8

enum ntb_device_type {		enum ntb_device_type {
NTB_XEON,		NTB_XEON_GEN1,
		NTB_XEON_GEN3,
NTB_ATOM		NTB_ATOM
};		};

/* ntb_conn_type are hardware numbers, cannot change. */		/* ntb_conn_type are hardware numbers, cannot change. */
enum ntb_conn_type {		enum ntb_conn_type {
NTB_CONN_TRANSPARENT = 0,		NTB_CONN_TRANSPARENT = 0,
NTB_CONN_B2B = 1,		NTB_CONN_B2B = 1,
NTB_CONN_RP = 2,		NTB_CONN_RP = 2,
▲ Show 20 Lines • Show All 234 Lines • ▼ Show 20 Lines
static void print_map_success(struct ntb_softc , struct ntb_pci_bar_info ,		static void print_map_success(struct ntb_softc , struct ntb_pci_bar_info ,
const char *);		const char *);
static int map_mmr_bar(struct ntb_softc ntb, struct ntb_pci_bar_info bar);		static int map_mmr_bar(struct ntb_softc ntb, struct ntb_pci_bar_info bar);
static int map_memory_window_bar(struct ntb_softc *ntb,		static int map_memory_window_bar(struct ntb_softc *ntb,
struct ntb_pci_bar_info *bar);		struct ntb_pci_bar_info *bar);
static void intel_ntb_unmap_pci_bar(struct ntb_softc *ntb);		static void intel_ntb_unmap_pci_bar(struct ntb_softc *ntb);
static int intel_ntb_remap_msix(device_t, uint32_t desired, uint32_t avail);		static int intel_ntb_remap_msix(device_t, uint32_t desired, uint32_t avail);
static int intel_ntb_init_isr(struct ntb_softc *ntb);		static int intel_ntb_init_isr(struct ntb_softc *ntb);
		static int intel_ntb_xeon_gen3_init_isr(struct ntb_softc *ntb);
static int intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb);		static int intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb);
static int intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors);		static int intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors);
static void intel_ntb_teardown_interrupts(struct ntb_softc *ntb);		static void intel_ntb_teardown_interrupts(struct ntb_softc *ntb);
static inline uint64_t intel_ntb_vec_mask(struct ntb_softc *, uint64_t db_vector);		static inline uint64_t intel_ntb_vec_mask(struct ntb_softc *, uint64_t db_vector);
static void intel_ntb_interrupt(struct ntb_softc *, uint32_t vec);		static void intel_ntb_interrupt(struct ntb_softc *, uint32_t vec);
static void ndev_vec_isr(void *arg);		static void ndev_vec_isr(void *arg);
static void ndev_irq_isr(void *arg);		static void ndev_irq_isr(void *arg);
static inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff);		static inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff);
static inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t);		static inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t);
static inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t);		static inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t);
static int intel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors);		static int intel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors);
static void intel_ntb_free_msix_vec(struct ntb_softc *ntb);		static void intel_ntb_free_msix_vec(struct ntb_softc *ntb);
static void intel_ntb_get_msix_info(struct ntb_softc *ntb);		static void intel_ntb_get_msix_info(struct ntb_softc *ntb);
static void intel_ntb_exchange_msix(void *);		static void intel_ntb_exchange_msix(void *);
static struct ntb_hw_info *intel_ntb_get_device_info(uint32_t device_id);		static struct ntb_hw_info *intel_ntb_get_device_info(uint32_t device_id);
static void intel_ntb_detect_max_mw(struct ntb_softc *ntb);		static void intel_ntb_detect_max_mw(struct ntb_softc *ntb);
static int intel_ntb_detect_xeon(struct ntb_softc *ntb);		static int intel_ntb_detect_xeon(struct ntb_softc *ntb);
		static int intel_ntb_detect_xeon_gen3(struct ntb_softc *ntb);
static int intel_ntb_detect_atom(struct ntb_softc *ntb);		static int intel_ntb_detect_atom(struct ntb_softc *ntb);
static int intel_ntb_xeon_init_dev(struct ntb_softc *ntb);		static int intel_ntb_xeon_init_dev(struct ntb_softc *ntb);
		static int intel_ntb_xeon_gen3_init_dev(struct ntb_softc *ntb);
static int intel_ntb_atom_init_dev(struct ntb_softc *ntb);		static int intel_ntb_atom_init_dev(struct ntb_softc *ntb);
static void intel_ntb_teardown_xeon(struct ntb_softc *ntb);		static void intel_ntb_teardown_xeon(struct ntb_softc *ntb);
static void configure_atom_secondary_side_bars(struct ntb_softc *ntb);		static void configure_atom_secondary_side_bars(struct ntb_softc *ntb);
static void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx,		static void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx,
enum ntb_bar regbar);		enum ntb_bar regbar);
static void xeon_set_sbar_base_and_limit(struct ntb_softc *,		static void xeon_set_sbar_base_and_limit(struct ntb_softc *,
uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar);		uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar);
static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr,		static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr,
enum ntb_bar idx);		enum ntb_bar idx);
static int xeon_setup_b2b_mw(struct ntb_softc *,		static int xeon_setup_b2b_mw(struct ntb_softc *,
const struct ntb_b2b_addr addr, const struct ntb_b2b_addr peer_addr);		const struct ntb_b2b_addr addr, const struct ntb_b2b_addr peer_addr);
		static int xeon_gen3_setup_b2b_mw(struct ntb_softc *);
		static int intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr,
		size_t size);
static inline bool link_is_up(struct ntb_softc *ntb);		static inline bool link_is_up(struct ntb_softc *ntb);
static inline bool _xeon_link_is_up(struct ntb_softc *ntb);		static inline bool _xeon_link_is_up(struct ntb_softc *ntb);
static inline bool atom_link_is_err(struct ntb_softc *ntb);		static inline bool atom_link_is_err(struct ntb_softc *ntb);
static inline enum ntb_speed intel_ntb_link_sta_speed(struct ntb_softc *);		static inline enum ntb_speed intel_ntb_link_sta_speed(struct ntb_softc *);
static inline enum ntb_width intel_ntb_link_sta_width(struct ntb_softc *);		static inline enum ntb_width intel_ntb_link_sta_width(struct ntb_softc *);
static void atom_link_hb(void *arg);		static void atom_link_hb(void *arg);
static void recover_atom_link(void *arg);		static void recover_atom_link(void *arg);
static bool intel_ntb_poll_link(struct ntb_softc *ntb);		static bool intel_ntb_poll_link(struct ntb_softc *ntb);
▲ Show 20 Lines • Show All 95 Lines • ▼ Show 20 Lines

/* Hardware owns the low 16 bits of features. */		/* Hardware owns the low 16 bits of features. */
#define NTB_BAR_SIZE_4K (1 << 0)		#define NTB_BAR_SIZE_4K (1 << 0)
#define NTB_SDOORBELL_LOCKUP (1 << 1)		#define NTB_SDOORBELL_LOCKUP (1 << 1)
#define NTB_SB01BASE_LOCKUP (1 << 2)		#define NTB_SB01BASE_LOCKUP (1 << 2)
#define NTB_B2BDOORBELL_BIT14 (1 << 3)		#define NTB_B2BDOORBELL_BIT14 (1 << 3)
/* Software/configuration owns the top 16 bits. */		/* Software/configuration owns the top 16 bits. */
#define NTB_SPLIT_BAR (1ull << 16)		#define NTB_SPLIT_BAR (1ull << 16)
		#define NTB_ONE_MSIX (1ull << 17)

#define NTB_FEATURES_STR \		#define NTB_FEATURES_STR \
"\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \		"\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \
"\02SDOORBELL_LOCKUP\01BAR_SIZE_4K"		"\02SDOORBELL_LOCKUP\01BAR_SIZE_4K"

static struct ntb_hw_info pci_ids[] = {		static struct ntb_hw_info pci_ids[] = {
/* XXX: PS/SS IDs left out until they are supported. */		/* XXX: PS/SS IDs left out until they are supported. */
{ 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B",		{ 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B",
NTB_ATOM, 0 },		NTB_ATOM, 0 },

{ 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B",		{ 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B",
NTB_XEON, NTB_SDOORBELL_LOCKUP \| NTB_B2BDOORBELL_BIT14 },		NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP \| NTB_B2BDOORBELL_BIT14 },
{ 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B",		{ 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B",
NTB_XEON, NTB_SDOORBELL_LOCKUP \| NTB_B2BDOORBELL_BIT14 },		NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP \| NTB_B2BDOORBELL_BIT14 },
{ 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON,		{ 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B",
NTB_SDOORBELL_LOCKUP \| NTB_B2BDOORBELL_BIT14 \|		NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP \| NTB_B2BDOORBELL_BIT14 \|
NTB_SB01BASE_LOCKUP \| NTB_BAR_SIZE_4K },		NTB_SB01BASE_LOCKUP \| NTB_BAR_SIZE_4K },
{ 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON,		{ 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B",
NTB_SDOORBELL_LOCKUP \| NTB_B2BDOORBELL_BIT14 \|		NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP \| NTB_B2BDOORBELL_BIT14 \|
NTB_SB01BASE_LOCKUP },		NTB_SB01BASE_LOCKUP },
{ 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON,		{ 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B",
NTB_SDOORBELL_LOCKUP \| NTB_B2BDOORBELL_BIT14 \|		NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP \| NTB_B2BDOORBELL_BIT14 \|
NTB_SB01BASE_LOCKUP },		NTB_SB01BASE_LOCKUP },

		{ 0x201C8086, "SKL Xeon E5 V5 Non-Transparent Bridge B2B",
		NTB_XEON_GEN3, 0 },
};		};

static const struct ntb_reg atom_reg = {		static const struct ntb_reg atom_reg = {
.ntb_ctl = ATOM_NTBCNTL_OFFSET,		.ntb_ctl = ATOM_NTBCNTL_OFFSET,
.lnk_sta = ATOM_LINK_STATUS_OFFSET,		.lnk_sta = ATOM_LINK_STATUS_OFFSET,
.db_size = sizeof(uint64_t),		.db_size = sizeof(uint64_t),
.mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 },		.mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 },
};		};
▲ Show 20 Lines • Show All 68 Lines • ▼ Show 20 Lines
static struct ntb_b2b_addr xeon_b2b_dsd_addr = {		static struct ntb_b2b_addr xeon_b2b_dsd_addr = {
.bar0_addr = XEON_B2B_BAR0_ADDR,		.bar0_addr = XEON_B2B_BAR0_ADDR,
.bar2_addr64 = XEON_B2B_BAR2_ADDR64,		.bar2_addr64 = XEON_B2B_BAR2_ADDR64,
.bar4_addr64 = XEON_B2B_BAR4_ADDR64,		.bar4_addr64 = XEON_B2B_BAR4_ADDR64,
.bar4_addr32 = XEON_B2B_BAR4_ADDR32,		.bar4_addr32 = XEON_B2B_BAR4_ADDR32,
.bar5_addr32 = XEON_B2B_BAR5_ADDR32,		.bar5_addr32 = XEON_B2B_BAR5_ADDR32,
};		};

		static const struct ntb_reg xeon_gen3_reg = {
		.ntb_ctl = XEON_GEN3_REG_IMNTB_CTRL,
		.lnk_sta = XEON_GEN3_INT_LNK_STS_OFFSET,
		.db_size = sizeof(uint32_t),
		.mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 },
		};

		static const struct ntb_alt_reg xeon_gen3_pri_reg = {
		.db_bell = XEON_GEN3_REG_EMDOORBELL,
		.db_mask = XEON_GEN3_REG_IMINT_DISABLE,
		.spad = XEON_GEN3_REG_IMSPAD,
		};

		static const struct ntb_alt_reg xeon_gen3_b2b_reg = {
		.db_bell = XEON_GEN3_REG_IMDOORBELL,
		.db_mask = XEON_GEN3_REG_EMINT_DISABLE,
		.spad = XEON_GEN3_REG_IMB2B_SSPAD,
		};

		static const struct ntb_xlat_reg xeon_gen3_sec_xlat = {
		.bar0_base = XEON_GEN3_EXT_REG_BAR0BASE,
		.bar2_base = XEON_GEN3_EXT_REG_BAR1BASE,
		.bar4_base = XEON_GEN3_EXT_REG_BAR2BASE,

		.bar2_limit = XEON_GEN3_REG_IMBAR1XLIMIT,
		.bar4_limit = XEON_GEN3_REG_IMBAR2XLIMIT,

		.bar2_xlat = XEON_GEN3_REG_IMBAR1XBASE,
		.bar4_xlat = XEON_GEN3_REG_IMBAR2XBASE,
		};

SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,		SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
"B2B MW segment overrides -- MUST be the same on both sides");		"B2B MW segment overrides -- MUST be the same on both sides");

SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN,		SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN,
&xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon "		&xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon "
"hardware, use this 64-bit address on the bus between the NTB devices for "		"hardware, use this 64-bit address on the bus between the NTB devices for "
"the window at BAR2, on the upstream side of the link. MUST be the same "		"the window at BAR2, on the upstream side of the link. MUST be the same "
"address on both sides.");		"address on both sides.");
▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines	intel_ntb_attach(device_t device)
/* Heartbeat timer for NTB_ATOM since there is no link interrupt */		/* Heartbeat timer for NTB_ATOM since there is no link interrupt */
callout_init(&ntb->heartbeat_timer, 1);		callout_init(&ntb->heartbeat_timer, 1);
callout_init(&ntb->lr_timer, 1);		callout_init(&ntb->lr_timer, 1);
callout_init(&ntb->peer_msix_work, 1);		callout_init(&ntb->peer_msix_work, 1);
mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN);		mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN);

if (ntb->type == NTB_ATOM)		if (ntb->type == NTB_ATOM)
error = intel_ntb_detect_atom(ntb);		error = intel_ntb_detect_atom(ntb);
		else if (ntb->type == NTB_XEON_GEN3)
		error = intel_ntb_detect_xeon_gen3(ntb);
else		else
error = intel_ntb_detect_xeon(ntb);		error = intel_ntb_detect_xeon(ntb);
if (error != 0)		if (error != 0)
goto out;		goto out;

intel_ntb_detect_max_mw(ntb);		intel_ntb_detect_max_mw(ntb);

pci_enable_busmaster(ntb->device);		pci_enable_busmaster(ntb->device);

error = intel_ntb_map_pci_bars(ntb);		error = intel_ntb_map_pci_bars(ntb);
if (error != 0)		if (error != 0)
goto out;		goto out;
if (ntb->type == NTB_ATOM)		if (ntb->type == NTB_ATOM)
error = intel_ntb_atom_init_dev(ntb);		error = intel_ntb_atom_init_dev(ntb);
		else if (ntb->type == NTB_XEON_GEN3)
		error = intel_ntb_xeon_gen3_init_dev(ntb);
else		else
error = intel_ntb_xeon_init_dev(ntb);		error = intel_ntb_xeon_init_dev(ntb);
if (error != 0)		if (error != 0)
goto out;		goto out;

intel_ntb_spad_clear(device);		intel_ntb_spad_clear(device);

intel_ntb_poll_link(ntb);		intel_ntb_poll_link(ntb);
Show All 23 Lines	if (ntb->self_reg != NULL) {
DB_MASK_LOCK(ntb);		DB_MASK_LOCK(ntb);
db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask);		db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask);
DB_MASK_UNLOCK(ntb);		DB_MASK_UNLOCK(ntb);
}		}
callout_drain(&ntb->heartbeat_timer);		callout_drain(&ntb->heartbeat_timer);
callout_drain(&ntb->lr_timer);		callout_drain(&ntb->lr_timer);
callout_drain(&ntb->peer_msix_work);		callout_drain(&ntb->peer_msix_work);
pci_disable_busmaster(ntb->device);		pci_disable_busmaster(ntb->device);
if (ntb->type == NTB_XEON)		if (ntb->type == NTB_XEON_GEN1)
intel_ntb_teardown_xeon(ntb);		intel_ntb_teardown_xeon(ntb);
intel_ntb_teardown_interrupts(ntb);		intel_ntb_teardown_interrupts(ntb);

mtx_destroy(&ntb->db_mask_lock);		mtx_destroy(&ntb->db_mask_lock);

intel_ntb_unmap_pci_bar(ntb);		intel_ntb_unmap_pci_bar(ntb);

return (0);		return (0);
▲ Show 20 Lines • Show All 93 Lines • ▼ Show 20 Lines	if (bus_dma_iommu_load_ident(ntb->bar0_dma_tag, ntb->bar0_dma_map,
return (ENOMEM);		return (ENOMEM);
}		}

bar = &ntb->bar_info[NTB_B2B_BAR_1];		bar = &ntb->bar_info[NTB_B2B_BAR_1];
bar->pci_resource_id = PCIR_BAR(2);		bar->pci_resource_id = PCIR_BAR(2);
rc = map_memory_window_bar(ntb, bar);		rc = map_memory_window_bar(ntb, bar);
if (rc != 0)		if (rc != 0)
goto out;		goto out;
		if (ntb->type == NTB_XEON_GEN3) {
		bar->psz_off = XEON_GEN3_INT_REG_IMBAR1SZ;
		bar->ssz_off = XEON_GEN3_INT_REG_EMBAR1SZ;
		bar->pbarxlat_off = XEON_GEN3_REG_EMBAR1XBASE;
		} else {
bar->psz_off = XEON_PBAR23SZ_OFFSET;		bar->psz_off = XEON_PBAR23SZ_OFFSET;
bar->ssz_off = XEON_SBAR23SZ_OFFSET;		bar->ssz_off = XEON_SBAR23SZ_OFFSET;
bar->pbarxlat_off = XEON_PBAR2XLAT_OFFSET;		bar->pbarxlat_off = XEON_PBAR2XLAT_OFFSET;
		}

bar = &ntb->bar_info[NTB_B2B_BAR_2];		bar = &ntb->bar_info[NTB_B2B_BAR_2];
bar->pci_resource_id = PCIR_BAR(4);		bar->pci_resource_id = PCIR_BAR(4);
rc = map_memory_window_bar(ntb, bar);		rc = map_memory_window_bar(ntb, bar);
if (rc != 0)		if (rc != 0)
goto out;		goto out;
		if (ntb->type == NTB_XEON_GEN3) {
		bar->psz_off = XEON_GEN3_INT_REG_IMBAR2SZ;
		bar->ssz_off = XEON_GEN3_INT_REG_EMBAR2SZ;
		bar->pbarxlat_off = XEON_GEN3_REG_EMBAR2XBASE;
		} else {
bar->psz_off = XEON_PBAR4SZ_OFFSET;		bar->psz_off = XEON_PBAR4SZ_OFFSET;
bar->ssz_off = XEON_SBAR4SZ_OFFSET;		bar->ssz_off = XEON_SBAR4SZ_OFFSET;
bar->pbarxlat_off = XEON_PBAR4XLAT_OFFSET;		bar->pbarxlat_off = XEON_PBAR4XLAT_OFFSET;
		}

if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR))		if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR))
goto out;		goto out;

		if (ntb->type == NTB_XEON_GEN3) {
		device_printf(ntb->device, "no split bar support\n");
		return (ENXIO);
		}

bar = &ntb->bar_info[NTB_B2B_BAR_3];		bar = &ntb->bar_info[NTB_B2B_BAR_3];
bar->pci_resource_id = PCIR_BAR(5);		bar->pci_resource_id = PCIR_BAR(5);
rc = map_memory_window_bar(ntb, bar);		rc = map_memory_window_bar(ntb, bar);
bar->psz_off = XEON_PBAR5SZ_OFFSET;		bar->psz_off = XEON_PBAR5SZ_OFFSET;
bar->ssz_off = XEON_SBAR5SZ_OFFSET;		bar->ssz_off = XEON_SBAR5SZ_OFFSET;
bar->pbarxlat_off = XEON_PBAR5XLAT_OFFSET;		bar->pbarxlat_off = XEON_PBAR5XLAT_OFFSET;

out:		out:
▲ Show 20 Lines • Show All 203 Lines • ▼ Show 20 Lines	for (i = 0; i < desired; i++)
vectors[i] = (i % avail) + 1;		vectors[i] = (i % avail) + 1;

rc = pci_remap_msix(dev, desired, vectors);		rc = pci_remap_msix(dev, desired, vectors);
free(vectors, M_NTB);		free(vectors, M_NTB);
return (rc);		return (rc);
}		}

static int		static int
		intel_ntb_xeon_gen3_init_isr(struct ntb_softc *ntb)
		{
		uint64_t i, reg;
		uint32_t desired_vectors, num_vectors;
		int rc;

		ntb->allocated_interrupts = 0;
		ntb->last_ts = ticks;

		/* Mask all the interrupts, including hardware interrupt */
		intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_DISABLE, ~0ULL);

		/* Clear Interrupt Status */
		reg = intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS);
		intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_STATUS, reg);

		num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device),
		XEON_GEN3_DB_MSIX_VECTOR_COUNT);

		rc = pci_alloc_msix(ntb->device, &num_vectors);
		if (rc != 0) {
		device_printf(ntb->device,
		"Interrupt allocation failed %d\n", rc);
		return (rc);
		}
		if (desired_vectors != num_vectors) {
		device_printf(ntb->device, "Couldn't get %d vectors\n",
		XEON_GEN3_DB_MSIX_VECTOR_COUNT);
		return (ENXIO);
		}
		/* 32 db + 1 hardware */
		if (num_vectors == XEON_GEN3_DB_MSIX_VECTOR_COUNT) {
		/* Program INTVECXX source register */
		for (i = 0; i < XEON_GEN3_DB_MSIX_VECTOR_COUNT; i++) {
		/* interrupt source i for vector i */
		intel_ntb_reg_write(1, XEON_GEN3_REG_IMINTVEC00 + i, i);
		if (i == (XEON_GEN3_DB_MSIX_VECTOR_COUNT - 1)) {
		intel_ntb_reg_write(1,
		XEON_GEN3_REG_IMINTVEC00 + i,
		XEON_GEN3_LINK_VECTOR_INDEX);
		}
		}

		intel_ntb_create_msix_vec(ntb, num_vectors);
		rc = intel_ntb_setup_msix(ntb, num_vectors);

		/* enable all interrupts */
		intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_DISABLE, 0ULL);
		} else {
		device_printf(ntb->device, "need to remap interrupts, giving up.\n");
		return (ENXIO);
		}

		return (0);
		}

		static int
intel_ntb_init_isr(struct ntb_softc *ntb)		intel_ntb_init_isr(struct ntb_softc *ntb)
{		{
uint32_t desired_vectors, num_vectors;		uint32_t desired_vectors, num_vectors;
int rc;		int rc;

ntb->allocated_interrupts = 0;		ntb->allocated_interrupts = 0;
ntb->last_ts = ticks;		ntb->last_ts = ticks;

Show All 22 Lines	if (rc == 0 && num_vectors < desired_vectors) {
else		else
pci_release_msi(ntb->device);		pci_release_msi(ntb->device);
}		}
if (rc != 0)		if (rc != 0)
num_vectors = 1;		num_vectors = 1;
} else		} else
num_vectors = 1;		num_vectors = 1;

if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) {		if (ntb->type == NTB_XEON_GEN1 && num_vectors < ntb->db_vec_count) {
if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {		if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
device_printf(ntb->device,		device_printf(ntb->device,
"Errata workaround does not support MSI or INTX\n");		"Errata workaround does not support MSI or INTX\n");
return (EINVAL);		return (EINVAL);
}		}

ntb->db_vec_count = 1;		ntb->db_vec_count = 1;
ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT;		ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT;
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines	if (current_int->res != NULL)
bus_release_resource(ntb->device, SYS_RES_IRQ,		bus_release_resource(ntb->device, SYS_RES_IRQ,
rman_get_rid(current_int->res), current_int->res);		rman_get_rid(current_int->res), current_int->res);
}		}

intel_ntb_free_msix_vec(ntb);		intel_ntb_free_msix_vec(ntb);
pci_release_msi(ntb->device);		pci_release_msi(ntb->device);
}		}

/*
* Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it
* out to make code clearer.
*/
static inline uint64_t		static inline uint64_t
db_ioread(struct ntb_softc *ntb, uint64_t regoff)		db_ioread(struct ntb_softc *ntb, uint64_t regoff)
{		{

if (ntb->type == NTB_ATOM)		switch (ntb->type) {
		case NTB_ATOM:
		case NTB_XEON_GEN3:
return (intel_ntb_reg_read(8, regoff));		return (intel_ntb_reg_read(8, regoff));
		case NTB_XEON_GEN1:
KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));

return (intel_ntb_reg_read(2, regoff));		return (intel_ntb_reg_read(2, regoff));
}		}
		}

static inline void		static inline void
db_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)		db_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
{		{

KASSERT((val & ~ntb->db_valid_mask) == 0,		KASSERT((val & ~ntb->db_valid_mask) == 0,
("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,		("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
(uintmax_t)(val & ~ntb->db_valid_mask),		(uintmax_t)(val & ~ntb->db_valid_mask),
(uintmax_t)ntb->db_valid_mask));		(uintmax_t)ntb->db_valid_mask));

if (regoff == ntb->self_reg->db_mask)		if (regoff == ntb->self_reg->db_mask)
DB_MASK_ASSERT(ntb, MA_OWNED);		DB_MASK_ASSERT(ntb, MA_OWNED);
db_iowrite_raw(ntb, regoff, val);		db_iowrite_raw(ntb, regoff, val);
}		}

static inline void		static inline void
db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)		db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
{		{

if (ntb->type == NTB_ATOM) {		switch (ntb->type) {
		case NTB_ATOM:
		case NTB_XEON_GEN3:
intel_ntb_reg_write(8, regoff, val);		intel_ntb_reg_write(8, regoff, val);
return;		break;
}		case NTB_XEON_GEN1:

KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
intel_ntb_reg_write(2, regoff, (uint16_t)val);		intel_ntb_reg_write(2, regoff, (uint16_t)val);
		break;
}		}
		}

static void		static void
intel_ntb_db_set_mask(device_t dev, uint64_t bits)		intel_ntb_db_set_mask(device_t dev, uint64_t bits)
{		{
struct ntb_softc *ntb = device_get_softc(dev);		struct ntb_softc *ntb = device_get_softc(dev);

DB_MASK_LOCK(ntb);		DB_MASK_LOCK(ntb);
ntb->db_mask \|= bits;		ntb->db_mask \|= bits;
Show All 32 Lines

static uint64_t		static uint64_t
intel_ntb_db_read(device_t dev)		intel_ntb_db_read(device_t dev)
{		{
struct ntb_softc *ntb = device_get_softc(dev);		struct ntb_softc *ntb = device_get_softc(dev);

if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))		if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))
return (ntb->fake_db);		return (ntb->fake_db);
		if (ntb->type == NTB_XEON_GEN3)
		return (intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS));
		else
return (db_ioread(ntb, ntb->self_reg->db_bell));		return (db_ioread(ntb, ntb->self_reg->db_bell));
}		}

static void		static void
intel_ntb_db_clear(device_t dev, uint64_t bits)		intel_ntb_db_clear(device_t dev, uint64_t bits)
{		{
struct ntb_softc *ntb = device_get_softc(dev);		struct ntb_softc *ntb = device_get_softc(dev);

KASSERT((bits & ~ntb->db_valid_mask) == 0,		KASSERT((bits & ~ntb->db_valid_mask) == 0,
("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,		("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
(uintmax_t)(bits & ~ntb->db_valid_mask),		(uintmax_t)(bits & ~ntb->db_valid_mask),
(uintmax_t)ntb->db_valid_mask));		(uintmax_t)ntb->db_valid_mask));

if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {		if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
DB_MASK_LOCK(ntb);		DB_MASK_LOCK(ntb);
ntb->fake_db &= ~bits;		ntb->fake_db &= ~bits;
DB_MASK_UNLOCK(ntb);		DB_MASK_UNLOCK(ntb);
return;		return;
}		}

		if (ntb->type == NTB_XEON_GEN3)
		intel_ntb_reg_write(4, XEON_GEN3_REG_IMINT_STATUS,
		(uint32_t)bits);
		else
db_iowrite(ntb, ntb->self_reg->db_bell, bits);		db_iowrite(ntb, ntb->self_reg->db_bell, bits);
}		}

static inline uint64_t		static inline uint64_t
intel_ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector)		intel_ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector)
{		{
uint64_t shift, mask;		uint64_t shift, mask;

if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {		if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
Show All 17 Lines
static void		static void
intel_ntb_interrupt(struct ntb_softc *ntb, uint32_t vec)		intel_ntb_interrupt(struct ntb_softc *ntb, uint32_t vec)
{		{
uint64_t vec_mask;		uint64_t vec_mask;

ntb->last_ts = ticks;		ntb->last_ts = ticks;
vec_mask = intel_ntb_vec_mask(ntb, vec);		vec_mask = intel_ntb_vec_mask(ntb, vec);

		if (ntb->type == NTB_XEON_GEN3 && vec == XEON_GEN3_LINK_VECTOR_INDEX)
		vec_mask \|= ntb->db_link_mask;
if ((vec_mask & ntb->db_link_mask) != 0) {		if ((vec_mask & ntb->db_link_mask) != 0) {
if (intel_ntb_poll_link(ntb))		if (intel_ntb_poll_link(ntb))
ntb_link_event(ntb->device);		ntb_link_event(ntb->device);
		if (ntb->type == NTB_XEON_GEN3)
		intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_STATUS,
		intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS));
}		}

if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) &&		if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) &&
(vec_mask & ntb->db_link_mask) == 0) {		(vec_mask & ntb->db_link_mask) == 0) {
DB_MASK_LOCK(ntb);		DB_MASK_LOCK(ntb);

/*		/*
* Do not report same DB events again if not cleared yet,		* Do not report same DB events again if not cleared yet,
▲ Show 20 Lines • Show All 108 Lines • ▼ Show 20 Lines	intel_ntb_teardown_xeon(struct ntb_softc *ntb)
if (ntb->reg != NULL)		if (ntb->reg != NULL)
intel_ntb_link_disable(ntb->device);		intel_ntb_link_disable(ntb->device);
}		}

static void		static void
intel_ntb_detect_max_mw(struct ntb_softc *ntb)		intel_ntb_detect_max_mw(struct ntb_softc *ntb)
{		{

if (ntb->type == NTB_ATOM) {		switch (ntb->type) {
		case NTB_ATOM:
ntb->mw_count = ATOM_MW_COUNT;		ntb->mw_count = ATOM_MW_COUNT;
return;		break;
}		case NTB_XEON_GEN1:

if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))		if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT;		ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT;
else		else
ntb->mw_count = XEON_SNB_MW_COUNT;		ntb->mw_count = XEON_SNB_MW_COUNT;
		break;
		case NTB_XEON_GEN3:
		if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
		ntb->mw_count = XEON_GEN3_SPLIT_MW_COUNT;
		else
		ntb->mw_count = XEON_GEN3_MW_COUNT;
		break;
}		}
		}

static int		static int
intel_ntb_detect_xeon(struct ntb_softc *ntb)		intel_ntb_detect_xeon(struct ntb_softc *ntb)
{		{
uint8_t ppd, conn_type;		uint8_t ppd, conn_type;

ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1);		ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1);
ntb->ppd = ppd;		ntb->ppd = ppd;
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines	intel_ntb_detect_atom(struct ntb_softc *ntb)
default:		default:
device_printf(ntb->device, "Unsupported NTB configuration\n");		device_printf(ntb->device, "Unsupported NTB configuration\n");
return (ENXIO);		return (ENXIO);
}		}
return (0);		return (0);
}		}

static int		static int
		intel_ntb_detect_xeon_gen3(struct ntb_softc *ntb)
		{
		uint8_t ppd, conn_type;

		ppd = pci_read_config(ntb->device, XEON_GEN3_INT_REG_PPD, 1);
		ntb->ppd = ppd;

		/* check port definition */
		conn_type = XEON_GEN3_REG_PPD_PORT_DEF_F(ppd);
		switch (conn_type) {
		case NTB_CONN_B2B:
		ntb->conn_type = conn_type;
		break;
		default:
		device_printf(ntb->device, "Unsupported connection type: %u\n",
		conn_type);
		return (ENXIO);
		}

		/* check cross link configuration status */
		if (XEON_GEN3_REG_PPD_CONF_STS_F(ppd)) {
		/* NTB Port is configured as DSD/USP */
		ntb->dev_type = NTB_DEV_DSD;
		} else {
		/* NTB Port is configured as USD/DSP */
		ntb->dev_type = NTB_DEV_USD;
		}

		if (XEON_GEN3_REG_PPD_ONE_MSIX_F(ppd)) {
		/*
		* This bit when set, causes only a single MSI-X message to be
		* generated if MSI-X is enabled.
		*/
		ntb->features \|= NTB_ONE_MSIX;
		}

		if (XEON_GEN3_REG_PPD_BAR45_SPL_F(ppd)) {
		/* BARs 4 and 5 are presented as two 32b non-prefetchable BARs */
		ntb->features \|= NTB_SPLIT_BAR;
		}

		device_printf(ntb->device, "conn type 0x%02x, dev type 0x%02x,"
		"features 0x%02x\n", ntb->conn_type, ntb->dev_type, ntb->features);

		return (0);
		}

		static int
intel_ntb_xeon_init_dev(struct ntb_softc *ntb)		intel_ntb_xeon_init_dev(struct ntb_softc *ntb)
{		{
int rc;		int rc;

ntb->spad_count = XEON_SPAD_COUNT;		ntb->spad_count = XEON_SPAD_COUNT;
ntb->db_count = XEON_DB_COUNT;		ntb->db_count = XEON_DB_COUNT;
ntb->db_link_mask = XEON_DB_LINK_BIT;		ntb->db_link_mask = XEON_DB_LINK_BIT;
ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT;		ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT;
▲ Show 20 Lines • Show All 68 Lines • ▼ Show 20 Lines	intel_ntb_xeon_init_dev(struct ntb_softc *ntb)
db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask);		db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask);
DB_MASK_UNLOCK(ntb);		DB_MASK_UNLOCK(ntb);

rc = intel_ntb_init_isr(ntb);		rc = intel_ntb_init_isr(ntb);
return (rc);		return (rc);
}		}

static int		static int
		intel_ntb_xeon_gen3_init_dev(struct ntb_softc *ntb)
		{
		int rc;

		ntb->spad_count = XEON_GEN3_SPAD_COUNT;
		ntb->db_count = XEON_GEN3_DB_COUNT;
		ntb->db_link_mask = XEON_GEN3_DB_LINK_BIT;
		ntb->db_vec_count = XEON_GEN3_DB_MSIX_VECTOR_COUNT;
		ntb->db_vec_shift = XEON_GEN3_DB_MSIX_VECTOR_SHIFT;

		if (ntb->conn_type != NTB_CONN_B2B) {
		device_printf(ntb->device, "Connection type %d not supported\n",
		ntb->conn_type);
		return (ENXIO);
		}

		ntb->reg = &xeon_gen3_reg;
		ntb->self_reg = &xeon_gen3_pri_reg;
		ntb->peer_reg = &xeon_gen3_b2b_reg;
		ntb->xlat_reg = &xeon_gen3_sec_xlat;

		ntb->db_valid_mask = (1ULL << ntb->db_count) - 1;

		xeon_gen3_setup_b2b_mw(ntb);

		/* Enable Bus Master and Memory Space on the External Side */
		intel_ntb_reg_write(2, XEON_GEN3_EXT_REG_PCI_CMD,
		PCIM_CMD_MEMEN \| PCIM_CMD_BUSMASTEREN);

		/* Setup Interrupt */
		rc = intel_ntb_xeon_gen3_init_isr(ntb);

		return (rc);
		}

		static int
intel_ntb_atom_init_dev(struct ntb_softc *ntb)		intel_ntb_atom_init_dev(struct ntb_softc *ntb)
{		{
int error;		int error;

KASSERT(ntb->conn_type == NTB_CONN_B2B,		KASSERT(ntb->conn_type == NTB_CONN_B2B,
("Unsupported NTB configuration (%d)\n", ntb->conn_type));		("Unsupported NTB configuration (%d)\n", ntb->conn_type));

ntb->spad_count = ATOM_SPAD_COUNT;		ntb->spad_count = ATOM_SPAD_COUNT;
▲ Show 20 Lines • Show All 274 Lines • ▼ Show 20 Lines	xeon_setup_b2b_mw(struct ntb_softc ntb, const struct ntb_b2b_addr addr,
* B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits		* B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits
* at a time.		* at a time.
*/		*/
intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff);		intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff);
intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32);		intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32);
return (0);		return (0);
}		}

		static int
		xeon_gen3_setup_b2b_mw(struct ntb_softc *ntb)
		{
		uint64_t reg;
		uint32_t embarsz, imbarsz;

		/* IMBAR1SZ should be equal to EMBAR1SZ */
		embarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_EMBAR1SZ, 1);
		imbarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_IMBAR1SZ, 1);
		if (embarsz != imbarsz) {
		device_printf(ntb->device,
		"IMBAR1SZ (%u) should be equal to EMBAR1SZ (%u)\n",
		imbarsz, embarsz);
		return (EIO);
		}

		/* IMBAR2SZ should be equal to EMBAR2SZ */
		embarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_EMBAR2SZ, 1);
		imbarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_IMBAR2SZ, 1);
		if (embarsz != imbarsz) {
		device_printf(ntb->device,
		"IMBAR2SZ (%u) should be equal to EMBAR2SZ (%u)\n",
		imbarsz, embarsz);
		return (EIO);
		}

		/* Client will provide the incoming IMBAR1/2XBASE, zero it for now */
		intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR1XBASE, 0);
		intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR2XBASE, 0);

		/*
		* If the value in EMBAR1LIMIT is set equal to the value in EMBAR1,
		* the memory window for EMBAR1 is disabled.
		* Note: It is needed to avoid malacious access.
		*/
		reg = pci_read_config(ntb->device, XEON_GEN3_EXT_REG_BAR1BASE, 8);
		intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR1XLIMIT, reg);

		reg = pci_read_config(ntb->device, XEON_GEN3_EXT_REG_BAR2BASE, 8);
		intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR2XLIMIT, reg);

		return (0);
		}

static inline bool		static inline bool
_xeon_link_is_up(struct ntb_softc *ntb)		_xeon_link_is_up(struct ntb_softc *ntb)
{		{

if (ntb->conn_type == NTB_CONN_TRANSPARENT)		if (ntb->conn_type == NTB_CONN_TRANSPARENT)
return (true);		return (true);
return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0);		return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0);
}		}

static inline bool		static inline bool
link_is_up(struct ntb_softc *ntb)		link_is_up(struct ntb_softc *ntb)
{		{

if (ntb->type == NTB_XEON)		if (ntb->type == NTB_XEON_GEN1 \|\| ntb->type == NTB_XEON_GEN3)
return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good \|\|		return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good \|\|
!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)));		!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)));

KASSERT(ntb->type == NTB_ATOM, ("ntb type"));		KASSERT(ntb->type == NTB_ATOM, ("ntb type"));
return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0);		return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0);
}		}

static inline bool		static inline bool
▲ Show 20 Lines • Show All 247 Lines • ▼ Show 20 Lines	intel_ntb_poll_link(struct ntb_softc *ntb)
if (ntb->type == NTB_ATOM) {		if (ntb->type == NTB_ATOM) {
ntb_cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl);		ntb_cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl);
if (ntb_cntl == ntb->ntb_ctl)		if (ntb_cntl == ntb->ntb_ctl)
return (false);		return (false);

ntb->ntb_ctl = ntb_cntl;		ntb->ntb_ctl = ntb_cntl;
ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta);		ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta);
} else {		} else {
db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask);		if (ntb->type == NTB_XEON_GEN1)
		db_iowrite_raw(ntb, ntb->self_reg->db_bell,
		ntb->db_link_mask);

reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);		reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);
if (reg_val == ntb->lnk_sta)		if (reg_val == ntb->lnk_sta)
return (false);		return (false);

ntb->lnk_sta = reg_val;		ntb->lnk_sta = reg_val;

if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {		if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
▲ Show 20 Lines • Show All 758 Lines • ▼ Show 20 Lines	intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size)
bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg);		bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg);

limit = 0;		limit = 0;
if (bar_is_64bit(ntb, bar_num)) {		if (bar_is_64bit(ntb, bar_num)) {
base = intel_ntb_reg_read(8, base_reg) & BAR_HIGH_MASK;		base = intel_ntb_reg_read(8, base_reg) & BAR_HIGH_MASK;

if (limit_reg != 0 && size != mw_size)		if (limit_reg != 0 && size != mw_size)
limit = base + size;		limit = base + size;
		else
		limit = base + mw_size;

/* Set and verify translation address */		/* Set and verify translation address */
intel_ntb_reg_write(8, xlat_reg, addr);		intel_ntb_reg_write(8, xlat_reg, addr);
reg_val = intel_ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK;		reg_val = intel_ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK;
if (reg_val != addr) {		if (reg_val != addr) {
intel_ntb_reg_write(8, xlat_reg, 0);		intel_ntb_reg_write(8, xlat_reg, 0);
return (EIO);		return (EIO);
}		}

/* Set and verify the limit */		/* Set and verify the limit */
intel_ntb_reg_write(8, limit_reg, limit);		intel_ntb_reg_write(8, limit_reg, limit);
reg_val = intel_ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK;		reg_val = intel_ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK;
if (reg_val != limit) {		if (reg_val != limit) {
intel_ntb_reg_write(8, limit_reg, base);		intel_ntb_reg_write(8, limit_reg, base);
intel_ntb_reg_write(8, xlat_reg, 0);		intel_ntb_reg_write(8, xlat_reg, 0);
return (EIO);		return (EIO);
}		}

		if (ntb->type == NTB_XEON_GEN3) {
		limit = base + size;

		/* set EMBAR1/2XLIMIT */
		if (!idx)
		intel_ntb_reg_write(8,
		XEON_GEN3_REG_EMBAR1XLIMIT, limit);
		else
		intel_ntb_reg_write(8,
		XEON_GEN3_REG_EMBAR2XLIMIT, limit);
		}
} else {		} else {
/* Configure 32-bit (split) BAR MW */		/* Configure 32-bit (split) BAR MW */
		if (ntb->type == NTB_XEON_GEN3)
		return (EIO);

if ((addr & UINT32_MAX) != addr)		if ((addr & UINT32_MAX) != addr)
return (ERANGE);		return (ERANGE);
if (((addr + size) & UINT32_MAX) != (addr + size))		if (((addr + size) & UINT32_MAX) != (addr + size))
return (ERANGE);		return (ERANGE);

base = intel_ntb_reg_read(4, base_reg) & BAR_HIGH_MASK;		base = intel_ntb_reg_read(4, base_reg) & BAR_HIGH_MASK;

▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Lines	intel_ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode)
rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode);		rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode);
if (rc == 0)		if (rc == 0)
bar->map_mode = mode;		bar->map_mode = mode;

return (rc);		return (rc);
}		}

static void		static void
intel_ntb_peer_db_set(device_t dev, uint64_t bit)		intel_ntb_peer_db_set(device_t dev, uint64_t bits)
{		{
struct ntb_softc *ntb = device_get_softc(dev);		struct ntb_softc *ntb = device_get_softc(dev);
		uint64_t db;

		if ((bits & ~ntb->db_valid_mask) != 0) {
		device_printf(ntb->device, "Invalid doorbell bits %lx\n", bits);
		return;
		}

if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {		if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
struct ntb_pci_bar_info *lapic;		struct ntb_pci_bar_info *lapic;
unsigned i;		unsigned i;

lapic = ntb->peer_lapic_bar;		lapic = ntb->peer_lapic_bar;

for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) {		for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) {
if ((bit & intel_ntb_db_vector_mask(dev, i)) != 0)		if ((bits & intel_ntb_db_vector_mask(dev, i)) != 0)
bus_space_write_4(lapic->pci_bus_tag,		bus_space_write_4(lapic->pci_bus_tag,
lapic->pci_bus_handle,		lapic->pci_bus_handle,
ntb->peer_msix_data[i].nmd_ofs,		ntb->peer_msix_data[i].nmd_ofs,
ntb->peer_msix_data[i].nmd_data);		ntb->peer_msix_data[i].nmd_data);
}		}
return;		return;
}		}

if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) {		if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) {
intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit);		intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bits);
return;		return;
}		}

db_iowrite(ntb, ntb->peer_reg->db_bell, bit);		if (ntb->type == NTB_XEON_GEN3) {
		while (bits != 0) {
		db = ffsll(bits);

		intel_ntb_reg_write(1,
		ntb->peer_reg->db_bell + (db - 1) * 4, 0x1);

		bits = bits & (bits - 1);
		}
		} else {
		db_iowrite(ntb, ntb->peer_reg->db_bell, bits);
		}
}		}

static int		static int
intel_ntb_peer_db_addr(device_t dev, bus_addr_t db_addr, vm_size_t db_size)		intel_ntb_peer_db_addr(device_t dev, bus_addr_t db_addr, vm_size_t db_size)
{		{
struct ntb_softc *ntb = device_get_softc(dev);		struct ntb_softc *ntb = device_get_softc(dev);
struct ntb_pci_bar_info *bar;		struct ntb_pci_bar_info *bar;
uint64_t regoff;		uint64_t regoff;
▲ Show 20 Lines • Show All 119 Lines • Show Last 20 Lines