Index: head/share/man/man4/ntb_hw_intel.4 =================================================================== --- head/share/man/man4/ntb_hw_intel.4 +++ head/share/man/man4/ntb_hw_intel.4 @@ -25,7 +25,7 @@ .\" .\" $FreeBSD$ .\" -.Dd August 30, 2017 +.Dd October 11, 2020 .Dt NTB_HW_INTEL 4 .Os .Sh NAME @@ -50,16 +50,18 @@ driver provides support for the Non-Transparent Bridge (NTB) hardware in Intel Xeon E3/E5 and S1200 processor families, which allow one of their PCIe ports to be switched from transparent to non-transparent bridge mode. -In this mode bridge looks not as a PCI bridge, but as PCI endpoint device. +In this mode the bridge looks not like a PCI bridge, but like a PCI endpoint +device. The driver hides hardware details, exposing memory windows, scratchpads -and doorbells of the other side via hardware independent KPI to +and doorbells of the other side via a hardware independent KPI to the .Xr ntb 4 subsystem. .Pp The hardware provides 2 or 3 memory windows to the other system's memory, -16 scratchpad registers and 14 or 34 doorbells to interrupt the other system. -On Xeon processors one of memory windows is typically consumed by the driver -itself to workaround multiple hardware erratas. +16 scratchpad registers and 14, 31 or 34 doorbells to interrupt the other +system, depending on the platform. +On Xeon processors one of the memory windows is typically consumed by the driver +itself to work around multiple hardware errata. .Sh CONFIGURATION The NTB configuration should be set by BIOS. It includes enabling NTB, choosing between NTB-to-NTB (back-to-back) or @@ -67,9 +69,10 @@ enabling split BAR mode (one of two 64-bit BARs can be split into two 32-bit ones) and configuring BAR sizes in bits (from 12 to 29/39) for both NTB sides. .Pp -The recommended configuration is NTB-to-NTB mode, split bar is enabled and -all BAR sizes are set to 20 (1 MiB). +The recommended configuration is NTB-to-NTB mode, split bar enabled and +all BAR sizes set to 20 (1 MiB). This needs to be done on both systems. +Note, on Xeon SkyLake and newer platforms, split bar mode is not available. .Sh SEE ALSO .Xr if_ntb 4 , .Xr ntb_transport 4 , Index: head/sys/dev/ntb/ntb_hw/ntb_hw_intel.h =================================================================== --- head/sys/dev/ntb/ntb_hw/ntb_hw_intel.h +++ head/sys/dev/ntb/ntb_hw/ntb_hw_intel.h @@ -30,7 +30,26 @@ #ifndef _NTB_REGS_H_ #define _NTB_REGS_H_ +#include +#include +/*--------------------------------------------------------------------------- + * Macro: M*_M : Create a mask to isolate a bit field of a data word. + * M*_F : Extract value from a bit field of a data word. + * M*_I : Insert value into a bit field of a data word. + * + * Purpose: Bit field manipulation macros for mask, insert and extract for + * 8-bit, 16-bit, 32-bit and 64-bit data words. + * + * Params: [in] P = Bit position of start of the bit field (lsb is 0). + * [in] N = Size of the bit field in bits. + * [in] X = Value to insert or remove from the bit field. + *--------------------------------------------------------------------------- + */ +#define M8_M(P, N) ((UINT8_MAX >> (8 - (N))) << (P)) +#define M8_F(X, P, N) (((uint8_t)(X) & M8_M(P, N)) >> (P)) +#define M8_I(X, P, N) (((uint8_t)(X) << (P)) & M8_M(P, N)) + #define NTB_LINK_STATUS_ACTIVE 0x2000 #define NTB_LINK_SPEED_MASK 0x000f #define NTB_LINK_WIDTH_MASK 0x03f0 @@ -164,5 +183,73 @@ /* The peer ntb secondary config space is 32KB fixed size */ #define XEON_B2B_MIN_SIZE 0x8000 +#define XEON_GEN3_MW_COUNT 2 +#define XEON_GEN3_SPLIT_MW_COUNT 3 +#define XEON_GEN3_SPAD_COUNT 16 +#define XEON_GEN3_DB_COUNT 32 +#define XEON_GEN3_DB_LINK 32 +#define XEON_GEN3_DB_LINK_BIT (1ULL << XEON_GEN3_DB_LINK) +#define XEON_GEN3_DB_MSIX_VECTOR_COUNT 33 +#define XEON_GEN3_DB_MSIX_VECTOR_SHIFT 1 + +#define XEON_GEN3_LINK_VECTOR_INDEX 31 + +/* Xeon Skylake NTB register definitions */ + +/* + * Internal EndPoint Configuration Registers + */ +#define XEON_GEN3_INT_REG_BAR0BASE 0x10 +#define XEON_GEN3_INT_REG_BAR1BASE 0x18 +#define XEON_GEN3_INT_REG_BAR2BASE 0x20 +#define XEON_GEN3_INT_REG_IMBAR1SZ 0xd0 +#define XEON_GEN3_INT_REG_IMBAR2SZ 0xd1 +#define XEON_GEN3_INT_REG_EMBAR1SZ 0xd2 +#define XEON_GEN3_INT_REG_EMBAR2SZ 0xd3 +#define XEON_GEN3_INT_REG_PPD 0xd4 +#define XEON_GEN3_INT_LNK_STS_OFFSET 0x01a2 + +/* + * External EndPoint Configuration Registers + * These are located within BAR0 of the internal endpoint. + */ +#define XEON_GEN3_EXT_REG_PCI_CMD 0x4504 +#define XEON_GEN3_EXT_REG_BAR0BASE 0x4510 +#define XEON_GEN3_EXT_REG_BAR1BASE 0x4518 +#define XEON_GEN3_EXT_REG_BAR2BASE 0x4520 + +/* + * Internal Endpoint Memory Mapped Registers + */ +#define XEON_GEN3_REG_IMNTB_CTRL 0x0000 +#define XEON_GEN3_REG_IMBAR1XBASE 0x0010 +#define XEON_GEN3_REG_IMBAR1XLIMIT 0x0018 +#define XEON_GEN3_REG_IMBAR2XBASE 0x0020 +#define XEON_GEN3_REG_IMBAR2XLIMIT 0x0028 +#define XEON_GEN3_REG_IMINT_STATUS 0x0040 +#define XEON_GEN3_REG_IMINT_DISABLE 0x0048 +#define XEON_GEN3_REG_IMSPAD 0x0080 +#define XEON_GEN3_REG_IMINTVEC00 0x00d0 +#define XEON_GEN3_REG_IMDOORBELL 0x0100 +#define XEON_GEN3_REG_IMB2B_SSPAD 0x0180 /* Pseudo SP registers */ + +/* + * External Endpoint Memory Mapped Registers + */ +#define XEON_GEN3_REG_EMBAR0XBASE 0x4008 +#define XEON_GEN3_REG_EMBAR1XBASE 0x4010 +#define XEON_GEN3_REG_EMBAR1XLIMIT 0x4018 +#define XEON_GEN3_REG_EMBAR2XBASE 0x4020 +#define XEON_GEN3_REG_EMBAR2XLIMIT 0x4028 +#define XEON_GEN3_REG_EMINT_STATUS 0x4040 +#define XEON_GEN3_REG_EMINT_DISABLE 0x4048 +#define XEON_GEN3_REG_EMSPAD 0x4080 +#define XEON_GEN3_REG_EMDOORBELL 0x4100 + +/* XEON_GEN3_INT_REG_PPD: PPD register */ +#define XEON_GEN3_REG_PPD_PORT_DEF_F(X) M8_F(X, 0, 2) +#define XEON_GEN3_REG_PPD_CONF_STS_F(X) M8_F(X, 4, 1) +#define XEON_GEN3_REG_PPD_ONE_MSIX_F(X) M8_F(X, 5, 1) +#define XEON_GEN3_REG_PPD_BAR45_SPL_F(X) M8_F(X, 6, 1) #endif /* _NTB_REGS_H_ */ Index: head/sys/dev/ntb/ntb_hw/ntb_hw_intel.c =================================================================== --- head/sys/dev/ntb/ntb_hw/ntb_hw_intel.c +++ head/sys/dev/ntb/ntb_hw/ntb_hw_intel.c @@ -68,7 +68,8 @@ #include "ntb_hw_intel.h" #include "../ntb.h" -#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) +#define MAX_MSIX_INTERRUPTS \ + MAX(MAX(XEON_DB_COUNT, ATOM_DB_COUNT), XEON_GEN3_DB_COUNT) #define NTB_HB_TIMEOUT 1 /* second */ #define ATOM_LINK_RECOVERY_TIME 500 /* ms */ @@ -87,7 +88,8 @@ #define PCI_MSIX_ENTRY_DATA 8 enum ntb_device_type { - NTB_XEON, + NTB_XEON_GEN1, + NTB_XEON_GEN3, NTB_ATOM }; @@ -338,6 +340,7 @@ static void intel_ntb_unmap_pci_bar(struct ntb_softc *ntb); static int intel_ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); static int intel_ntb_init_isr(struct ntb_softc *ntb); +static int intel_ntb_xeon_gen3_init_isr(struct ntb_softc *ntb); static int intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb); static int intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); static void intel_ntb_teardown_interrupts(struct ntb_softc *ntb); @@ -355,8 +358,10 @@ static struct ntb_hw_info *intel_ntb_get_device_info(uint32_t device_id); static void intel_ntb_detect_max_mw(struct ntb_softc *ntb); static int intel_ntb_detect_xeon(struct ntb_softc *ntb); +static int intel_ntb_detect_xeon_gen3(struct ntb_softc *ntb); static int intel_ntb_detect_atom(struct ntb_softc *ntb); static int intel_ntb_xeon_init_dev(struct ntb_softc *ntb); +static int intel_ntb_xeon_gen3_init_dev(struct ntb_softc *ntb); static int intel_ntb_atom_init_dev(struct ntb_softc *ntb); static void intel_ntb_teardown_xeon(struct ntb_softc *ntb); static void configure_atom_secondary_side_bars(struct ntb_softc *ntb); @@ -368,6 +373,9 @@ enum ntb_bar idx); static int xeon_setup_b2b_mw(struct ntb_softc *, const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); +static int xeon_gen3_setup_b2b_mw(struct ntb_softc *); +static int intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, + size_t size); static inline bool link_is_up(struct ntb_softc *ntb); static inline bool _xeon_link_is_up(struct ntb_softc *ntb); static inline bool atom_link_is_err(struct ntb_softc *ntb); @@ -479,6 +487,7 @@ #define NTB_B2BDOORBELL_BIT14 (1 << 3) /* Software/configuration owns the top 16 bits. */ #define NTB_SPLIT_BAR (1ull << 16) +#define NTB_ONE_MSIX (1ull << 17) #define NTB_FEATURES_STR \ "\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \ @@ -490,18 +499,21 @@ NTB_ATOM, 0 }, { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", - NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, + NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", - NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, - { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, - NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | + NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, + { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", + NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, - { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, - NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | + { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", + NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP }, - { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, - NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | + { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", + NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP }, + + { 0x201C8086, "SKL Xeon E5 V5 Non-Transparent Bridge B2B", + NTB_XEON_GEN3, 0 }, }; static const struct ntb_reg atom_reg = { @@ -586,6 +598,37 @@ .bar5_addr32 = XEON_B2B_BAR5_ADDR32, }; +static const struct ntb_reg xeon_gen3_reg = { + .ntb_ctl = XEON_GEN3_REG_IMNTB_CTRL, + .lnk_sta = XEON_GEN3_INT_LNK_STS_OFFSET, + .db_size = sizeof(uint32_t), + .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, +}; + +static const struct ntb_alt_reg xeon_gen3_pri_reg = { + .db_bell = XEON_GEN3_REG_EMDOORBELL, + .db_mask = XEON_GEN3_REG_IMINT_DISABLE, + .spad = XEON_GEN3_REG_IMSPAD, +}; + +static const struct ntb_alt_reg xeon_gen3_b2b_reg = { + .db_bell = XEON_GEN3_REG_IMDOORBELL, + .db_mask = XEON_GEN3_REG_EMINT_DISABLE, + .spad = XEON_GEN3_REG_IMB2B_SSPAD, +}; + +static const struct ntb_xlat_reg xeon_gen3_sec_xlat = { + .bar0_base = XEON_GEN3_EXT_REG_BAR0BASE, + .bar2_base = XEON_GEN3_EXT_REG_BAR1BASE, + .bar4_base = XEON_GEN3_EXT_REG_BAR2BASE, + + .bar2_limit = XEON_GEN3_REG_IMBAR1XLIMIT, + .bar4_limit = XEON_GEN3_REG_IMBAR2XLIMIT, + + .bar2_xlat = XEON_GEN3_REG_IMBAR1XBASE, + .bar4_xlat = XEON_GEN3_REG_IMBAR2XBASE, +}; + SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "B2B MW segment overrides -- MUST be the same on both sides"); @@ -662,6 +705,8 @@ if (ntb->type == NTB_ATOM) error = intel_ntb_detect_atom(ntb); + else if (ntb->type == NTB_XEON_GEN3) + error = intel_ntb_detect_xeon_gen3(ntb); else error = intel_ntb_detect_xeon(ntb); if (error != 0) @@ -676,6 +721,8 @@ goto out; if (ntb->type == NTB_ATOM) error = intel_ntb_atom_init_dev(ntb); + else if (ntb->type == NTB_XEON_GEN3) + error = intel_ntb_xeon_gen3_init_dev(ntb); else error = intel_ntb_xeon_init_dev(ntb); if (error != 0) @@ -715,7 +762,7 @@ callout_drain(&ntb->lr_timer); callout_drain(&ntb->peer_msix_work); pci_disable_busmaster(ntb->device); - if (ntb->type == NTB_XEON) + if (ntb->type == NTB_XEON_GEN1) intel_ntb_teardown_xeon(ntb); intel_ntb_teardown_interrupts(ntb); @@ -825,22 +872,39 @@ rc = map_memory_window_bar(ntb, bar); if (rc != 0) goto out; - bar->psz_off = XEON_PBAR23SZ_OFFSET; - bar->ssz_off = XEON_SBAR23SZ_OFFSET; - bar->pbarxlat_off = XEON_PBAR2XLAT_OFFSET; + if (ntb->type == NTB_XEON_GEN3) { + bar->psz_off = XEON_GEN3_INT_REG_IMBAR1SZ; + bar->ssz_off = XEON_GEN3_INT_REG_EMBAR1SZ; + bar->pbarxlat_off = XEON_GEN3_REG_EMBAR1XBASE; + } else { + bar->psz_off = XEON_PBAR23SZ_OFFSET; + bar->ssz_off = XEON_SBAR23SZ_OFFSET; + bar->pbarxlat_off = XEON_PBAR2XLAT_OFFSET; + } bar = &ntb->bar_info[NTB_B2B_BAR_2]; bar->pci_resource_id = PCIR_BAR(4); rc = map_memory_window_bar(ntb, bar); if (rc != 0) goto out; - bar->psz_off = XEON_PBAR4SZ_OFFSET; - bar->ssz_off = XEON_SBAR4SZ_OFFSET; - bar->pbarxlat_off = XEON_PBAR4XLAT_OFFSET; + if (ntb->type == NTB_XEON_GEN3) { + bar->psz_off = XEON_GEN3_INT_REG_IMBAR2SZ; + bar->ssz_off = XEON_GEN3_INT_REG_EMBAR2SZ; + bar->pbarxlat_off = XEON_GEN3_REG_EMBAR2XBASE; + } else { + bar->psz_off = XEON_PBAR4SZ_OFFSET; + bar->ssz_off = XEON_SBAR4SZ_OFFSET; + bar->pbarxlat_off = XEON_PBAR4XLAT_OFFSET; + } if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR)) goto out; + if (ntb->type == NTB_XEON_GEN3) { + device_printf(ntb->device, "no split bar support\n"); + return (ENXIO); + } + bar = &ntb->bar_info[NTB_B2B_BAR_3]; bar->pci_resource_id = PCIR_BAR(5); rc = map_memory_window_bar(ntb, bar); @@ -1060,6 +1124,63 @@ } static int +intel_ntb_xeon_gen3_init_isr(struct ntb_softc *ntb) +{ + uint64_t i, reg; + uint32_t desired_vectors, num_vectors; + int rc; + + ntb->allocated_interrupts = 0; + ntb->last_ts = ticks; + + /* Mask all the interrupts, including hardware interrupt */ + intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_DISABLE, ~0ULL); + + /* Clear Interrupt Status */ + reg = intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS); + intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_STATUS, reg); + + num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), + XEON_GEN3_DB_MSIX_VECTOR_COUNT); + + rc = pci_alloc_msix(ntb->device, &num_vectors); + if (rc != 0) { + device_printf(ntb->device, + "Interrupt allocation failed %d\n", rc); + return (rc); + } + if (desired_vectors != num_vectors) { + device_printf(ntb->device, "Couldn't get %d vectors\n", + XEON_GEN3_DB_MSIX_VECTOR_COUNT); + return (ENXIO); + } + /* 32 db + 1 hardware */ + if (num_vectors == XEON_GEN3_DB_MSIX_VECTOR_COUNT) { + /* Program INTVECXX source register */ + for (i = 0; i < XEON_GEN3_DB_MSIX_VECTOR_COUNT; i++) { + /* interrupt source i for vector i */ + intel_ntb_reg_write(1, XEON_GEN3_REG_IMINTVEC00 + i, i); + if (i == (XEON_GEN3_DB_MSIX_VECTOR_COUNT - 1)) { + intel_ntb_reg_write(1, + XEON_GEN3_REG_IMINTVEC00 + i, + XEON_GEN3_LINK_VECTOR_INDEX); + } + } + + intel_ntb_create_msix_vec(ntb, num_vectors); + rc = intel_ntb_setup_msix(ntb, num_vectors); + + /* enable all interrupts */ + intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_DISABLE, 0ULL); + } else { + device_printf(ntb->device, "need to remap interrupts, giving up.\n"); + return (ENXIO); + } + + return (0); +} + +static int intel_ntb_init_isr(struct ntb_softc *ntb) { uint32_t desired_vectors, num_vectors; @@ -1098,7 +1219,7 @@ } else num_vectors = 1; - if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { + if (ntb->type == NTB_XEON_GEN1 && num_vectors < ntb->db_vec_count) { if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { device_printf(ntb->device, "Errata workaround does not support MSI or INTX\n"); @@ -1177,20 +1298,17 @@ pci_release_msi(ntb->device); } -/* - * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it - * out to make code clearer. - */ static inline uint64_t db_ioread(struct ntb_softc *ntb, uint64_t regoff) { - if (ntb->type == NTB_ATOM) + switch (ntb->type) { + case NTB_ATOM: + case NTB_XEON_GEN3: return (intel_ntb_reg_read(8, regoff)); - - KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); - - return (intel_ntb_reg_read(2, regoff)); + case NTB_XEON_GEN1: + return (intel_ntb_reg_read(2, regoff)); + } } static inline void @@ -1211,13 +1329,15 @@ db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) { - if (ntb->type == NTB_ATOM) { + switch (ntb->type) { + case NTB_ATOM: + case NTB_XEON_GEN3: intel_ntb_reg_write(8, regoff, val); - return; + break; + case NTB_XEON_GEN1: + intel_ntb_reg_write(2, regoff, (uint16_t)val); + break; } - - KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); - intel_ntb_reg_write(2, regoff, (uint16_t)val); } static void @@ -1267,8 +1387,10 @@ if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) return (ntb->fake_db); - - return (db_ioread(ntb, ntb->self_reg->db_bell)); + if (ntb->type == NTB_XEON_GEN3) + return (intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS)); + else + return (db_ioread(ntb, ntb->self_reg->db_bell)); } static void @@ -1288,7 +1410,11 @@ return; } - db_iowrite(ntb, ntb->self_reg->db_bell, bits); + if (ntb->type == NTB_XEON_GEN3) + intel_ntb_reg_write(4, XEON_GEN3_REG_IMINT_STATUS, + (uint32_t)bits); + else + db_iowrite(ntb, ntb->self_reg->db_bell, bits); } static inline uint64_t @@ -1322,9 +1448,14 @@ ntb->last_ts = ticks; vec_mask = intel_ntb_vec_mask(ntb, vec); + if (ntb->type == NTB_XEON_GEN3 && vec == XEON_GEN3_LINK_VECTOR_INDEX) + vec_mask |= ntb->db_link_mask; if ((vec_mask & ntb->db_link_mask) != 0) { if (intel_ntb_poll_link(ntb)) ntb_link_event(ntb->device); + if (ntb->type == NTB_XEON_GEN3) + intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_STATUS, + intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS)); } if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && @@ -1449,15 +1580,23 @@ intel_ntb_detect_max_mw(struct ntb_softc *ntb) { - if (ntb->type == NTB_ATOM) { + switch (ntb->type) { + case NTB_ATOM: ntb->mw_count = ATOM_MW_COUNT; - return; + break; + case NTB_XEON_GEN1: + if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) + ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; + else + ntb->mw_count = XEON_SNB_MW_COUNT; + break; + case NTB_XEON_GEN3: + if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) + ntb->mw_count = XEON_GEN3_SPLIT_MW_COUNT; + else + ntb->mw_count = XEON_GEN3_MW_COUNT; + break; } - - if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) - ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; - else - ntb->mw_count = XEON_SNB_MW_COUNT; } static int @@ -1534,6 +1673,54 @@ } static int +intel_ntb_detect_xeon_gen3(struct ntb_softc *ntb) +{ + uint8_t ppd, conn_type; + + ppd = pci_read_config(ntb->device, XEON_GEN3_INT_REG_PPD, 1); + ntb->ppd = ppd; + + /* check port definition */ + conn_type = XEON_GEN3_REG_PPD_PORT_DEF_F(ppd); + switch (conn_type) { + case NTB_CONN_B2B: + ntb->conn_type = conn_type; + break; + default: + device_printf(ntb->device, "Unsupported connection type: %u\n", + conn_type); + return (ENXIO); + } + + /* check cross link configuration status */ + if (XEON_GEN3_REG_PPD_CONF_STS_F(ppd)) { + /* NTB Port is configured as DSD/USP */ + ntb->dev_type = NTB_DEV_DSD; + } else { + /* NTB Port is configured as USD/DSP */ + ntb->dev_type = NTB_DEV_USD; + } + + if (XEON_GEN3_REG_PPD_ONE_MSIX_F(ppd)) { + /* + * This bit when set, causes only a single MSI-X message to be + * generated if MSI-X is enabled. + */ + ntb->features |= NTB_ONE_MSIX; + } + + if (XEON_GEN3_REG_PPD_BAR45_SPL_F(ppd)) { + /* BARs 4 and 5 are presented as two 32b non-prefetchable BARs */ + ntb->features |= NTB_SPLIT_BAR; + } + + device_printf(ntb->device, "conn type 0x%02x, dev type 0x%02x," + "features 0x%02x\n", ntb->conn_type, ntb->dev_type, ntb->features); + + return (0); +} + +static int intel_ntb_xeon_init_dev(struct ntb_softc *ntb) { int rc; @@ -1618,6 +1805,42 @@ } static int +intel_ntb_xeon_gen3_init_dev(struct ntb_softc *ntb) +{ + int rc; + + ntb->spad_count = XEON_GEN3_SPAD_COUNT; + ntb->db_count = XEON_GEN3_DB_COUNT; + ntb->db_link_mask = XEON_GEN3_DB_LINK_BIT; + ntb->db_vec_count = XEON_GEN3_DB_MSIX_VECTOR_COUNT; + ntb->db_vec_shift = XEON_GEN3_DB_MSIX_VECTOR_SHIFT; + + if (ntb->conn_type != NTB_CONN_B2B) { + device_printf(ntb->device, "Connection type %d not supported\n", + ntb->conn_type); + return (ENXIO); + } + + ntb->reg = &xeon_gen3_reg; + ntb->self_reg = &xeon_gen3_pri_reg; + ntb->peer_reg = &xeon_gen3_b2b_reg; + ntb->xlat_reg = &xeon_gen3_sec_xlat; + + ntb->db_valid_mask = (1ULL << ntb->db_count) - 1; + + xeon_gen3_setup_b2b_mw(ntb); + + /* Enable Bus Master and Memory Space on the External Side */ + intel_ntb_reg_write(2, XEON_GEN3_EXT_REG_PCI_CMD, + PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); + + /* Setup Interrupt */ + rc = intel_ntb_xeon_gen3_init_isr(ntb); + + return (rc); +} + +static int intel_ntb_atom_init_dev(struct ntb_softc *ntb) { int error; @@ -1908,6 +2131,50 @@ return (0); } +static int +xeon_gen3_setup_b2b_mw(struct ntb_softc *ntb) +{ + uint64_t reg; + uint32_t embarsz, imbarsz; + + /* IMBAR1SZ should be equal to EMBAR1SZ */ + embarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_EMBAR1SZ, 1); + imbarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_IMBAR1SZ, 1); + if (embarsz != imbarsz) { + device_printf(ntb->device, + "IMBAR1SZ (%u) should be equal to EMBAR1SZ (%u)\n", + imbarsz, embarsz); + return (EIO); + } + + /* IMBAR2SZ should be equal to EMBAR2SZ */ + embarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_EMBAR2SZ, 1); + imbarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_IMBAR2SZ, 1); + if (embarsz != imbarsz) { + device_printf(ntb->device, + "IMBAR2SZ (%u) should be equal to EMBAR2SZ (%u)\n", + imbarsz, embarsz); + return (EIO); + } + + /* Client will provide the incoming IMBAR1/2XBASE, zero it for now */ + intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR1XBASE, 0); + intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR2XBASE, 0); + + /* + * If the value in EMBAR1LIMIT is set equal to the value in EMBAR1, + * the memory window for EMBAR1 is disabled. + * Note: It is needed to avoid malacious access. + */ + reg = pci_read_config(ntb->device, XEON_GEN3_EXT_REG_BAR1BASE, 8); + intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR1XLIMIT, reg); + + reg = pci_read_config(ntb->device, XEON_GEN3_EXT_REG_BAR2BASE, 8); + intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR2XLIMIT, reg); + + return (0); +} + static inline bool _xeon_link_is_up(struct ntb_softc *ntb) { @@ -1921,7 +2188,7 @@ link_is_up(struct ntb_softc *ntb) { - if (ntb->type == NTB_XEON) + if (ntb->type == NTB_XEON_GEN1 || ntb->type == NTB_XEON_GEN3) return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || !HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))); @@ -2185,7 +2452,9 @@ ntb->ntb_ctl = ntb_cntl; ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta); } else { - db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); + if (ntb->type == NTB_XEON_GEN1) + db_iowrite_raw(ntb, ntb->self_reg->db_bell, + ntb->db_link_mask); reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); if (reg_val == ntb->lnk_sta) @@ -2960,6 +3229,8 @@ if (limit_reg != 0 && size != mw_size) limit = base + size; + else + limit = base + mw_size; /* Set and verify translation address */ intel_ntb_reg_write(8, xlat_reg, addr); @@ -2977,8 +3248,22 @@ intel_ntb_reg_write(8, xlat_reg, 0); return (EIO); } + + if (ntb->type == NTB_XEON_GEN3) { + limit = base + size; + + /* set EMBAR1/2XLIMIT */ + if (!idx) + intel_ntb_reg_write(8, + XEON_GEN3_REG_EMBAR1XLIMIT, limit); + else + intel_ntb_reg_write(8, + XEON_GEN3_REG_EMBAR2XLIMIT, limit); + } } else { /* Configure 32-bit (split) BAR MW */ + if (ntb->type == NTB_XEON_GEN3) + return (EIO); if ((addr & UINT32_MAX) != addr) return (ERANGE); @@ -3062,10 +3347,16 @@ } static void -intel_ntb_peer_db_set(device_t dev, uint64_t bit) +intel_ntb_peer_db_set(device_t dev, uint64_t bits) { struct ntb_softc *ntb = device_get_softc(dev); + uint64_t db; + if ((bits & ~ntb->db_valid_mask) != 0) { + device_printf(ntb->device, "Invalid doorbell bits %lx\n", bits); + return; + } + if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { struct ntb_pci_bar_info *lapic; unsigned i; @@ -3073,7 +3364,7 @@ lapic = ntb->peer_lapic_bar; for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { - if ((bit & intel_ntb_db_vector_mask(dev, i)) != 0) + if ((bits & intel_ntb_db_vector_mask(dev, i)) != 0) bus_space_write_4(lapic->pci_bus_tag, lapic->pci_bus_handle, ntb->peer_msix_data[i].nmd_ofs, @@ -3083,11 +3374,22 @@ } if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { - intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); + intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bits); return; } - db_iowrite(ntb, ntb->peer_reg->db_bell, bit); + if (ntb->type == NTB_XEON_GEN3) { + while (bits != 0) { + db = ffsll(bits); + + intel_ntb_reg_write(1, + ntb->peer_reg->db_bell + (db - 1) * 4, 0x1); + + bits = bits & (bits - 1); + } + } else { + db_iowrite(ntb, ntb->peer_reg->db_bell, bits); + } } static int