Index: head/sys/amd64/conf/NOTES =================================================================== --- head/sys/amd64/conf/NOTES +++ head/sys/amd64/conf/NOTES @@ -383,6 +383,7 @@ device if_ntb # Virtual NTB network interface device ntb_transport # NTB packet transport driver device ntb # NTB hardware interface +device ntb_hw_amd # AMD NTB hardware driver device ntb_hw_intel # Intel NTB hardware driver device ntb_hw_plx # PLX NTB hardware driver Index: head/sys/conf/files.amd64 =================================================================== --- head/sys/conf/files.amd64 +++ head/sys/conf/files.amd64 @@ -352,8 +352,9 @@ dev/nfe/if_nfe.c optional nfe pci dev/ntb/if_ntb/if_ntb.c optional if_ntb dev/ntb/ntb_transport.c optional ntb_transport | if_ntb -dev/ntb/ntb.c optional ntb | ntb_transport | if_ntb | ntb_hw_intel | ntb_hw_plx | ntb_hw -dev/ntb/ntb_if.m optional ntb | ntb_transport | if_ntb | ntb_hw_intel | ntb_hw_plx | ntb_hw +dev/ntb/ntb.c optional ntb | ntb_transport | if_ntb | ntb_hw_amd | ntb_hw_intel | ntb_hw_plx | ntb_hw +dev/ntb/ntb_if.m optional ntb | ntb_transport | if_ntb | ntb_hw_amd | ntb_hw_intel | ntb_hw_plx | ntb_hw +dev/ntb/ntb_hw/ntb_hw_amd.c optional ntb_hw_amd | ntb_hw dev/ntb/ntb_hw/ntb_hw_intel.c optional ntb_hw_intel | ntb_hw dev/ntb/ntb_hw/ntb_hw_plx.c optional ntb_hw_plx | ntb_hw dev/nvd/nvd.c optional nvd nvme Index: head/sys/conf/files.i386 =================================================================== --- head/sys/conf/files.i386 +++ head/sys/conf/files.i386 @@ -284,8 +284,9 @@ dev/nfe/if_nfe.c optional nfe pci dev/ntb/if_ntb/if_ntb.c optional if_ntb dev/ntb/ntb_transport.c optional ntb_transport | if_ntb -dev/ntb/ntb.c optional ntb | ntb_transport | if_ntb | ntb_hw_intel | ntb_hw_plx | ntb_hw -dev/ntb/ntb_if.m optional ntb | ntb_transport | if_ntb | ntb_hw_intel | ntb_hw_plx | ntb_hw +dev/ntb/ntb.c optional ntb | ntb_transport | if_ntb | ntb_hw_amd | ntb_hw_intel | ntb_hw_plx | ntb_hw +dev/ntb/ntb_if.m optional ntb | ntb_transport | if_ntb | ntb_hw_amd | ntb_hw_intel | ntb_hw_plx | ntb_hw +dev/ntb/ntb_hw/ntb_hw_amd.c optional ntb_hw_amd | ntb_hw dev/ntb/ntb_hw/ntb_hw_intel.c optional ntb_hw_intel | ntb_hw dev/ntb/ntb_hw/ntb_hw_plx.c optional ntb_hw_plx | ntb_hw dev/nvd/nvd.c optional nvd nvme Index: head/sys/dev/ntb/ntb.h =================================================================== --- head/sys/dev/ntb/ntb.h +++ head/sys/dev/ntb/ntb.h @@ -64,6 +64,51 @@ */ void ntb_db_event(device_t ntb, uint32_t vec); +/** + * ntb_port_number() - get the local port number + * @ntb: NTB device context. + * + * Hardware driver returns local port number in compliance with topology. + * + * Return: the local port number + */ +int ntb_port_number(device_t ntb); + +/** + * ntb_port_count() - get the number of peer device ports + * @ntb: NTB device context. + * + * By default hardware driver supports just one peer device. + * + * Return: the number of peer ports + */ +int ntb_peer_port_count(device_t ntb); + +/** + * ntb_peer_port_number() - get the peer port by given index + * @ntb: NTB device context. + * @idx: Peer port index (should be zero for now). + * + * By default hardware driver supports just one peer device, so this method + * shall return the corresponding value. + * + * Return: the peer device port or an error number + */ +int ntb_peer_port_number(device_t ntb, int pidx); + +/* + * ntb_peer_port_idx() - get the peer device port index by given port + * number + * @ntb: NTB device context. + * @port: Peer port number + * + * By default hardware driver supports just one peer device, so given a + * valid peer port number, the return value shall be zero. + * + * Return: the peer port index or an error number + */ +int ntb_peer_port_idx(device_t ntb, int port); + /* * ntb_link_is_up() - get the current ntb link state * @ntb: NTB device context Index: head/sys/dev/ntb/ntb.c =================================================================== --- head/sys/dev/ntb/ntb.c +++ head/sys/dev/ntb/ntb.c @@ -243,6 +243,30 @@ } } +int +ntb_port_number(device_t ntb) +{ + return (NTB_PORT_NUMBER(device_get_parent(ntb))); +} + +int +ntb_peer_port_count(device_t ntb) +{ + return (NTB_PEER_PORT_COUNT(device_get_parent(ntb))); +} + +int +ntb_peer_port_number(device_t ntb, int pidx) +{ + return (NTB_PEER_PORT_NUMBER(device_get_parent(ntb), pidx)); +} + +int +ntb_peer_port_idx(device_t ntb, int port) +{ + return (NTB_PEER_PORT_IDX(device_get_parent(ntb), port)); +} + bool ntb_link_is_up(device_t ntb, enum ntb_speed *speed, enum ntb_width *width) { Index: head/sys/dev/ntb/ntb_hw/ntb_hw_amd.h =================================================================== --- head/sys/dev/ntb/ntb_hw/ntb_hw_amd.h +++ head/sys/dev/ntb/ntb_hw/ntb_hw_amd.h @@ -0,0 +1,259 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright (C) 2019 Advanced Micro Devices, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * BSD LICENSE + * + * Copyright (C) 2019 Advanced Micro Devices, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copy + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the distribution. + * 3. Neither the name of AMD corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Contact Information : + * Rajesh Kumar + * + * $FreeBSD$ + */ + +#ifndef NTB_HW_AMD_H +#define NTB_HW_AMD_H + +#define NTB_HW_AMD_VENDOR_ID 0x1022 +#define NTB_HW_AMD_DEVICE_ID 0x145B + +#define NTB_DEF_PEER_CNT 1 +#define NTB_DEF_PEER_IDX 0 + +#define BIT(n) (1 << n) +#define AMD_LINK_HB_TIMEOUT (1 * hz) + +#define NTB_LIN_STA_ACTIVE_BIT 0x00000002 +#define NTB_LNK_STA_SPEED_MASK 0x000F0000 +#define NTB_LNK_STA_WIDTH_MASK 0x03F00000 +#define NTB_LNK_STA_ACTIVE(x) (!!((x) & NTB_LIN_STA_ACTIVE_BIT)) +#define NTB_LNK_STA_SPEED(x) (((x) & NTB_LNK_STA_SPEED_MASK) >> 16) +#define NTB_LNK_STA_WIDTH(x) (((x) & NTB_LNK_STA_WIDTH_MASK) >> 20) + +#define amd_ntb_bar_read(SIZE, bar, offset) \ + bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ + ntb->bar_info[(bar)].pci_bus_handle, (offset)) +#define amd_ntb_bar_write(SIZE, bar, offset, val) \ + bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ + ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) +#define amd_ntb_reg_read(SIZE, offset) \ + amd_ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) +#define amd_ntb_reg_write(SIZE, offset, val) \ + amd_ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) +#define amd_ntb_peer_reg_read(SIZE, offset) \ + amd_ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset + AMD_PEER_OFFSET) +#define amd_ntb_peer_reg_write(SIZE, offset, val) \ + amd_ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset + AMD_PEER_OFFSET, val) + +#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) +#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) +#define DB_MASK_ASSERT(sc, f) mtx_assert(&(sc)->db_mask_lock, (f)) + +/* amd_ntb_conn_type are hardware numbers, cannot change. */ +enum amd_ntb_conn_type { + NTB_CONN_NONE = -1, + NTB_CONN_PRI, + NTB_CONN_SEC, +}; + +enum ntb_default_port { + NTB_PORT_PRI_USD, + NTB_PORT_SEC_DSD +}; + +enum amd_ntb_bar { + NTB_CONFIG_BAR = 0, + NTB_BAR_1, + NTB_BAR_2, + NTB_BAR_3, + NTB_MAX_BARS +}; + +struct amd_ntb_hw_info { + uint32_t device_id; + const char *desc; +}; + +struct amd_ntb_pci_bar_info { + bus_space_tag_t pci_bus_tag; + bus_space_handle_t pci_bus_handle; + struct resource *pci_resource; + vm_paddr_t pbase; + caddr_t vbase; + vm_size_t size; + vm_memattr_t map_mode; + int pci_resource_id; + + /* Configuration register offsets */ + uint32_t xlat_off; + uint32_t limit_off; +}; + +struct amd_ntb_int_info { + struct resource *res; + void *tag; + int rid; +}; + +struct amd_ntb_vec { + struct amd_ntb_softc *ntb; + uint32_t num; + unsigned masked; +}; + +enum { + /* AMD NTB Capability */ + AMD_MW_CNT = 3, + AMD_DB_CNT = 16, + AMD_MSIX_VECTOR_CNT = 24, + AMD_SPADS_CNT = 16, + + /* AMD NTB Link Status Offset */ + AMD_LINK_STATUS_OFFSET = 0x68, + + /* AMD NTB register offset */ + AMD_CNTL_OFFSET = 0x200, + + /* NTB control register bits */ + PMM_REG_CTL = BIT(21), + SMM_REG_CTL = BIT(20), + SMM_REG_ACC_PATH = BIT(18), + PMM_REG_ACC_PATH = BIT(17), + NTB_CLK_EN = BIT(16), + + AMD_STA_OFFSET = 0x204, + AMD_PGSLV_OFFSET = 0x208, + AMD_SPAD_MUX_OFFSET = 0x20C, + AMD_SPAD_OFFSET = 0x210, + AMD_RSMU_HCID = 0x250, + AMD_RSMU_SIID = 0x254, + AMD_PSION_OFFSET = 0x300, + AMD_SSION_OFFSET = 0x330, + AMD_MMINDEX_OFFSET = 0x400, + AMD_MMDATA_OFFSET = 0x404, + AMD_SIDEINFO_OFFSET = 0x408, + + AMD_SIDE_MASK = BIT(0), + AMD_SIDE_READY = BIT(1), + + /* limit register */ + AMD_ROMBARLMT_OFFSET = 0x410, + AMD_BAR1LMT_OFFSET = 0x414, + AMD_BAR23LMT_OFFSET = 0x418, + AMD_BAR45LMT_OFFSET = 0x420, + + /* xlat address */ + AMD_ROMBARXLAT_OFFSET = 0x428, + AMD_BAR1XLAT_OFFSET = 0x430, + AMD_BAR23XLAT_OFFSET = 0x438, + AMD_BAR45XLAT_OFFSET = 0x440, + + /* doorbell and interrupt */ + AMD_DBFM_OFFSET = 0x450, + AMD_DBREQ_OFFSET = 0x454, + AMD_MIRRDBSTAT_OFFSET = 0x458, + AMD_DBMASK_OFFSET = 0x45C, + AMD_DBSTAT_OFFSET = 0x460, + AMD_INTMASK_OFFSET = 0x470, + AMD_INTSTAT_OFFSET = 0x474, + + /* event type */ + AMD_PEER_FLUSH_EVENT = BIT(0), + AMD_PEER_RESET_EVENT = BIT(1), + AMD_PEER_D3_EVENT = BIT(2), + AMD_PEER_PMETO_EVENT = BIT(3), + AMD_PEER_D0_EVENT = BIT(4), + AMD_LINK_UP_EVENT = BIT(5), + AMD_LINK_DOWN_EVENT = BIT(6), + AMD_EVENT_INTMASK = (AMD_PEER_FLUSH_EVENT | + AMD_PEER_RESET_EVENT | AMD_PEER_D3_EVENT | + AMD_PEER_PMETO_EVENT | AMD_PEER_D0_EVENT | + AMD_LINK_UP_EVENT | AMD_LINK_DOWN_EVENT), + + AMD_PMESTAT_OFFSET = 0x480, + AMD_PMSGTRIG_OFFSET = 0x490, + AMD_LTRLATENCY_OFFSET = 0x494, + AMD_FLUSHTRIG_OFFSET = 0x498, + + /* SMU register*/ + AMD_SMUACK_OFFSET = 0x4A0, + AMD_SINRST_OFFSET = 0x4A4, + AMD_RSPNUM_OFFSET = 0x4A8, + AMD_SMU_SPADMUTEX = 0x4B0, + AMD_SMU_SPADOFFSET = 0x4B4, + + AMD_PEER_OFFSET = 0x400, +}; + +struct amd_ntb_softc { + /* ntb.c context. Do not move! Must go first! */ + void *ntb_store; + + device_t device; + enum amd_ntb_conn_type conn_type; + + struct amd_ntb_pci_bar_info bar_info[NTB_MAX_BARS]; + struct amd_ntb_int_info int_info[AMD_MSIX_VECTOR_CNT]; + struct amd_ntb_vec *msix_vec; + uint16_t allocated_interrupts; + + struct callout hb_timer; + + uint8_t mw_count; + uint8_t spad_count; + uint8_t db_count; + uint8_t msix_vec_count; + + struct mtx db_mask_lock; + + volatile uint32_t ntb_ctl; + volatile uint32_t lnk_sta; + volatile uint32_t peer_sta; + volatile uint32_t cntl_sta; + + uint16_t db_valid_mask; + uint16_t db_mask; + uint32_t int_mask; + + unsigned int self_spad; + unsigned int peer_spad; +}; + +static void amd_init_side_info(struct amd_ntb_softc *ntb); +static void amd_deinit_side_info(struct amd_ntb_softc *ntb); +static int amd_ntb_detach(device_t device); + +#endif Index: head/sys/dev/ntb/ntb_hw/ntb_hw_amd.c =================================================================== --- head/sys/dev/ntb/ntb_hw/ntb_hw_amd.c +++ head/sys/dev/ntb/ntb_hw/ntb_hw_amd.c @@ -0,0 +1,1247 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright (C) 2019 Advanced Micro Devices, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * BSD LICENSE + * + * Copyright (c) 2019 Advanced Micro Devices, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of AMD corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Contact Information : + * Rajesh Kumar + */ + +/* + * The Non-Transparent Bridge (NTB) is a device that allows you to connect + * two or more systems using a PCI-e links, providing remote memory access. + * + * This module contains a driver for NTB hardware in AMD CPUs + * + * Much of the code in this module is shared with Linux. Any patches may + * be picked up and redistributed in Linux with a dual GPL/BSD license. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include + +#include "ntb_hw_amd.h" +#include "dev/ntb/ntb.h" + +MALLOC_DEFINE(M_AMD_NTB, "amd_ntb_hw", "amd_ntb_hw driver memory allocations"); + +struct pci_device_table amd_ntb_devs[] = { + { PCI_DEV(NTB_HW_AMD_VENDOR_ID, NTB_HW_AMD_DEVICE_ID), + PCI_DESCR("AMD Non-Transparent Bridge") } +}; + +static unsigned g_amd_ntb_hw_debug_level; +SYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, + &g_amd_ntb_hw_debug_level, 0, "amd_ntb_hw log level -- higher is verbose"); + +#define amd_ntb_printf(lvl, ...) do { \ + if (lvl <= g_amd_ntb_hw_debug_level) \ + device_printf(ntb->device, __VA_ARGS__); \ +} while (0) + +/* + * AMD NTB INTERFACE ROUTINES + */ +static int +amd_ntb_port_number(device_t dev) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + + amd_ntb_printf(1, "%s: conn_type %d\n", __func__, ntb->conn_type); + + switch (ntb->conn_type) { + case NTB_CONN_PRI: + return (NTB_PORT_PRI_USD); + case NTB_CONN_SEC: + return (NTB_PORT_SEC_DSD); + default: + break; + } + + return (-EINVAL); +} + +static int +amd_ntb_peer_port_count(device_t dev) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + + amd_ntb_printf(1, "%s: peer cnt %d\n", __func__, NTB_DEF_PEER_CNT); + return (NTB_DEF_PEER_CNT); +} + +static int +amd_ntb_peer_port_number(device_t dev, int pidx) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + + amd_ntb_printf(1, "%s: pidx %d conn type %d\n", + __func__, pidx, ntb->conn_type); + + if (pidx != NTB_DEF_PEER_IDX) + return (-EINVAL); + + switch (ntb->conn_type) { + case NTB_CONN_PRI: + return (NTB_PORT_SEC_DSD); + case NTB_CONN_SEC: + return (NTB_PORT_PRI_USD); + default: + break; + } + + return (-EINVAL); +} + +static int +amd_ntb_peer_port_idx(device_t dev, int port) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + int peer_port; + + peer_port = amd_ntb_peer_port_number(dev, NTB_DEF_PEER_IDX); + + amd_ntb_printf(1, "%s: port %d peer_port %d\n", + __func__, port, peer_port); + + if (peer_port == -EINVAL || port != peer_port) + return (-EINVAL); + + return (0); +} + +/* + * AMD NTB INTERFACE - LINK ROUTINES + */ +static inline int +amd_link_is_up(struct amd_ntb_softc *ntb) +{ + + amd_ntb_printf(2, "%s: peer_sta 0x%x cntl_sta 0x%x\n", + __func__, ntb->peer_sta, ntb->cntl_sta); + + if (!ntb->peer_sta) + return (NTB_LNK_STA_ACTIVE(ntb->cntl_sta)); + + return (0); +} + +static inline enum ntb_speed +amd_ntb_link_sta_speed(struct amd_ntb_softc *ntb) +{ + + if (!amd_link_is_up(ntb)) + return (NTB_SPEED_NONE); + + return (NTB_LNK_STA_SPEED(ntb->lnk_sta)); +} + +static inline enum ntb_width +amd_ntb_link_sta_width(struct amd_ntb_softc *ntb) +{ + + if (!amd_link_is_up(ntb)) + return (NTB_WIDTH_NONE); + + return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); +} + +static bool +amd_ntb_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + + if (speed != NULL) + *speed = amd_ntb_link_sta_speed(ntb); + if (width != NULL) + *width = amd_ntb_link_sta_width(ntb); + + return (amd_link_is_up(ntb)); +} + +static int +amd_ntb_link_enable(device_t dev, enum ntb_speed max_speed, + enum ntb_width max_width) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + uint32_t ntb_ctl; + + amd_ntb_printf(1, "%s: int_mask 0x%x conn_type %d\n", + __func__, ntb->int_mask, ntb->conn_type); + + amd_init_side_info(ntb); + + /* Enable event interrupt */ + ntb->int_mask &= ~AMD_EVENT_INTMASK; + amd_ntb_reg_write(4, AMD_INTMASK_OFFSET, ntb->int_mask); + + if (ntb->conn_type == NTB_CONN_SEC) + return (EINVAL); + + amd_ntb_printf(0, "%s: Enabling Link.\n", __func__); + + ntb_ctl = amd_ntb_reg_read(4, AMD_CNTL_OFFSET); + ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL); + amd_ntb_printf(1, "%s: ntb_ctl 0x%x\n", __func__, ntb_ctl); + amd_ntb_reg_write(4, AMD_CNTL_OFFSET, ntb_ctl); + + return (0); +} + +static int +amd_ntb_link_disable(device_t dev) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + uint32_t ntb_ctl; + + amd_ntb_printf(1, "%s: int_mask 0x%x conn_type %d\n", + __func__, ntb->int_mask, ntb->conn_type); + + amd_deinit_side_info(ntb); + + /* Disable event interrupt */ + ntb->int_mask |= AMD_EVENT_INTMASK; + amd_ntb_reg_write(4, AMD_INTMASK_OFFSET, ntb->int_mask); + + if (ntb->conn_type == NTB_CONN_SEC) + return (EINVAL); + + amd_ntb_printf(0, "%s: Disabling Link.\n", __func__); + + ntb_ctl = amd_ntb_reg_read(4, AMD_CNTL_OFFSET); + ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL); + amd_ntb_printf(1, "%s: ntb_ctl 0x%x\n", __func__, ntb_ctl); + amd_ntb_reg_write(4, AMD_CNTL_OFFSET, ntb_ctl); + + return (0); +} + +/* + * AMD NTB memory window routines + */ +static uint8_t +amd_ntb_mw_count(device_t dev) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + + return (ntb->mw_count); +} + +static int +amd_ntb_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, + caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, + bus_addr_t *plimit) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + struct amd_ntb_pci_bar_info *bar_info; + + if (mw_idx < 0 || mw_idx >= ntb->mw_count) + return (EINVAL); + + bar_info = &ntb->bar_info[mw_idx+1]; + + if (base != NULL) + *base = bar_info->pbase; + + if (vbase != NULL) + *vbase = bar_info->vbase; + + if (align != NULL) + *align = bar_info->size; + + if (size != NULL) + *size = bar_info->size; + + if (align_size != NULL) + *align_size = 1; + + if (plimit != NULL) { + if (mw_idx != 0) + *plimit = BUS_SPACE_MAXADDR; + else + *plimit = BUS_SPACE_MAXADDR_32BIT; + } + + amd_ntb_printf(1, "%s: mw %d padd %p vadd %p psize 0x%lx " + "align 0x%lx asize 0x%lx alimit %p\n", __func__, mw_idx, + (void *)*base, (void *)*vbase, (uint64_t)*size, (uint64_t)*align, + (uint64_t)*align_size, (void *)*plimit); + + return (0); +} + +static int +amd_ntb_mw_set_trans(device_t dev, unsigned mw_idx, bus_addr_t addr, size_t size) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + struct amd_ntb_pci_bar_info *bar_info; + + if (mw_idx < 0 || mw_idx >= ntb->mw_count) + return (EINVAL); + + bar_info = &ntb->bar_info[mw_idx+1]; + + /* make sure the range fits in the usable mw size */ + if (size > bar_info->size) { + amd_ntb_printf(0, "%s: size 0x%x greater than mw_size 0x%x\n", + __func__, (uint32_t)size, (uint32_t)bar_info->size); + return (EINVAL); + } + + amd_ntb_printf(1, "%s: mw %d mw_size 0x%x size 0x%x base %p\n", + __func__, mw_idx, (uint32_t)bar_info->size, + (uint32_t)size, (void *)bar_info->pci_bus_handle); + + /* + * AMD NTB XLAT and Limit registers needs to be written only after + * link enable + * + * set and verify setting the translation address + */ + amd_ntb_peer_reg_write(8, bar_info->xlat_off, (uint64_t)addr); + amd_ntb_printf(0, "%s: mw %d xlat_off 0x%x cur_val 0x%lx addr %p\n", + __func__, mw_idx, bar_info->xlat_off, + amd_ntb_peer_reg_read(8, bar_info->xlat_off), (void *)addr); + + /* set and verify setting the limit */ + if (mw_idx != 0) { + amd_ntb_reg_write(8, bar_info->limit_off, (uint64_t)size); + amd_ntb_printf(1, "%s: limit_off 0x%x cur_val 0x%lx limit 0x%x\n", + __func__, bar_info->limit_off, + amd_ntb_peer_reg_read(8, bar_info->limit_off), (uint32_t)size); + } else { + amd_ntb_reg_write(4, bar_info->limit_off, (uint64_t)size); + amd_ntb_printf(1, "%s: limit_off 0x%x cur_val 0x%x limit 0x%x\n", + __func__, bar_info->limit_off, + amd_ntb_peer_reg_read(4, bar_info->limit_off), (uint32_t)size); + } + + return (0); +} + +static int +amd_ntb_mw_clear_trans(device_t dev, unsigned mw_idx) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + + amd_ntb_printf(1, "%s: mw_idx %d\n", __func__, mw_idx); + + if (mw_idx < 0 || mw_idx >= ntb->mw_count) + return (EINVAL); + + return (amd_ntb_mw_set_trans(dev, mw_idx, 0, 0)); +} + +static int +amd_ntb_mw_set_wc(device_t dev, unsigned int mw_idx, vm_memattr_t mode) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + struct amd_ntb_pci_bar_info *bar_info; + int rc; + + if (mw_idx < 0 || mw_idx >= ntb->mw_count) + return (EINVAL); + + bar_info = &ntb->bar_info[mw_idx+1]; + if (mode == bar_info->map_mode) + return (0); + + rc = pmap_change_attr((vm_offset_t)bar_info->vbase, bar_info->size, mode); + if (rc == 0) + bar_info->map_mode = mode; + + return (rc); +} + +static int +amd_ntb_mw_get_wc(device_t dev, unsigned mw_idx, vm_memattr_t *mode) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + struct amd_ntb_pci_bar_info *bar_info; + + amd_ntb_printf(1, "%s: mw_idx %d\n", __func__, mw_idx); + + if (mw_idx < 0 || mw_idx >= ntb->mw_count) + return (EINVAL); + + bar_info = &ntb->bar_info[mw_idx+1]; + *mode = bar_info->map_mode; + + return (0); +} + +/* + * AMD NTB doorbell routines + */ +static int +amd_ntb_db_vector_count(device_t dev) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + + amd_ntb_printf(1, "%s: db_count 0x%x\n", __func__, ntb->db_count); + + return (ntb->db_count); +} + +static uint64_t +amd_ntb_db_valid_mask(device_t dev) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + + amd_ntb_printf(1, "%s: db_valid_mask 0x%x\n", + __func__, ntb->db_valid_mask); + + return (ntb->db_valid_mask); +} + +static uint64_t +amd_ntb_db_vector_mask(device_t dev, uint32_t vector) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + + amd_ntb_printf(1, "%s: vector %d db_count 0x%x db_valid_mask 0x%x\n", + __func__, vector, ntb->db_count, ntb->db_valid_mask); + + if (vector < 0 || vector >= ntb->db_count) + return (0); + + return (ntb->db_valid_mask & (1 << vector)); +} + +static uint64_t +amd_ntb_db_read(device_t dev) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + uint64_t dbstat_off; + + dbstat_off = (uint64_t)amd_ntb_reg_read(2, AMD_DBSTAT_OFFSET); + + amd_ntb_printf(1, "%s: dbstat_off 0x%lx\n", __func__, dbstat_off); + + return (dbstat_off); +} + +static void +amd_ntb_db_clear(device_t dev, uint64_t db_bits) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + + amd_ntb_printf(1, "%s: db_bits 0x%lx\n", __func__, db_bits); + amd_ntb_reg_write(2, AMD_DBSTAT_OFFSET, (uint16_t)db_bits); +} + +static void +amd_ntb_db_set_mask(device_t dev, uint64_t db_bits) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + + DB_MASK_LOCK(ntb); + amd_ntb_printf(1, "%s: db_mask 0x%x db_bits 0x%lx\n", + __func__, ntb->db_mask, db_bits); + + ntb->db_mask |= db_bits; + amd_ntb_reg_write(2, AMD_DBMASK_OFFSET, ntb->db_mask); + DB_MASK_UNLOCK(ntb); +} + +static void +amd_ntb_db_clear_mask(device_t dev, uint64_t db_bits) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + + DB_MASK_LOCK(ntb); + amd_ntb_printf(1, "%s: db_mask 0x%x db_bits 0x%lx\n", + __func__, ntb->db_mask, db_bits); + + ntb->db_mask &= ~db_bits; + amd_ntb_reg_write(2, AMD_DBMASK_OFFSET, ntb->db_mask); + DB_MASK_UNLOCK(ntb); +} + +static void +amd_ntb_peer_db_set(device_t dev, uint64_t db_bits) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + + amd_ntb_printf(1, "%s: db_bits 0x%lx\n", __func__, db_bits); + amd_ntb_reg_write(2, AMD_DBREQ_OFFSET, (uint16_t)db_bits); +} + +/* + * AMD NTB scratchpad routines + */ +static uint8_t +amd_ntb_spad_count(device_t dev) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + + amd_ntb_printf(1, "%s: spad_count 0x%x\n", __func__, ntb->spad_count); + + return (ntb->spad_count); +} + +static int +amd_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + uint32_t offset; + + amd_ntb_printf(2, "%s: idx %d\n", __func__, idx); + + if (idx < 0 || idx >= ntb->spad_count) + return (EINVAL); + + offset = ntb->self_spad + (idx << 2); + *val = amd_ntb_reg_read(4, AMD_SPAD_OFFSET + offset); + amd_ntb_printf(2, "%s: offset 0x%x val 0x%x\n", __func__, offset, *val); + + return (0); +} + +static int +amd_ntb_spad_write(device_t dev, unsigned int idx, uint32_t val) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + uint32_t offset; + + amd_ntb_printf(2, "%s: idx %d\n", __func__, idx); + + if (idx < 0 || idx >= ntb->spad_count) + return (EINVAL); + + offset = ntb->self_spad + (idx << 2); + amd_ntb_reg_write(4, AMD_SPAD_OFFSET + offset, val); + amd_ntb_printf(2, "%s: offset 0x%x val 0x%x\n", __func__, offset, val); + + return (0); +} + +static void +amd_ntb_spad_clear(struct amd_ntb_softc *ntb) +{ + uint8_t i; + + for (i = 0; i < ntb->spad_count; i++) + amd_ntb_spad_write(ntb->device, i, 0); +} + +static int +amd_ntb_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + uint32_t offset; + + amd_ntb_printf(2, "%s: idx %d\n", __func__, idx); + + if (idx < 0 || idx >= ntb->spad_count) + return (EINVAL); + + offset = ntb->peer_spad + (idx << 2); + *val = amd_ntb_reg_read(4, AMD_SPAD_OFFSET + offset); + amd_ntb_printf(2, "%s: offset 0x%x val 0x%x\n", __func__, offset, *val); + + return (0); +} + +static int +amd_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) +{ + struct amd_ntb_softc *ntb = device_get_softc(dev); + uint32_t offset; + + amd_ntb_printf(2, "%s: idx %d\n", __func__, idx); + + if (idx < 0 || idx >= ntb->spad_count) + return (EINVAL); + + offset = ntb->peer_spad + (idx << 2); + amd_ntb_reg_write(4, AMD_SPAD_OFFSET + offset, val); + amd_ntb_printf(2, "%s: offset 0x%x val 0x%x\n", __func__, offset, val); + + return (0); +} + + +/* + * AMD NTB INIT + */ +static int +amd_ntb_hw_info_handler(SYSCTL_HANDLER_ARGS) +{ + struct amd_ntb_softc* ntb = arg1; + struct sbuf *sb; + int rc = 0; + + sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); + if (sb == NULL) + return (sb->s_error); + + sbuf_printf(sb, "NTB AMD Hardware info:\n\n"); + sbuf_printf(sb, "AMD NTB side: %s\n", + (ntb->conn_type == NTB_CONN_PRI)? "PRIMARY" : "SECONDARY"); + sbuf_printf(sb, "AMD LNK STA: 0x%#06x\n", ntb->lnk_sta); + + if (!amd_link_is_up(ntb)) + sbuf_printf(sb, "AMD Link Status: Down\n"); + else { + sbuf_printf(sb, "AMD Link Status: Up\n"); + sbuf_printf(sb, "AMD Link Speed: PCI-E Gen %u\n", + NTB_LNK_STA_SPEED(ntb->lnk_sta)); + sbuf_printf(sb, "AMD Link Width: PCI-E Width %u\n", + NTB_LNK_STA_WIDTH(ntb->lnk_sta)); + } + + sbuf_printf(sb, "AMD Memory window count: %d\n", + ntb->mw_count); + sbuf_printf(sb, "AMD Spad count: %d\n", + ntb->spad_count); + sbuf_printf(sb, "AMD Doorbell count: %d\n", + ntb->db_count); + sbuf_printf(sb, "AMD MSI-X vec count: %d\n\n", + ntb->msix_vec_count); + sbuf_printf(sb, "AMD Doorbell valid mask: 0x%x\n", + ntb->db_valid_mask); + sbuf_printf(sb, "AMD Doorbell Mask: 0x%x\n", + amd_ntb_reg_read(4, AMD_DBMASK_OFFSET)); + sbuf_printf(sb, "AMD Doorbell: 0x%x\n", + amd_ntb_reg_read(4, AMD_DBSTAT_OFFSET)); + sbuf_printf(sb, "AMD NTB Incoming XLAT: \n"); + sbuf_printf(sb, "AMD XLAT1: 0x%lx\n", + amd_ntb_peer_reg_read(8, AMD_BAR1XLAT_OFFSET)); + sbuf_printf(sb, "AMD XLAT23: 0x%lx\n", + amd_ntb_peer_reg_read(8, AMD_BAR23XLAT_OFFSET)); + sbuf_printf(sb, "AMD XLAT45: 0x%lx\n", + amd_ntb_peer_reg_read(8, AMD_BAR45XLAT_OFFSET)); + sbuf_printf(sb, "AMD LMT1: 0x%x\n", + amd_ntb_reg_read(4, AMD_BAR1LMT_OFFSET)); + sbuf_printf(sb, "AMD LMT23: 0x%lx\n", + amd_ntb_reg_read(8, AMD_BAR23LMT_OFFSET)); + sbuf_printf(sb, "AMD LMT45: 0x%lx\n", + amd_ntb_reg_read(8, AMD_BAR45LMT_OFFSET)); + + rc = sbuf_finish(sb); + sbuf_delete(sb); + return (rc); +} + +static void +amd_ntb_sysctl_init(struct amd_ntb_softc *ntb) +{ + struct sysctl_oid_list *globals; + struct sysctl_ctx_list *ctx; + + ctx = device_get_sysctl_ctx(ntb->device); + globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)); + + SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "info", + CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, ntb, 0, + amd_ntb_hw_info_handler, "A", "AMD NTB HW Information"); +} + +/* + * Polls the HW link status register(s); returns true if something has changed. + */ +static bool +amd_ntb_poll_link(struct amd_ntb_softc *ntb) +{ + uint32_t fullreg, reg, stat; + + fullreg = amd_ntb_peer_reg_read(4, AMD_SIDEINFO_OFFSET); + reg = fullreg & NTB_LIN_STA_ACTIVE_BIT; + + if (reg == ntb->cntl_sta) + return (false); + + amd_ntb_printf(0, "%s: SIDEINFO reg_val = 0x%x cntl_sta 0x%x\n", + __func__, fullreg, ntb->cntl_sta); + + ntb->cntl_sta = reg; + + stat = pci_read_config(ntb->device, AMD_LINK_STATUS_OFFSET, 4); + + amd_ntb_printf(0, "%s: LINK_STATUS stat = 0x%x lnk_sta 0x%x.\n", + __func__, stat, ntb->lnk_sta); + + ntb->lnk_sta = stat; + + return (true); +} + +static void +amd_link_hb(void *arg) +{ + struct amd_ntb_softc *ntb = arg; + + if (amd_ntb_poll_link(ntb)) + ntb_link_event(ntb->device); + + if (!amd_link_is_up(ntb)) { + callout_reset(&ntb->hb_timer, AMD_LINK_HB_TIMEOUT, + amd_link_hb, ntb); + } else { + callout_reset(&ntb->hb_timer, (AMD_LINK_HB_TIMEOUT * 10), + amd_link_hb, ntb); + } +} + +static void +amd_ntb_interrupt(struct amd_ntb_softc *ntb, uint16_t vec) +{ + if (vec < AMD_DB_CNT) + ntb_db_event(ntb->device, vec); + else + amd_ntb_printf(0, "Invalid vector %d\n", vec); +} + +static void +amd_ntb_vec_isr(void *arg) +{ + struct amd_ntb_vec *nvec = arg; + + amd_ntb_interrupt(nvec->ntb, nvec->num); +} + +static void +amd_ntb_irq_isr(void *arg) +{ + /* If we couldn't set up MSI-X, we only have the one vector. */ + amd_ntb_interrupt(arg, 0); +} + +static void +amd_init_side_info(struct amd_ntb_softc *ntb) +{ + unsigned int reg; + + reg = amd_ntb_reg_read(4, AMD_SIDEINFO_OFFSET); + if (!(reg & AMD_SIDE_READY)) { + reg |= AMD_SIDE_READY; + amd_ntb_reg_write(4, AMD_SIDEINFO_OFFSET, reg); + } + reg = amd_ntb_reg_read(4, AMD_SIDEINFO_OFFSET); +} + +static void +amd_deinit_side_info(struct amd_ntb_softc *ntb) +{ + unsigned int reg; + + reg = amd_ntb_reg_read(4, AMD_SIDEINFO_OFFSET); + if (reg & AMD_SIDE_READY) { + reg &= ~AMD_SIDE_READY; + amd_ntb_reg_write(4, AMD_SIDEINFO_OFFSET, reg); + amd_ntb_reg_read(4, AMD_SIDEINFO_OFFSET); + } +} + +static int +amd_ntb_setup_isr(struct amd_ntb_softc *ntb, uint16_t num_vectors, bool msi, + bool intx) +{ + uint16_t i; + int flags = 0, rc = 0; + + flags |= RF_ACTIVE; + if (intx) + flags |= RF_SHAREABLE; + + for (i = 0; i < num_vectors; i++) { + + /* RID should be 0 for intx */ + if (intx) + ntb->int_info[i].rid = i; + else + ntb->int_info[i].rid = i + 1; + + ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, + SYS_RES_IRQ, &ntb->int_info[i].rid, flags); + if (ntb->int_info[i].res == NULL) { + amd_ntb_printf(0, "bus_alloc_resource IRQ failed\n"); + return (ENOMEM); + } + + ntb->int_info[i].tag = NULL; + ntb->allocated_interrupts++; + + if (msi || intx) { + rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, + INTR_MPSAFE | INTR_TYPE_MISC, NULL, amd_ntb_irq_isr, + ntb, &ntb->int_info[i].tag); + } else { + rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, + INTR_MPSAFE | INTR_TYPE_MISC, NULL, amd_ntb_vec_isr, + &ntb->msix_vec[i], &ntb->int_info[i].tag); + } + + if (rc != 0) { + amd_ntb_printf(0, "bus_setup_intr %d failed\n", i); + return (ENXIO); + } + } + + return (0); +} + +static int +amd_ntb_create_msix_vec(struct amd_ntb_softc *ntb, uint32_t max_vectors) +{ + uint8_t i; + + ntb->msix_vec = malloc(max_vectors * sizeof(*ntb->msix_vec), M_AMD_NTB, + M_ZERO | M_WAITOK); + + for (i = 0; i < max_vectors; i++) { + ntb->msix_vec[i].num = i; + ntb->msix_vec[i].ntb = ntb; + } + + return (0); +} + +static void +amd_ntb_free_msix_vec(struct amd_ntb_softc *ntb) +{ + if (ntb->msix_vec_count) { + pci_release_msi(ntb->device); + ntb->msix_vec_count = 0; + } + + if (ntb->msix_vec != NULL) { + free(ntb->msix_vec, M_AMD_NTB); + ntb->msix_vec = NULL; + } +} + +static int +amd_ntb_init_isr(struct amd_ntb_softc *ntb) +{ + uint32_t supported_vectors, num_vectors; + bool msi = false, intx = false; + int rc = 0; + + ntb->db_mask = ntb->db_valid_mask; + + rc = amd_ntb_create_msix_vec(ntb, AMD_MSIX_VECTOR_CNT); + if (rc != 0) { + amd_ntb_printf(0, "Error creating msix vectors: %d\n", rc); + return (ENOMEM); + } + + /* + * Check the number of MSI-X message supported by the device. + * Minimum necessary MSI-X message count should be equal to db_count + */ + supported_vectors = pci_msix_count(ntb->device); + num_vectors = MIN(supported_vectors, ntb->db_count); + if (num_vectors < ntb->db_count) { + amd_ntb_printf(0, "No minimum msix: supported %d db %d\n", + supported_vectors, ntb->db_count); + msi = true; + goto err_msix_enable; + } + + /* Allocate the necessary number of MSI-x messages */ + rc = pci_alloc_msix(ntb->device, &num_vectors); + if (rc != 0) { + amd_ntb_printf(0, "Error allocating msix vectors: %d\n", rc); + msi = true; + goto err_msix_enable; + } + + if (num_vectors < ntb->db_count) { + amd_ntb_printf(0, "Allocated only %d MSI-X\n", num_vectors); + msi = true; + /* + * Else set ntb->db_count = ntb->msix_vec_count = num_vectors, + * msi=false and dont release msi + */ + } + +err_msix_enable: + + if (msi) { + free(ntb->msix_vec, M_AMD_NTB); + ntb->msix_vec = NULL; + pci_release_msi(ntb->device); + num_vectors = 1; + rc = pci_alloc_msi(ntb->device, &num_vectors); + if (rc != 0) { + amd_ntb_printf(0, "Error allocating msix vectors: %d\n", rc); + msi = false; + intx = true; + } + } + + ntb->db_count = ntb->msix_vec_count = num_vectors; + + if (intx) { + num_vectors = 1; + ntb->db_count = 1; + ntb->msix_vec_count = 0; + } + + amd_ntb_printf(0, "%s: db %d msix %d msi %d intx %d\n", + __func__, ntb->db_count, ntb->msix_vec_count, (int)msi, (int)intx); + + rc = amd_ntb_setup_isr(ntb, num_vectors, msi, intx); + if (rc != 0) { + amd_ntb_printf(0, "Error setting up isr: %d\n", rc); + amd_ntb_free_msix_vec(ntb); + } + + return (rc); +} + +static void +amd_ntb_deinit_isr(struct amd_ntb_softc *ntb) +{ + struct amd_ntb_int_info *current_int; + int i; + + /* Mask all doorbell interrupts */ + ntb->db_mask = ntb->db_valid_mask; + amd_ntb_reg_write(4, AMD_DBMASK_OFFSET, ntb->db_mask); + + for (i = 0; i < ntb->allocated_interrupts; i++) { + current_int = &ntb->int_info[i]; + if (current_int->tag != NULL) + bus_teardown_intr(ntb->device, current_int->res, + current_int->tag); + + if (current_int->res != NULL) + bus_release_resource(ntb->device, SYS_RES_IRQ, + rman_get_rid(current_int->res), current_int->res); + } + + amd_ntb_free_msix_vec(ntb); +} + +static enum amd_ntb_conn_type +amd_ntb_get_topo(struct amd_ntb_softc *ntb) +{ + uint32_t info; + + info = amd_ntb_reg_read(4, AMD_SIDEINFO_OFFSET); + + if (info & AMD_SIDE_MASK) + return (NTB_CONN_SEC); + + return (NTB_CONN_PRI); +} + +static int +amd_ntb_init_dev(struct amd_ntb_softc *ntb) +{ + ntb->mw_count = AMD_MW_CNT; + ntb->spad_count = AMD_SPADS_CNT; + ntb->db_count = AMD_DB_CNT; + ntb->db_valid_mask = (1ull << ntb->db_count) - 1; + mtx_init(&ntb->db_mask_lock, "amd ntb db bits", NULL, MTX_SPIN); + + switch (ntb->conn_type) { + case NTB_CONN_PRI: + case NTB_CONN_SEC: + ntb->spad_count >>= 1; + + if (ntb->conn_type == NTB_CONN_PRI) { + ntb->self_spad = 0; + ntb->peer_spad = 0x20; + } else { + ntb->self_spad = 0x20; + ntb->peer_spad = 0; + } + + callout_init(&ntb->hb_timer, 1); + callout_reset(&ntb->hb_timer, AMD_LINK_HB_TIMEOUT, + amd_link_hb, ntb); + + break; + + default: + amd_ntb_printf(0, "Unsupported AMD NTB topology %d\n", + ntb->conn_type); + return (EINVAL); + } + + ntb->int_mask = AMD_EVENT_INTMASK; + amd_ntb_reg_write(4, AMD_INTMASK_OFFSET, ntb->int_mask); + + return (0); +} + +static int +amd_ntb_init(struct amd_ntb_softc *ntb) +{ + int rc = 0; + + ntb->conn_type = amd_ntb_get_topo(ntb); + amd_ntb_printf(0, "AMD NTB Side: %s\n", + (ntb->conn_type == NTB_CONN_PRI)? "PRIMARY" : "SECONDARY"); + + rc = amd_ntb_init_dev(ntb); + if (rc != 0) + return (rc); + + rc = amd_ntb_init_isr(ntb); + if (rc != 0) + return (rc); + + return (0); +} + +static void +print_map_success(struct amd_ntb_softc *ntb, struct amd_ntb_pci_bar_info *bar, + const char *kind) +{ + amd_ntb_printf(0, "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", + PCI_RID2BAR(bar->pci_resource_id), bar->vbase, + (char *)bar->vbase + bar->size - 1, (void *)bar->pbase, + (void *)(bar->pbase + bar->size - 1), (uintmax_t)bar->size, kind); +} + +static void +save_bar_parameters(struct amd_ntb_pci_bar_info *bar) +{ + bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); + bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); + bar->pbase = rman_get_start(bar->pci_resource); + bar->size = rman_get_size(bar->pci_resource); + bar->vbase = rman_get_virtual(bar->pci_resource); + bar->map_mode = VM_MEMATTR_UNCACHEABLE; +} + +static int +map_bar(struct amd_ntb_softc *ntb, struct amd_ntb_pci_bar_info *bar) +{ + bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, + &bar->pci_resource_id, RF_ACTIVE); + if (bar->pci_resource == NULL) + return (ENXIO); + + save_bar_parameters(bar); + print_map_success(ntb, bar, "mmr"); + + return (0); +} + +static int +amd_ntb_map_pci_bars(struct amd_ntb_softc *ntb) +{ + int rc = 0; + + /* NTB Config/Control registers - BAR 0 */ + ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); + rc = map_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); + if (rc != 0) + goto out; + + /* Memory Window 0 BAR - BAR 1*/ + ntb->bar_info[NTB_BAR_1].pci_resource_id = PCIR_BAR(1); + rc = map_bar(ntb, &ntb->bar_info[NTB_BAR_1]); + if (rc != 0) + goto out; + ntb->bar_info[NTB_BAR_1].xlat_off = AMD_BAR1XLAT_OFFSET; + ntb->bar_info[NTB_BAR_1].limit_off = AMD_BAR1LMT_OFFSET; + + /* Memory Window 1 BAR - BAR 2&3 */ + ntb->bar_info[NTB_BAR_2].pci_resource_id = PCIR_BAR(2); + rc = map_bar(ntb, &ntb->bar_info[NTB_BAR_2]); + if (rc != 0) + goto out; + ntb->bar_info[NTB_BAR_2].xlat_off = AMD_BAR23XLAT_OFFSET; + ntb->bar_info[NTB_BAR_2].limit_off = AMD_BAR23LMT_OFFSET; + + /* Memory Window 2 BAR - BAR 4&5 */ + ntb->bar_info[NTB_BAR_3].pci_resource_id = PCIR_BAR(4); + rc = map_bar(ntb, &ntb->bar_info[NTB_BAR_3]); + if (rc != 0) + goto out; + ntb->bar_info[NTB_BAR_3].xlat_off = AMD_BAR45XLAT_OFFSET; + ntb->bar_info[NTB_BAR_3].limit_off = AMD_BAR45LMT_OFFSET; + +out: + if (rc != 0) + amd_ntb_printf(0, "unable to allocate pci resource\n"); + + return (rc); +} + +static void +amd_ntb_unmap_pci_bars(struct amd_ntb_softc *ntb) +{ + struct amd_ntb_pci_bar_info *bar_info; + int i; + + for (i = 0; i < NTB_MAX_BARS; i++) { + bar_info = &ntb->bar_info[i]; + if (bar_info->pci_resource != NULL) + bus_release_resource(ntb->device, SYS_RES_MEMORY, + bar_info->pci_resource_id, bar_info->pci_resource); + } +} + +static int +amd_ntb_probe(device_t device) +{ + const struct pci_device_table *tbl; + + tbl = PCI_MATCH(device, amd_ntb_devs); + if (tbl == NULL) + return (ENXIO); + + device_set_desc(device, tbl->descr); + + return (BUS_PROBE_GENERIC); +} + +static int +amd_ntb_attach(device_t device) +{ + struct amd_ntb_softc *ntb = device_get_softc(device); + int error; + + ntb->device = device; + + /* Enable PCI bus mastering for "device" */ + pci_enable_busmaster(ntb->device); + + error = amd_ntb_map_pci_bars(ntb); + if (error) + goto out; + + error = amd_ntb_init(ntb); + if (error) + goto out; + + amd_init_side_info(ntb); + + amd_ntb_spad_clear(ntb); + + amd_ntb_sysctl_init(ntb); + + /* Attach children to this controller */ + error = ntb_register_device(device); + +out: + if (error) + amd_ntb_detach(device); + + return (error); +} + +static int +amd_ntb_detach(device_t device) +{ + struct amd_ntb_softc *ntb = device_get_softc(device); + + ntb_unregister_device(device); + amd_deinit_side_info(ntb); + callout_drain(&ntb->hb_timer); + amd_ntb_deinit_isr(ntb); + mtx_destroy(&ntb->db_mask_lock); + pci_disable_busmaster(ntb->device); + amd_ntb_unmap_pci_bars(ntb); + + return (0); +} + +static device_method_t ntb_amd_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, amd_ntb_probe), + DEVMETHOD(device_attach, amd_ntb_attach), + DEVMETHOD(device_detach, amd_ntb_detach), + + /* Bus interface */ + DEVMETHOD(bus_child_location_str, ntb_child_location_str), + DEVMETHOD(bus_print_child, ntb_print_child), + + /* NTB interface */ + DEVMETHOD(ntb_port_number, amd_ntb_port_number), + DEVMETHOD(ntb_peer_port_count, amd_ntb_peer_port_count), + DEVMETHOD(ntb_peer_port_number, amd_ntb_peer_port_number), + DEVMETHOD(ntb_peer_port_idx, amd_ntb_peer_port_idx), + DEVMETHOD(ntb_link_is_up, amd_ntb_link_is_up), + DEVMETHOD(ntb_link_enable, amd_ntb_link_enable), + DEVMETHOD(ntb_link_disable, amd_ntb_link_disable), + DEVMETHOD(ntb_mw_count, amd_ntb_mw_count), + DEVMETHOD(ntb_mw_get_range, amd_ntb_mw_get_range), + DEVMETHOD(ntb_mw_set_trans, amd_ntb_mw_set_trans), + DEVMETHOD(ntb_mw_clear_trans, amd_ntb_mw_clear_trans), + DEVMETHOD(ntb_mw_set_wc, amd_ntb_mw_set_wc), + DEVMETHOD(ntb_mw_get_wc, amd_ntb_mw_get_wc), + DEVMETHOD(ntb_db_valid_mask, amd_ntb_db_valid_mask), + DEVMETHOD(ntb_db_vector_count, amd_ntb_db_vector_count), + DEVMETHOD(ntb_db_vector_mask, amd_ntb_db_vector_mask), + DEVMETHOD(ntb_db_read, amd_ntb_db_read), + DEVMETHOD(ntb_db_clear, amd_ntb_db_clear), + DEVMETHOD(ntb_db_set_mask, amd_ntb_db_set_mask), + DEVMETHOD(ntb_db_clear_mask, amd_ntb_db_clear_mask), + DEVMETHOD(ntb_peer_db_set, amd_ntb_peer_db_set), + DEVMETHOD(ntb_spad_count, amd_ntb_spad_count), + DEVMETHOD(ntb_spad_read, amd_ntb_spad_read), + DEVMETHOD(ntb_spad_write, amd_ntb_spad_write), + DEVMETHOD(ntb_peer_spad_read, amd_ntb_peer_spad_read), + DEVMETHOD(ntb_peer_spad_write, amd_ntb_peer_spad_write), + DEVMETHOD_END +}; + +static DEFINE_CLASS_0(ntb_hw, ntb_amd_driver, ntb_amd_methods, + sizeof(struct amd_ntb_softc)); +DRIVER_MODULE(ntb_hw_amd, pci, ntb_amd_driver, ntb_hw_devclass, NULL, NULL); +MODULE_DEPEND(ntb_hw_amd, ntb, 1, 1, 1); +MODULE_VERSION(ntb_hw_amd, 1); +PCI_PNP_INFO(amd_ntb_devs); Index: head/sys/dev/ntb/ntb_if.m =================================================================== --- head/sys/dev/ntb/ntb_if.m +++ head/sys/dev/ntb/ntb_if.m @@ -61,6 +61,24 @@ }; }; +METHOD int port_number { + device_t ntb; +}; + +METHOD int peer_port_count { + device_t ntb; +}; + +METHOD int peer_port_number { + device_t ntb; + int pidx; +}; + +METHOD int peer_port_idx { + device_t ntb; + int port; +}; + METHOD bool link_is_up { device_t ntb; enum ntb_speed *speed; Index: head/sys/dev/ntb/ntb_transport.c =================================================================== --- head/sys/dev/ntb/ntb_transport.c +++ head/sys/dev/ntb/ntb_transport.c @@ -179,7 +179,6 @@ bus_addr_t addr_limit; /* Tx buff is off vbase / phys_addr */ caddr_t vbase; - size_t xlat_size; size_t buff_size; /* Rx buff is off virt_addr / dma_addr */ bus_dma_tag_t dma_tag; @@ -376,7 +375,6 @@ goto err; mw->buff_size = 0; - mw->xlat_size = 0; mw->virt_addr = NULL; mw->dma_addr = 0; @@ -462,6 +460,13 @@ ntb_link_enable(dev, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); + for (i = 0; i < nt->mw_count; i++) { + mw = &nt->mw_vec[i]; + rc = ntb_mw_set_trans(nt->dev, i, mw->dma_addr, mw->buff_size); + if (rc != 0) + ntb_printf(0, "load time mw%d xlat fails, rc %d\n", i, rc); + } + if (enable_xeon_watchdog != 0) callout_reset(&nt->link_watchdog, 0, xeon_link_watchdog_hb, nt); @@ -1088,6 +1093,7 @@ ntb_transport_link_work(void *arg) { struct ntb_transport_ctx *nt = arg; + struct ntb_transport_mw *mw; device_t dev = nt->dev; struct ntb_transport_qp *qp; uint64_t val64, size; @@ -1132,9 +1138,26 @@ ntb_spad_read(dev, NTBT_MW0_SZ_LOW + (i * 2), &val); val64 |= val; - rc = ntb_set_mw(nt, i, val64); - if (rc != 0) - goto free_mws; + mw = &nt->mw_vec[i]; + val64 = roundup(val64, mw->xlat_align_size); + if (mw->buff_size != val64) { + + rc = ntb_set_mw(nt, i, val64); + if (rc != 0) { + ntb_printf(0, "link up set mw%d fails, rc %d\n", + i, rc); + goto free_mws; + } + + /* Notify HW the memory location of the receive buffer */ + rc = ntb_mw_set_trans(nt->dev, i, mw->dma_addr, + mw->buff_size); + if (rc != 0) { + ntb_printf(0, "link up mw%d xlat fails, rc %d\n", + i, rc); + goto free_mws; + } + } } nt->link_is_up = true; @@ -1179,42 +1202,37 @@ { struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; struct ntb_load_cb_args cba; - size_t xlat_size, buff_size; - int rc; + size_t buff_size; if (size == 0) return (EINVAL); - xlat_size = roundup(size, mw->xlat_align_size); - buff_size = xlat_size; + buff_size = roundup(size, mw->xlat_align_size); /* No need to re-setup */ - if (mw->xlat_size == xlat_size) + if (mw->buff_size == buff_size) return (0); if (mw->buff_size != 0) ntb_free_mw(nt, num_mw); /* Alloc memory for receiving data. Must be aligned */ - mw->xlat_size = xlat_size; mw->buff_size = buff_size; if (bus_dma_tag_create(bus_get_dma_tag(nt->dev), mw->xlat_align, 0, mw->addr_limit, BUS_SPACE_MAXADDR, NULL, NULL, mw->buff_size, 1, mw->buff_size, 0, NULL, NULL, &mw->dma_tag)) { - ntb_printf(0, "Unable to create MW tag of size %zu/%zu\n", - mw->buff_size, mw->xlat_size); - mw->xlat_size = 0; + ntb_printf(0, "Unable to create MW tag of size %zu\n", + mw->buff_size); mw->buff_size = 0; return (ENOMEM); } if (bus_dmamem_alloc(mw->dma_tag, (void **)&mw->virt_addr, BUS_DMA_WAITOK | BUS_DMA_ZERO, &mw->dma_map)) { bus_dma_tag_destroy(mw->dma_tag); - ntb_printf(0, "Unable to allocate MW buffer of size %zu/%zu\n", - mw->buff_size, mw->xlat_size); - mw->xlat_size = 0; + ntb_printf(0, "Unable to allocate MW buffer of size %zu\n", + mw->buff_size); mw->buff_size = 0; return (ENOMEM); } @@ -1222,22 +1240,13 @@ mw->buff_size, ntb_load_cb, &cba, BUS_DMA_NOWAIT) || cba.error) { bus_dmamem_free(mw->dma_tag, mw->virt_addr, mw->dma_map); bus_dma_tag_destroy(mw->dma_tag); - ntb_printf(0, "Unable to load MW buffer of size %zu/%zu\n", - mw->buff_size, mw->xlat_size); - mw->xlat_size = 0; + ntb_printf(0, "Unable to load MW buffer of size %zu\n", + mw->buff_size); mw->buff_size = 0; return (ENOMEM); } mw->dma_addr = cba.addr; - /* Notify HW the memory location of the receive buffer */ - rc = ntb_mw_set_trans(nt->dev, num_mw, mw->dma_addr, mw->xlat_size); - if (rc) { - ntb_printf(0, "Unable to set mw%d translation\n", num_mw); - ntb_free_mw(nt, num_mw); - return (rc); - } - return (0); } @@ -1253,7 +1262,6 @@ bus_dmamap_unload(mw->dma_tag, mw->dma_map); bus_dmamem_free(mw->dma_tag, mw->virt_addr, mw->dma_map); bus_dma_tag_destroy(mw->dma_tag); - mw->xlat_size = 0; mw->buff_size = 0; mw->virt_addr = NULL; } @@ -1280,7 +1288,7 @@ else num_qps_mw = nt->qp_count / mw_count; - rx_size = mw->xlat_size / num_qps_mw; + rx_size = mw->buff_size / num_qps_mw; qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count); rx_size -= sizeof(struct ntb_rx_info); Index: head/sys/i386/conf/NOTES =================================================================== --- head/sys/i386/conf/NOTES +++ head/sys/i386/conf/NOTES @@ -618,6 +618,7 @@ device if_ntb # Virtual NTB network interface device ntb_transport # NTB packet transport driver device ntb # NTB hardware interface +device ntb_hw_amd # AMD NTB hardware driver device ntb_hw_intel # Intel NTB hardware driver device ntb_hw_plx # PLX NTB hardware driver Index: head/sys/modules/ntb/Makefile =================================================================== --- head/sys/modules/ntb/Makefile +++ head/sys/modules/ntb/Makefile @@ -1,5 +1,5 @@ # $FreeBSD$ -SUBDIR= ntb ntb_hw_intel ntb_hw_plx ntb_transport if_ntb +SUBDIR= ntb ntb_hw_amd ntb_hw_intel ntb_hw_plx ntb_transport if_ntb .include Index: head/sys/modules/ntb/ntb_hw_amd/Makefile =================================================================== --- head/sys/modules/ntb/ntb_hw_amd/Makefile +++ head/sys/modules/ntb/ntb_hw_amd/Makefile @@ -0,0 +1,9 @@ +# $FreeBSD$ + +.PATH: ${SRCTOP}/sys/dev/ntb/ntb_hw + +KMOD = ntb_hw_amd +SRCS = ntb_hw_amd.c +SRCS += device_if.h bus_if.h pci_if.h ntb_if.h + +.include