Index: head/share/man/man4/Makefile =================================================================== --- head/share/man/man4/Makefile +++ head/share/man/man4/Makefile @@ -218,6 +218,7 @@ iicbus.4 \ iicsmb.4 \ iir.4 \ + ${_imcsmb.4} \ inet.4 \ inet6.4 \ intpm.4 \ @@ -824,6 +825,7 @@ _if_vtnet.4= if_vtnet.4 _if_vxge.4= if_vxge.4 _if_wpi.4= if_wpi.4 +_imcsmb.4= imcsmb.4 _ipmi.4= ipmi.4 _io.4= io.4 _linux.4= linux.4 Index: head/share/man/man4/imcsmb.4 =================================================================== --- head/share/man/man4/imcsmb.4 +++ head/share/man/man4/imcsmb.4 @@ -0,0 +1,133 @@ +.\" +.\" SPDX-License-Identifier: BSD-2-Clause-FreeBSD +.\" +.\" Copyright (c) 2018 Panasas +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +.\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +.\" IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +.\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +.\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +.\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +.\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +.\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +.\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd March 2, 2018 +.Dt IMCSMB 4 +.Os +.Sh NAME +.Nm imcsmb +.Nd Intel integrated Memory Controller (iMC) SMBus controller driver +.Sh SYNOPSIS +.Cd device pci +.Cd device smbus +.Cd device imcsmb +.Pp +Alternatively, to load the driver as a module at boot time, place the following +line in +.Xr loader.conf 5 : +.Bd -literal -offset indent +imcsmb_load="YES" +.Ed +.Sh DESCRIPTION +The +.Nm +driver provides +.Xr smbus 4 +support for the SMBus controller functionality in the integrated Memory +Controllers (iMCs) embedded in Intel Sandybridge-Xeon, Ivybridge-Xeon, +Haswell-Xeon, and Broadwell-Xeon CPUs. +Each CPU implements one or more iMCs, depending on the number of cores; +each iMC implements two SMBus controllers (iMC-SMBs). +The iMC-SMBs are used by the iMCs to read configuration information from the +DIMMs during POST. +They may also be used, by motherboard firmware or a BMC, to monitor the +temperature of the DIMMs. +.Pp +The iMC-SMBs are +.Sy not +general-purpose SMBus controllers. +By their nature, they are only ever attached to DIMMs, so they implement only +the SMBus operations need for communicating with DIMMs. +Specifically: +.Pp +.Bl -dash -offset indent -compact +.It +READB +.It +READW +.It +WRITEB +.It +WRITEW +.El +.Pp +A more detailed discussion of the hardware and driver architecture can be found +at the top of +.Pa sys/dev/imcsmb/imcsmb_pci.c . +.Sh WARNINGS +As mentioned above, firmware might use the iMC-SMBs to read DIMM temperatures. +The public iMC documentation does not describe any sort of coordination +mechanism to prevent requests from different sources -- such as the motherboard +firmware, a BMC, or the operating system -- from interfering with each other. +.Pp +.Bf Sy +Therefore, it is highly recommended that developers contact the motherboard +vendor for any board-specific instructions on how to disable and re-enable DIMM +temperature monitoring. +.Ef +.Pp +DIMM temperature monitoring should be disabled before returning from +.Fn imcsmb_pci_request_bus , +and re-enabled before returning from +.Fn imcsmb_pci_release_bus . +The driver includes comments to that effect at the appropriate locations. +The driver has been tested and shown to work, with only that type of +modification, on certain motherboards from Intel. +.Po +Unfortunately, those modifications were based on material covered under a +non-disclosure agreement, and therefore are not included in this driver. +.Pc +The driver has also been tested and shown to work as-is on various motherboards +from SuperMicro. +.Pp +The +.Xr smb 4 +driver will connect to the +.Xr smbus 4 +instances created by +.Nm . +However, since the IMC-SMBs are not general-purpose SMBus controllers, using +.Xr smbmsg 8 +with those +.Xr smb 4 +devices is not supported. +.Sh SEE ALSO +.Xr jedec_dimm 4 , +.Xr smbus 4 +.Sh HISTORY +The +.Nm +driver first appeared in +.Fx 12.0 . +.Sh AUTHORS +The +.Nm +driver was originally written for Panasas by +.An Joe Kloss . +It was substantially refactored, and this manual page was written, by +.An Ravi Pokala Aq Mt rpokala@freebsd.org Index: head/sys/amd64/conf/NOTES =================================================================== --- head/sys/amd64/conf/NOTES +++ head/sys/amd64/conf/NOTES @@ -468,6 +468,11 @@ device ips # +# Intel integrated Memory Controller (iMC) SMBus controller +# Sandybridge-Xeon, Ivybridge-Xeon, Haswell-Xeon, Broadwell-Xeon +device imcsmb + +# # Intel C600 (Patsburg) integrated SAS controller device isci options ISCI_LOGGING # enable debugging in isci HAL Index: head/sys/conf/files.amd64 =================================================================== --- head/sys/conf/files.amd64 +++ head/sys/conf/files.amd64 @@ -244,6 +244,8 @@ dev/if_ndis/if_ndis_pccard.c optional ndis pccard dev/if_ndis/if_ndis_pci.c optional ndis cardbus | ndis pci dev/if_ndis/if_ndis_usb.c optional ndis usb +dev/imcsmb/imcsmb.c optional imcsmb +dev/imcsmb/imcsmb_pci.c optional imcsmb pci dev/intel/spi.c optional intelspi dev/io/iodev.c optional io dev/ioat/ioat.c optional ioat pci Index: head/sys/conf/files.i386 =================================================================== --- head/sys/conf/files.i386 +++ head/sys/conf/files.i386 @@ -266,6 +266,8 @@ dev/if_ndis/if_ndis_pccard.c optional ndis pccard dev/if_ndis/if_ndis_pci.c optional ndis cardbus | ndis pci dev/if_ndis/if_ndis_usb.c optional ndis usb +dev/imcsmb/imcsmb.c optional imcsmb +dev/imcsmb/imcsmb_pci.c optional imcsmb pci dev/intel/spi.c optional intelspi dev/io/iodev.c optional io dev/ipmi/ipmi.c optional ipmi Index: head/sys/dev/imcsmb/imcsmb.c =================================================================== --- head/sys/dev/imcsmb/imcsmb.c +++ head/sys/dev/imcsmb/imcsmb.c @@ -0,0 +1,557 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Authors: Joe Kloss; Ravi Pokala (rpokala@freebsd.org) + * + * Copyright (c) 2017-2018 Panasas + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +/* A detailed description of this device is present in imcsmb_pci.c */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include + +#include "imcsmb_reg.h" +#include "imcsmb_var.h" + +/* Device methods */ +static int imcsmb_attach(device_t dev); +static int imcsmb_detach(device_t dev); +static int imcsmb_probe(device_t dev); + +/* SMBus methods */ +static int imcsmb_callback(device_t dev, int index, void *data); +static int imcsmb_readb(device_t dev, u_char slave, char cmd, char *byte); +static int imcsmb_readw(device_t dev, u_char slave, char cmd, short *word); +static int imcsmb_writeb(device_t dev, u_char slave, char cmd, char byte); +static int imcsmb_writew(device_t dev, u_char slave, char cmd, short word); + +/* All the read/write methods wrap around this. */ +static int imcsmb_transfer(device_t dev, u_char slave, char cmd, void *data, + int word_op, int write_op); + +/** + * device_attach() method. Set up the softc, including getting the set of the + * parent imcsmb_pci's registers that we will use. Create the smbus(4) device, + * which any SMBus slave device drivers will connect to. + * + * @author rpokala + * + * @param[in,out] dev + * Device being attached. + */ +static int +imcsmb_attach(device_t dev) +{ + struct imcsmb_softc *sc; + int rc; + + /* Initialize private state */ + sc = device_get_softc(dev); + sc->dev = dev; + sc->imcsmb_pci = device_get_parent(dev); + sc->regs = device_get_ivars(dev); + + /* Create the smbus child */ + sc->smbus = device_add_child(dev, "smbus", -1); + if (sc->smbus == NULL) { + /* Nothing has been allocated, so there's no cleanup. */ + device_printf(dev, "Child smbus not added\n"); + rc = ENXIO; + goto out; + } + + /* Attach the smbus child. */ + if ((rc = bus_generic_attach(dev)) != 0) { + device_printf(dev, "Failed to attach smbus: %d\n", rc); + } + +out: + return (rc); +} + +/** + * device_detach() method. attach() didn't do any allocations, so all that's + * needed here is to free up any downstream drivers and children. + * + * @author Joe Kloss + * + * @param[in] dev + * Device being detached. + */ +static int +imcsmb_detach(device_t dev) +{ + int rc; + + /* Detach any attached drivers */ + rc = bus_generic_detach(dev); + if (rc == 0) { + /* Remove all children */ + rc = device_delete_children(dev); + } + + return (rc); +} + +/** + * device_probe() method. All the actual probing was done by the imcsmb_pci + * parent, so just report success. + * + * @author Joe Kloss + * + * @param[in,out] dev + * Device being probed. + */ +static int +imcsmb_probe(device_t dev) +{ + + device_set_desc(dev, "iMC SMBus controller"); + return (BUS_PROBE_DEFAULT); +} + +/** + * smbus_callback() method. Call the parent imcsmb_pci's request or release + * function to quiesce / restart firmware tasks which might use the SMBus. + * + * @author rpokala + * + * @param[in] dev + * Device being requested or released. + * + * @param[in] index + * Either SMB_REQUEST_BUS or SMB_RELEASE_BUS. + * + * @param[in] data + * Tell's the rest of the SMBus subsystem to allow or disallow waiting; + * this driver only works with SMB_DONTWAIT. + */ +static int +imcsmb_callback(device_t dev, int index, void *data) +{ + struct imcsmb_softc *sc; + int *how; + int rc; + + sc = device_get_softc(dev); + how = (int *) data; + + switch (index) { + case SMB_REQUEST_BUS: { + if (*how != SMB_DONTWAIT) { + rc = EINVAL; + goto out; + } + rc = imcsmb_pci_request_bus(sc->imcsmb_pci); + break; + } + case SMB_RELEASE_BUS: + imcsmb_pci_release_bus(sc->imcsmb_pci); + rc = 0; + break; + default: + rc = EINVAL; + break; + } + +out: + return (rc); +} + +/** + * smbus_readb() method. Thin wrapper around imcsmb_transfer(). + * + * @author Joe Kloss + * + * @param[in] dev + * + * @param[in] slave + * The SMBus address of the target device. + * + * @param[in] cmd + * The SMBus command for the target device; this is the offset for SPDs, + * or the register number for TSODs. + * + * @param[out] byte + * The byte which was read. + */ +static int +imcsmb_readb(device_t dev, u_char slave, char cmd, char *byte) +{ + + return (imcsmb_transfer(dev, slave, cmd, byte, FALSE, FALSE)); +} + +/** + * smbus_readw() method. Thin wrapper around imcsmb_transfer(). + * + * @author Joe Kloss + * + * @param[in] dev + * + * @param[in] slave + * The SMBus address of the target device. + * + * @param[in] cmd + * The SMBus command for the target device; this is the offset for SPDs, + * or the register number for TSODs. + * + * @param[out] word + * The word which was read. + */ +static int +imcsmb_readw(device_t dev, u_char slave, char cmd, short *word) +{ + + return (imcsmb_transfer(dev, slave, cmd, word, TRUE, FALSE)); +} + +/** + * smbus_writeb() method. Thin wrapper around imcsmb_transfer(). + * + * @author Joe Kloss + * + * @param[in] dev + * + * @param[in] slave + * The SMBus address of the target device. + * + * @param[in] cmd + * The SMBus command for the target device; this is the offset for SPDs, + * or the register number for TSODs. + * + * @param[in] byte + * The byte to write. + */ +static int +imcsmb_writeb(device_t dev, u_char slave, char cmd, char byte) +{ + + return (imcsmb_transfer(dev, slave, cmd, &byte, FALSE, TRUE)); +} + +/** + * smbus_writew() method. Thin wrapper around imcsmb_transfer(). + * + * @author Joe Kloss + * + * @param[in] dev + * + * @param[in] slave + * The SMBus address of the target device. + * + * @param[in] cmd + * The SMBus command for the target device; this is the offset for SPDs, + * or the register number for TSODs. + * + * @param[in] word + * The word to write. + */ +static int +imcsmb_writew(device_t dev, u_char slave, char cmd, short word) +{ + + return (imcsmb_transfer(dev, slave, cmd, &word, TRUE, TRUE)); +} + +/** + * Manipulate the PCI control registers to read data from or write data to the + * SMBus controller. + * + * @author Joe Kloss, rpokala + * + * @param[in] dev + * + * @param[in] slave + * The SMBus address of the target device. + * + * @param[in] cmd + * The SMBus command for the target device; this is the offset for SPDs, + * or the register number for TSODs. + * + * @param[in,out] data + * Pointer to either the value to be written, or where to place the value + * which was read. + * + * @param[in] word_op + * Bool: is this a word operation? + * + * @param[in] write_op + * Bool: is this a write operation? + */ +static int +imcsmb_transfer(device_t dev, u_char slave, char cmd, void *data, int word_op, + int write_op) +{ + struct imcsmb_softc *sc; + int i; + int rc; + uint32_t cmd_val; + uint32_t cntl_val; + uint32_t orig_cntl_val; + uint32_t stat_val; + uint16_t *word; + uint16_t lword; + uint8_t *byte; + uint8_t lbyte; + + sc = device_get_softc(dev); + byte = data; + word = data; + lbyte = *byte; + lword = *word; + + /* We modify the value of the control register; save the original, so + * we can restore it later + */ + orig_cntl_val = pci_read_config(sc->imcsmb_pci, + sc->regs->smb_cntl, 4); + cntl_val = orig_cntl_val; + + /* + * Set up the SMBCNTL register + */ + + /* [31:28] Clear the existing value of the DTI bits, then set them to + * the four high bits of the slave address. + */ + cntl_val &= ~IMCSMB_CNTL_DTI_MASK; + cntl_val |= ((uint32_t) slave & 0xf0) << 24; + + /* [27:27] Set the CLK_OVERRIDE bit, to enable normal operation */ + cntl_val |= IMCSMB_CNTL_CLK_OVERRIDE; + + /* [26:26] Clear the WRITE_DISABLE bit; the datasheet says this isn't + * necessary, but empirically, it is. + */ + cntl_val &= ~IMCSMB_CNTL_WRITE_DISABLE_BIT; + + /* [9:9] Clear the POLL_EN bit, to stop the hardware TSOD polling. */ + cntl_val &= ~IMCSMB_CNTL_POLL_EN; + + /* + * Set up the SMBCMD register + */ + + /* [31:31] Set the TRIGGER bit; when this gets written, the controller + * will issue the command. + */ + cmd_val = IMCSMB_CMD_TRIGGER_BIT; + + /* [29:29] For word operations, set the WORD_ACCESS bit. */ + if (word_op) { + cmd_val |= IMCSMB_CMD_WORD_ACCESS; + } + + /* [27:27] For write operations, set the WRITE bit. */ + if (write_op) { + cmd_val |= IMCSMB_CMD_WRITE_BIT; + } + + /* [26:24] The three non-DTI, non-R/W bits of the slave address. */ + cmd_val |= (uint32_t) ((slave & 0xe) << 23); + + /* [23:16] The command (offset in the case of an EEPROM, or register in + * the case of TSOD or NVDIMM controller). + */ + cmd_val |= (uint32_t) ((uint8_t) cmd << 16); + + /* [15:0] The data to be written for a write operation. */ + if (write_op) { + if (word_op) { + /* The datasheet says the controller uses different + * endianness for word operations on I2C vs SMBus! + * I2C: [15:8] = MSB; [7:0] = LSB + * SMB: [15:8] = LSB; [7:0] = MSB + * As a practical matter, this controller is very + * specifically for use with DIMMs, the SPD (and + * NVDIMM controllers) are only accessed as bytes, + * the temperature sensor is only accessed as words, and + * the temperature sensors are I2C. Thus, byte-swap the + * word. + */ + lword = htobe16(lword); + } else { + /* For byte operations, the data goes in the LSB, and + * the MSB is a don't care. + */ + lword = (uint16_t) (lbyte & 0xff); + } + cmd_val |= lword; + } + + /* Write the updated value to the control register first, to disable + * the hardware TSOD polling. + */ + pci_write_config(sc->imcsmb_pci, sc->regs->smb_cntl, cntl_val, 4); + + /* Poll on the BUSY bit in the status register until clear, or timeout. + * We just cleared the auto-poll bit, so we need to make sure the device + * is idle before issuing a command. We can safely timeout after 35 ms, + * as this is the maximum time the SMBus spec allows for a transaction. + */ + for (i = 4; i != 0; i--) { + stat_val = pci_read_config(sc->imcsmb_pci, sc->regs->smb_stat, + 4); + if ((stat_val & IMCSMB_STATUS_BUSY_BIT) == 0) { + break; + } + pause("imcsmb", 10 * hz / 1000); + } + + if (i == 0) { + device_printf(sc->dev, + "transfer: timeout waiting for device to settle\n"); + } + + /* Now that polling has stopped, we can write the command register. This + * starts the SMBus command. + */ + pci_write_config(sc->imcsmb_pci, sc->regs->smb_cmd, cmd_val, 4); + + /* Wait for WRITE_DATA_DONE/READ_DATA_VALID to be set, or timeout and + * fail. We wait up to 35ms. + */ + for (i = 35000; i != 0; i -= 10) + { + DELAY(10); + stat_val = pci_read_config(sc->imcsmb_pci, sc->regs->smb_stat, + 4); + /* For a write, the bits holding the data contain the data being + * written. You'd think that would cause the READ_DATA_VALID bit + * to be cleared, because the data bits no longer contain valid + * data from the most recent read operation. While that would be + * logical, that's not the case here: READ_DATA_VALID is only + * cleared when starting a read operation, and WRITE_DATA_DONE + * is only cleared when starting a write operation. + */ + if (write_op) { + if ((stat_val & IMCSMB_STATUS_WRITE_DATA_DONE) != 0) { + break; + } + } else { + if ((stat_val & IMCSMB_STATUS_READ_DATA_VALID) != 0) { + break; + } + } + } + if (i == 0) { + rc = SMB_ETIMEOUT; + device_printf(dev, "transfer timeout\n"); + goto out; + } + + /* It is generally the case that this bit indicates non-ACK, but it + * could also indicate other bus errors. There's no way to tell the + * difference. + */ + if ((stat_val & IMCSMB_STATUS_BUS_ERROR_BIT) != 0) { + /* While it is not documented, empirically, SPD page-change + * commands (writes with DTI = 0x60) always complete with the + * error bit set. So, ignore it in those cases. + */ + if ((slave & 0xf0) != 0x60) { + rc = SMB_ENOACK; + goto out; + } + } + + /* For a read operation, copy the data out */ + if (write_op == 0) { + if (word_op) { + /* The data is returned in bits [15:0]; as discussed + * above, byte-swap. + */ + lword = (uint16_t) (stat_val & 0xffff); + lword = htobe16(lword); + *word = lword; + } else { + /* The data is returned in bits [7:0] */ + lbyte = (uint8_t) (stat_val & 0xff); + *byte = lbyte; + } + } + + /* A lack of an error is, de facto, success. */ + rc = SMB_ENOERR; + +out: + /* Restore the original value of the control register. */ + pci_write_config(sc->imcsmb_pci, sc->regs->smb_cntl, orig_cntl_val, 4); + return (rc); +} + +/* Our device class */ +static devclass_t imcsmb_devclass; + +/* Device methods */ +static device_method_t imcsmb_methods[] = { + /* Device interface */ + DEVMETHOD(device_attach, imcsmb_attach), + DEVMETHOD(device_detach, imcsmb_detach), + DEVMETHOD(device_probe, imcsmb_probe), + + /* smbus methods */ + DEVMETHOD(smbus_callback, imcsmb_callback), + DEVMETHOD(smbus_readb, imcsmb_readb), + DEVMETHOD(smbus_readw, imcsmb_readw), + DEVMETHOD(smbus_writeb, imcsmb_writeb), + DEVMETHOD(smbus_writew, imcsmb_writew), + + DEVMETHOD_END +}; + +static driver_t imcsmb_driver = { + .name = "imcsmb", + .methods = imcsmb_methods, + .size = sizeof(struct imcsmb_softc), +}; + +DRIVER_MODULE(imcsmb, imcsmb_pci, imcsmb_driver, imcsmb_devclass, 0, 0); +MODULE_DEPEND(imcsmb, smbus, SMBUS_MINVER, SMBUS_PREFVER, SMBUS_MAXVER); +MODULE_VERSION(imcsmb, 1); + +DRIVER_MODULE(smbus, imcsmb, smbus_driver, smbus_devclass, 0, 0); + +/* vi: set ts=8 sw=4 sts=8 noet: */ Index: head/sys/dev/imcsmb/imcsmb_pci.c =================================================================== --- head/sys/dev/imcsmb/imcsmb_pci.c +++ head/sys/dev/imcsmb/imcsmb_pci.c @@ -0,0 +1,345 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Authors: Joe Kloss; Ravi Pokala (rpokala@freebsd.org) + * + * Copyright (c) 2017-2018 Panasas + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include + +#include "imcsmb_reg.h" +#include "imcsmb_var.h" + +/* (Sandy,Ivy)bridge-Xeon and (Has,Broad)well-Xeon CPUs contain one or two + * "Integrated Memory Controllers" (iMCs), and each iMC contains two separate + * SMBus controllers. These are used for reading SPD data from the DIMMs, and + * for reading the "Thermal Sensor on DIMM" (TSODs). The iMC SMBus controllers + * are very simple devices, and have limited functionality compared to + * full-fledged SMBus controllers, like the one in Intel ICHs and PCHs. + * + * The publicly available documentation for the iMC SMBus controllers can be + * found in the CPU datasheets for (Sandy,Ivy)bridge-Xeon and + * (Has,broad)well-Xeon, respectively: + * + * https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/ + * Sandybridge xeon-e5-1600-2600-vol-2-datasheet.pdf + * Ivybridge xeon-e5-v2-datasheet-vol-2.pdf + * Haswell xeon-e5-v3-datasheet-vol-2.pdf + * Broadwell xeon-e5-v4-datasheet-vol-2.pdf + * + * Another useful resource is the Linux driver. It is not in the main tree. + * + * https://www.mail-archive.com/linux-kernel@vger.kernel.org/msg840043.html + * + * The iMC SMBus controllers do not support interrupts (thus, they must be + * polled for IO completion). All of the iMC registers are in PCI configuration + * space; there is no support for PIO or MMIO. As a result, this driver does + * not need to perform and newbus resource manipulation. + * + * Because there are multiple SMBus controllers sharing the same PCI device, + * this driver is actually *two* drivers: + * + * - "imcsmb" is an smbus(4)-compliant SMBus controller driver + * + * - "imcsmb_pci" recognizes the PCI device and assigns the appropriate set of + * PCI config registers to a specific "imcsmb" instance. + */ + +/* Depending on the motherboard and firmware, the TSODs might be polled by + * firmware. Therefore, when this driver accesses these SMBus controllers, the + * firmware polling must be disabled as part of requesting the bus, and + * re-enabled when releasing the bus. Unfortunately, the details of how to do + * this are vendor-specific. Contact your motherboard vendor to get the + * information you need to do proper implementations. + * + * For NVDIMMs which conform to the ACPI "NFIT" standard, the ACPI firmware + * manages the NVDIMM; for those which pre-date the standard, the operating + * system interacts with the NVDIMM controller using a vendor-proprietary API + * over the SMBus. In that case, the NVDIMM driver would be an SMBus slave + * device driver, and would interface with the hardware via an SMBus controller + * driver such as this one. + */ + +/* PCIe device IDs for (Sandy,Ivy)bridge)-Xeon and (Has,Broad)well-Xeon */ +#define PCI_VENDOR_INTEL 0x8086 +#define IMCSMB_PCI_DEV_ID_IMC0_SBX 0x3ca8 +#define IMCSMB_PCI_DEV_ID_IMC0_IBX 0x0ea8 +#define IMCSMB_PCI_DEV_ID_IMC0_HSX 0x2fa8 +#define IMCSMB_PCI_DEV_ID_IMC0_BDX 0x6fa8 +/* (Sandy,Ivy)bridge-Xeon only have a single memory controller per socket */ +#define IMCSMB_PCI_DEV_ID_IMC1_HSX 0x2f68 +#define IMCSMB_PCI_DEV_ID_IMC1_BDX 0x6f68 + +/* There are two SMBus controllers in each device. These define the registers + * for each of these devices. + */ +static struct imcsmb_reg_set imcsmb_regs[] = { + { + .smb_stat = IMCSMB_REG_STATUS0, + .smb_cmd = IMCSMB_REG_COMMAND0, + .smb_cntl = IMCSMB_REG_CONTROL0 + }, + { + .smb_stat = IMCSMB_REG_STATUS1, + .smb_cmd = IMCSMB_REG_COMMAND1, + .smb_cntl = IMCSMB_REG_CONTROL1 + }, +}; + +static struct imcsmb_pci_device { + uint16_t id; + char *name; +} imcsmb_pci_devices[] = { + {IMCSMB_PCI_DEV_ID_IMC0_SBX, + "Intel Sandybridge Xeon iMC 0 SMBus controllers" }, + {IMCSMB_PCI_DEV_ID_IMC0_IBX, + "Intel Ivybridge Xeon iMC 0 SMBus controllers" }, + {IMCSMB_PCI_DEV_ID_IMC0_HSX, + "Intel Haswell Xeon iMC 0 SMBus controllers" }, + {IMCSMB_PCI_DEV_ID_IMC1_HSX, + "Intel Haswell Xeon iMC 1 SMBus controllers" }, + {IMCSMB_PCI_DEV_ID_IMC0_BDX, + "Intel Broadwell Xeon iMC 0 SMBus controllers" }, + {IMCSMB_PCI_DEV_ID_IMC1_BDX, + "Intel Broadwell Xeon iMC 1 SMBus controllers" }, + {0, NULL}, +}; + +/* Device methods. */ +static int imcsmb_pci_attach(device_t dev); +static int imcsmb_pci_detach(device_t dev); +static int imcsmb_pci_probe(device_t dev); + +/** + * device_attach() method. Set up the PCI device's softc, then explicitly create + * children for the actual imcsmbX controllers. Set up the child's ivars to + * point to the proper set of the PCI device's config registers. + * + * @author Joe Kloss, rpokala + * + * @param[in,out] dev + * Device being attached. + */ +static int +imcsmb_pci_attach(device_t dev) +{ + struct imcsmb_pci_softc *sc; + device_t child; + int rc; + int unit; + + /* Initialize private state */ + sc = device_get_softc(dev); + sc->dev = dev; + sc->semaphore = 0; + + /* Create the imcsmbX children */ + for (unit = 0; unit < 2; unit++) { + child = device_add_child(dev, "imcsmb", -1); + if (child == NULL) { + /* Nothing has been allocated, so there's no cleanup. */ + device_printf(dev, "Child imcsmb not added\n"); + rc = ENXIO; + goto out; + } + /* Set the child's ivars to point to the appropriate set of + * the PCI device's registers. + */ + device_set_ivars(child, &imcsmb_regs[unit]); + } + + /* Attach the imcsmbX children. */ + if ((rc = bus_generic_attach(dev)) != 0) { + device_printf(dev, "failed to attach children: %d\n", rc); + goto out; + } + +out: + return (rc); +} + +/** + * device_detach() method. attach() didn't do any allocations, so all that's + * needed here is to free up any downstream drivers and children. + * + * @author Joe Kloss + * + * @param[in] dev + * Device being detached. + */ +static int +imcsmb_pci_detach(device_t dev) +{ + int rc; + + /* Detach any attached drivers */ + rc = bus_generic_detach(dev); + if (rc == 0) { + /* Remove all children */ + rc = device_delete_children(dev); + } + + return (rc); +} + +/** + * device_probe() method. Look for the right PCI vendor/device IDs. + * + * @author Joe Kloss, rpokala + * + * @param[in,out] dev + * Device being probed. + */ +static int +imcsmb_pci_probe(device_t dev) +{ + struct imcsmb_pci_device *pci_device; + int rc; + uint16_t pci_dev_id; + + rc = ENXIO; + + if (pci_get_vendor(dev) != PCI_VENDOR_INTEL) { + goto out; + } + + pci_dev_id = pci_get_device(dev); + for (pci_device = imcsmb_pci_devices; + pci_device->name != NULL; + pci_device++) { + if (pci_dev_id == pci_device->id) { + device_set_desc(dev, pci_device->name); + rc = BUS_PROBE_DEFAULT; + goto out; + } + } + +out: + return (rc); +} + +/** + * Invoked via smbus_callback() -> imcsmb_callback(); clear the semaphore, and + * re-enable motherboard-specific DIMM temperature monitoring if needed. This + * gets called after the transaction completes. + * + * @author Joe Kloss + * + * @param[in,out] dev + * The device whose busses to release. + */ +void +imcsmb_pci_release_bus(device_t dev) +{ + struct imcsmb_pci_softc *sc; + + sc = device_get_softc(dev); + + /* + * IF NEEDED, INSERT MOTHERBOARD-SPECIFIC CODE TO RE-ENABLE DIMM + * TEMPERATURE MONITORING HERE. + */ + + atomic_store_rel_int(&sc->semaphore, 0); +} + +/** + * Invoked via smbus_callback() -> imcsmb_callback(); set the semaphore, and + * disable motherboard-specific DIMM temperature monitoring if needed. This gets + * called before the transaction starts. + * + * @author Joe Kloss + * + * @param[in,out] dev + * The device whose busses to request. + */ +int +imcsmb_pci_request_bus(device_t dev) +{ + struct imcsmb_pci_softc *sc; + int rc; + + sc = device_get_softc(dev); + rc = 0; + + /* We don't want to block. Use a simple test-and-set semaphore to + * protect the bus. + */ + if (atomic_cmpset_acq_int(&sc->semaphore, 0, 1) == 0) { + rc = EWOULDBLOCK; + } + + /* + * IF NEEDED, INSERT MOTHERBOARD-SPECIFIC CODE TO DISABLE DIMM + * TEMPERATURE MONITORING HERE. + */ + + return (rc); +} + +/* Our device class */ +static devclass_t imcsmb_pci_devclass; + +/* Device methods */ +static device_method_t imcsmb_pci_methods[] = { + /* Device interface */ + DEVMETHOD(device_attach, imcsmb_pci_attach), + DEVMETHOD(device_detach, imcsmb_pci_detach), + DEVMETHOD(device_probe, imcsmb_pci_probe), + + DEVMETHOD_END +}; + +static driver_t imcsmb_pci_driver = { + .name = "imcsmb_pci", + .methods = imcsmb_pci_methods, + .size = sizeof(struct imcsmb_pci_softc), +}; + +DRIVER_MODULE(imcsmb_pci, pci, imcsmb_pci_driver, imcsmb_pci_devclass, 0, 0); +MODULE_DEPEND(imcsmb_pci, pci, 1, 1, 1); +MODULE_VERSION(imcsmb_pci, 1); + +/* vi: set ts=8 sw=4 sts=8 noet: */ Index: head/sys/dev/imcsmb/imcsmb_reg.h =================================================================== --- head/sys/dev/imcsmb/imcsmb_reg.h +++ head/sys/dev/imcsmb/imcsmb_reg.h @@ -0,0 +1,86 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Authors: Joe Kloss; Ravi Pokala (rpokala@freebsd.org) + * + * Copyright (c) 2017-2018 Panasas + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _DEV__IMCSMB__IMCSMB_REG_H_ +#define _DEV__IMCSMB__IMCSMB_REG_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include + +/* Intel (Sandy,Ivy)bridge and (Has,Broad)well CPUs have integrated memory + * controllers (iMCs), each of which having up to two SMBus controllers. They + * are programmed via sets of registers in the same PCI device, which are + * identical other than the register numbers. + * + * The full documentation for these registers can be found in volume two of the + * datasheets for the CPUs. Refer to the links in imcsmb_pci.c + */ + +#define IMCSMB_REG_STATUS0 0x0180 +#define IMCSMB_REG_STATUS1 0x0190 +#define IMCSMB_STATUS_BUSY_BIT 0x10000000 +#define IMCSMB_STATUS_BUS_ERROR_BIT 0x20000000 +#define IMCSMB_STATUS_WRITE_DATA_DONE 0x40000000 +#define IMCSMB_STATUS_READ_DATA_VALID 0x80000000 + +#define IMCSMB_REG_COMMAND0 0x0184 +#define IMCSMB_REG_COMMAND1 0x0194 +#define IMCSMB_CMD_WORD_ACCESS 0x20000000 +#define IMCSMB_CMD_WRITE_BIT 0x08000000 +#define IMCSMB_CMD_TRIGGER_BIT 0x80000000 + +#define IMCSMB_REG_CONTROL0 0x0188 +#define IMCSMB_REG_CONTROL1 0x0198 +#define IMCSMB_CNTL_POLL_EN 0x00000100 +#define IMCSMB_CNTL_CLK_OVERRIDE 0x08000000 +#define IMCSMB_CNTL_DTI_MASK 0xf0000000 +#define IMCSMB_CNTL_WRITE_DISABLE_BIT 0x04000000 + +#endif /* _DEV__IMCSMB__IMCSMB_REG_H_ */ + +/* vi: set ts=8 sw=4 sts=8 noet: */ Index: head/sys/dev/imcsmb/imcsmb_var.h =================================================================== --- head/sys/dev/imcsmb/imcsmb_var.h +++ head/sys/dev/imcsmb/imcsmb_var.h @@ -0,0 +1,107 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Authors: Joe Kloss; Ravi Pokala (rpokala@freebsd.org) + * + * Copyright (c) 2017-2018 Panasas + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _DEV__IMCSMB__IMCSMB_VAR_H_ +#define _DEV__IMCSMB__IMCSMB_VAR_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include + +#include "smbus_if.h" + +/* A detailed description of this device is present in imcsmb_pci.c */ + +/** + * The softc for a particular instance of the PCI device associated with a pair + * of iMC-SMB controllers. + * + * Ordinarily, locking would be done with a mutex. However, we might have an + * NVDIMM connected to this SMBus, and we might need to issue the SAVE command + * to the NVDIMM from a panic context. Mutex operations are not allowed while + * the scheduler is stopped, so just use a simple semaphore. + * + * If, as described in the manpage, additional steps are needed to stop/restart + * firmware operations before/after using the controller, then additional fields + * can be added to this softc. + */ +struct imcsmb_pci_softc { + device_t dev; + volatile int semaphore; +}; + +void imcsmb_pci_release_bus(device_t dev); +int imcsmb_pci_request_bus(device_t dev); + +/** + * PCI config registers for each individual SMBus controller within the iMC. + * Each iMC-SMB has a separate set of registers. There is an array of these + * structures for the PCI device, and one of them is passed to driver for the + * actual iMC-SMB as the IVAR. + */ +struct imcsmb_reg_set { + uint16_t smb_stat; + uint16_t smb_cmd; + uint16_t smb_cntl; +}; + +/** + * The softc for the device associated with a particular iMC-SMB controller. + * There are two such controllers for each of the PCI devices. The PCI driver + * tells the iMC-SMB driver which set of registers to use via the IVAR. This + * technique was suggested by John Baldwin (jhb@). + */ +struct imcsmb_softc { + device_t dev; + device_t imcsmb_pci; /* The SMBus controller's parent iMC */ + device_t smbus; /* The child smbusX interface */ + struct imcsmb_reg_set *regs; /* The registers this controller uses */ +}; + +#endif /* _DEV__IMCSMB__IMCSMB_VAR_H_ */ + +/* vi: set ts=8 sw=4 sts=8 noet: */ Index: head/sys/i386/conf/NOTES =================================================================== --- head/sys/i386/conf/NOTES +++ head/sys/i386/conf/NOTES @@ -710,6 +710,11 @@ device hptiop # +# Intel integrated Memory Controller (iMC) SMBus controller +# Sandybridge-Xeon, Ivybridge-Xeon, Haswell-Xeon, Broadwell-Xeon +device imcsmb + +# # IBM (now Adaptec) ServeRAID controllers device ips Index: head/sys/modules/i2c/controllers/Makefile =================================================================== --- head/sys/modules/i2c/controllers/Makefile +++ head/sys/modules/i2c/controllers/Makefile @@ -2,4 +2,8 @@ SUBDIR = alpm amdpm amdsmb ichiic ichsmb intpm ismt nfsmb viapm lpbb pcf +.if ${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "amd64" +SUBDIR += imcsmb +.endif + .include Index: head/sys/modules/i2c/controllers/imcsmb/Makefile =================================================================== --- head/sys/modules/i2c/controllers/imcsmb/Makefile +++ head/sys/modules/i2c/controllers/imcsmb/Makefile @@ -0,0 +1,8 @@ +#$FreeBSD$ + +.PATH: ${SRCTOP}/sys/dev/imcsmb +KMOD = imcsmb +SRCS = device_if.h bus_if.h pci_if.h smbus_if.h \ + imcsmb.c imcsmb_pci.c imcsmb_reg.h imcsmb_var.h + +.include