Index: stable/11/sys/amd64/conf/NOTES =================================================================== --- stable/11/sys/amd64/conf/NOTES (revision 329158) +++ stable/11/sys/amd64/conf/NOTES (revision 329159) @@ -1,702 +1,703 @@ # # NOTES -- Lines that can be cut/pasted into kernel and hints configs. # # This file contains machine dependent kernel configuration notes. For # machine independent notes, look in /sys/conf/NOTES. # # $FreeBSD$ # # # We want LINT to cover profiling as well. profile 2 # # Enable the kernel DTrace hooks which are required to load the DTrace # kernel modules. # options KDTRACE_HOOKS # DTrace core # NOTE: introduces CDDL-licensed components into the kernel #device dtrace # DTrace modules #device dtrace_profile #device dtrace_sdt #device dtrace_fbt #device dtrace_systrace #device dtrace_prototype #device dtnfscl #device dtmalloc # Alternatively include all the DTrace modules #device dtraceall ##################################################################### # SMP OPTIONS: # # Notes: # # IPI_PREEMPTION instructs the kernel to preempt threads running on other # CPUS if needed. Relies on the PREEMPTION option # Optional: options IPI_PREEMPTION device atpic # Optional legacy pic support device mptable # Optional MPSPEC mptable support # # Watchdog routines. # options MP_WATCHDOG # Debugging options. # options COUNT_XINVLTLB_HITS # Counters for TLB events options COUNT_IPIS # Per-CPU IPI interrupt counters ##################################################################### # CPU OPTIONS # # You must specify at least one CPU (the one you intend to run on); # deleting the specification for CPUs you don't need to use may make # parts of the system run faster. # cpu HAMMER # aka K8, aka Opteron & Athlon64 # # Options for CPU features. # # # PERFMON causes the driver for Pentium/Pentium Pro performance counters # to be compiled. See perfmon(4) for more information. # #XXX#options PERFMON ##################################################################### # NETWORKING OPTIONS # # DEVICE_POLLING adds support for mixed interrupt-polling handling # of network device drivers, which has significant benefits in terms # of robustness to overloads and responsivity, as well as permitting # accurate scheduling of the CPU time between kernel network processing # and other activities. The drawback is a moderate (up to 1/HZ seconds) # potential increase in response times. # It is strongly recommended to use HZ=1000 or 2000 with DEVICE_POLLING # to achieve smoother behaviour. # Additionally, you can enable/disable polling at runtime with help of # the ifconfig(8) utility, and select the CPU fraction reserved to # userland with the sysctl variable kern.polling.user_frac # (default 50, range 0..100). # # Not all device drivers support this mode of operation at the time of # this writing. See polling(4) for more details. options DEVICE_POLLING # BPF_JITTER adds support for BPF just-in-time compiler. options BPF_JITTER # OpenFabrics Enterprise Distribution (Infiniband). options OFED options OFED_DEBUG_INIT # Sockets Direct Protocol options SDP options SDP_DEBUG # IP over Infiniband options IPOIB options IPOIB_DEBUG options IPOIB_CM ##################################################################### # CLOCK OPTIONS # Provide read/write access to the memory in the clock chip. device nvram # Access to rtc cmos via /dev/nvram ##################################################################### # MISCELLANEOUS DEVICES AND OPTIONS device speaker #Play IBM BASIC-style noises out your speaker hint.speaker.0.at="isa" hint.speaker.0.port="0x61" device gzip #Exec gzipped a.out's. REQUIRES COMPAT_AOUT! ##################################################################### # HARDWARE BUS CONFIGURATION # # ISA bus # device isa # # Options for `isa': # # AUTO_EOI_1 enables the `automatic EOI' feature for the master 8259A # interrupt controller. This saves about 0.7-1.25 usec for each interrupt. # This option breaks suspend/resume on some portables. # # AUTO_EOI_2 enables the `automatic EOI' feature for the slave 8259A # interrupt controller. This saves about 0.7-1.25 usec for each interrupt. # Automatic EOI is documented not to work for for the slave with the # original i8259A, but it works for some clones and some integrated # versions. # # MAXMEM specifies the amount of RAM on the machine; if this is not # specified, FreeBSD will first read the amount of memory from the CMOS # RAM, so the amount of memory will initially be limited to 64MB or 16MB # depending on the BIOS. If the BIOS reports 64MB, a memory probe will # then attempt to detect the installed amount of RAM. If this probe # fails to detect >64MB RAM you will have to use the MAXMEM option. # The amount is in kilobytes, so for a machine with 128MB of RAM, it would # be 131072 (128 * 1024). # # BROKEN_KEYBOARD_RESET disables the use of the keyboard controller to # reset the CPU for reboot. This is needed on some systems with broken # keyboard controllers. options AUTO_EOI_1 #options AUTO_EOI_2 options MAXMEM=(128*1024) #options BROKEN_KEYBOARD_RESET # # AGP GART support device agp # # AGP debugging. # options AGP_DEBUG ##################################################################### # HARDWARE DEVICE CONFIGURATION # To include support for VGA VESA video modes options VESA # Turn on extra debugging checks and output for VESA support. options VESA_DEBUG device dpms # DPMS suspend & resume via VESA BIOS # x86 real mode BIOS emulator, required by atkbdc/dpms/vesa options X86BIOS # # Optional devices: # # PS/2 mouse device psm hint.psm.0.at="atkbdc" hint.psm.0.irq="12" # Options for psm: options PSM_HOOKRESUME #hook the system resume event, useful #for some laptops options PSM_RESETAFTERSUSPEND #reset the device at the resume event # The keyboard controller; it controls the keyboard and the PS/2 mouse. device atkbdc hint.atkbdc.0.at="isa" hint.atkbdc.0.port="0x060" # The AT keyboard device atkbd hint.atkbd.0.at="atkbdc" hint.atkbd.0.irq="1" # Options for atkbd: options ATKBD_DFLT_KEYMAP # specify the built-in keymap makeoptions ATKBD_DFLT_KEYMAP=fr.dvorak # `flags' for atkbd: # 0x01 Force detection of keyboard, else we always assume a keyboard # 0x02 Don't reset keyboard, useful for some newer ThinkPads # 0x03 Force detection and avoid reset, might help with certain # dockingstations # 0x04 Old-style (XT) keyboard support, useful for older ThinkPads # Video card driver for VGA adapters. device vga hint.vga.0.at="isa" # Options for vga: # Try the following option if the mouse pointer is not drawn correctly # or font does not seem to be loaded properly. May cause flicker on # some systems. options VGA_ALT_SEQACCESS # If you can dispense with some vga driver features, you may want to # use the following options to save some memory. #options VGA_NO_FONT_LOADING # don't save/load font #options VGA_NO_MODE_CHANGE # don't change video modes # Older video cards may require this option for proper operation. options VGA_SLOW_IOACCESS # do byte-wide i/o's to TS and GDC regs # The following option probably won't work with the LCD displays. options VGA_WIDTH90 # support 90 column modes # Debugging. options VGA_DEBUG # vt(4) drivers. device vt_vga # VGA device vt_efifb # EFI framebuffer # Linear framebuffer driver for S3 VESA 1.2 cards. Works on top of VESA. device s3pci # 3Dfx Voodoo Graphics, Voodoo II /dev/3dfx CDEV support. This will create # the /dev/3dfx0 device to work with glide implementations. This should get # linked to /dev/3dfx and /dev/voodoo. Note that this is not the same as # the tdfx DRI module from XFree86 and is completely unrelated. # # To enable Linuxulator support, one must also include COMPAT_LINUX in the # config as well. The other option is to load both as modules. device tdfx # Enable 3Dfx Voodoo support #XXX#device tdfx_linux # Enable Linuxulator support # # ACPI support using the Intel ACPI Component Architecture reference # implementation. # # ACPI_DEBUG enables the use of the debug.acpi.level and debug.acpi.layer # kernel environment variables to select initial debugging levels for the # Intel ACPICA code. (Note that the Intel code must also have USE_DEBUGGER # defined when it is built). device acpi options ACPI_DEBUG # The cpufreq(4) driver provides support for non-ACPI CPU frequency control device cpufreq # Direct Rendering modules for 3D acceleration. device drm # DRM core module required by DRM drivers device i915drm # Intel i830 through i915 device mach64drm # ATI Rage Pro, Rage Mobility P/M, Rage XL device mgadrm # AGP Matrox G200, G400, G450, G550 device r128drm # ATI Rage 128 device radeondrm # ATI Radeon device savagedrm # S3 Savage3D, Savage4 device sisdrm # SiS 300/305, 540, 630 device tdfxdrm # 3dfx Voodoo 3/4/5 and Banshee device viadrm # VIA options DRM_DEBUG # Include debug printfs (slow) # # Network interfaces: # # bxe: Broadcom NetXtreme II (BCM5771X/BCM578XX) PCIe 10Gb Ethernet # adapters. # ed: Western Digital and SMC 80xx; Novell NE1000 and NE2000; 3Com 3C503 # HP PC Lan+, various PC Card devices # (requires miibus) # ipw: Intel PRO/Wireless 2100 IEEE 802.11 adapter # Requires the ipw firmware module # iwi: Intel PRO/Wireless 2200BG/2225BG/2915ABG IEEE 802.11 adapters # Requires the iwi firmware module # iwn: Intel Wireless WiFi Link 1000/105/135/2000/4965/5000/6000/6050 abgn # 802.11 network adapters # Requires the iwn firmware module # ixl: Intel XL710 40Gbe PCIE Ethernet # ixlv: Intel XL710 40Gbe VF PCIE Ethernet # mlx4ib: Mellanox ConnectX HCA InfiniBand -# mlxen: Mellanox ConnectX HCA Ethernet +# mlx4en: Mellanox ConnectX HCA Ethernet # mthca: Mellanox HCA InfiniBand # nfe: nVidia nForce MCP on-board Ethernet Networking (BSD open source) # sfxge: Solarflare SFC9000 family 10Gb Ethernet adapters # vmx: VMware VMXNET3 Ethernet (BSD open source) # wpi: Intel 3945ABG Wireless LAN controller # Requires the wpi firmware module device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE device ed # NE[12]000, SMC Ultra, 3c503, DS8390 cards options ED_3C503 options ED_HPP options ED_SIC device ipw # Intel 2100 wireless NICs. device iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. device iwn # Intel 4965/1000/5000/6000 wireless NICs. device ixl # Intel XL710 40Gbe PCIE Ethernet options IXL_IW # Enable iWARP Client Interface in ixl(4) device ixlv # Intel XL710 40Gbe VF PCIE Ethernet +device mlx4 # Shared code module between IB and Ethernet device mlx4ib # Mellanox ConnectX HCA InfiniBand -device mlxen # Mellanox ConnectX HCA Ethernet +device mlx4en # Mellanox ConnectX HCA Ethernet device mthca # Mellanox HCA InfiniBand device nfe # nVidia nForce MCP on-board Ethernet device sfxge # Solarflare SFC9000 10Gb Ethernet device vmx # VMware VMXNET3 Ethernet device wpi # Intel 3945ABG wireless NICs. # IEEE 802.11 adapter firmware modules # Intel PRO/Wireless 2100 firmware: # ipwfw: BSS/IBSS/monitor mode firmware # ipwbssfw: BSS mode firmware # ipwibssfw: IBSS mode firmware # ipwmonitorfw: Monitor mode firmware # Intel PRO/Wireless 2200BG/2225BG/2915ABG firmware: # iwifw: BSS/IBSS/monitor mode firmware # iwibssfw: BSS mode firmware # iwiibssfw: IBSS mode firmware # iwimonitorfw: Monitor mode firmware # Intel Wireless WiFi Link 4965/1000/5000/6000 series firmware: # iwnfw: Single module to support all devices # iwn1000fw: Specific module for the 1000 only # iwn105fw: Specific module for the 105 only # iwn135fw: Specific module for the 135 only # iwn2000fw: Specific module for the 2000 only # iwn2030fw: Specific module for the 2030 only # iwn4965fw: Specific module for the 4965 only # iwn5000fw: Specific module for the 5000 only # iwn5150fw: Specific module for the 5150 only # iwn6000fw: Specific module for the 6000 only # iwn6000g2afw: Specific module for the 6000g2a only # iwn6000g2bfw: Specific module for the 6000g2b only # iwn6050fw: Specific module for the 6050 only # wpifw: Intel 3945ABG Wireless LAN Controller firmware device iwifw device iwibssfw device iwiibssfw device iwimonitorfw device ipwfw device ipwbssfw device ipwibssfw device ipwmonitorfw device iwnfw device iwn1000fw device iwn105fw device iwn135fw device iwn2000fw device iwn2030fw device iwn4965fw device iwn5000fw device iwn5150fw device iwn6000fw device iwn6000g2afw device iwn6000g2bfw device iwn6050fw device wpifw # # Non-Transparent Bridge (NTB) drivers # device if_ntb # Virtual NTB network interface device ntb_transport # NTB packet transport driver device ntb # NTB hardware interface device ntb_hw_intel # Intel NTB hardware driver device ntb_hw_plx # PLX NTB hardware driver # #XXX this stores pointers in a 32bit field that is defined by the hardware #device pst # # Areca 11xx and 12xx series of SATA II RAID controllers. # CAM is required. # device arcmsr # Areca SATA II RAID # # 3ware 9000 series PATA/SATA RAID controller driver and options. # The driver is implemented as a SIM, and so, needs the CAM infrastructure. # options TWA_DEBUG # 0-10; 10 prints the most messages. options TWA_FLASH_FIRMWARE # firmware image bundled when defined. device twa # 3ware 9000 series PATA/SATA RAID # # SCSI host adapters: # # ncv: NCR 53C500 based SCSI host adapters. # nsp: Workbit Ninja SCSI-3 based PC Card SCSI host adapters. # stg: TMC 18C30, 18C50 based SCSI host adapters. device ncv device nsp device stg # # Adaptec FSA RAID controllers, including integrated DELL controllers, # the Dell PERC 2/QC and the HP NetRAID-4M device aac device aacp # SCSI Passthrough interface (optional, CAM required) # # Adaptec by PMC RAID controllers, Series 6/7/8 and upcoming families device aacraid # Container interface, CAM required # # Highpoint RocketRAID 27xx. device hpt27xx # # Highpoint RocketRAID 182x. device hptmv # # Highpoint DC7280 and R750. device hptnr # # Highpoint RocketRAID. Supports RR172x, RR222x, RR2240, RR232x, RR2340, # RR2210, RR174x, RR2522, RR231x, RR230x. device hptrr # # Highpoint RocketRaid 3xxx series SATA RAID device hptiop # # IBM (now Adaptec) ServeRAID controllers device ips # # Intel C600 (Patsburg) integrated SAS controller device isci options ISCI_LOGGING # enable debugging in isci HAL # # NVM Express (NVMe) support device nvme # base NVMe driver device nvd # expose NVMe namespaces as disks, depends on nvme # # PMC-Sierra SAS/SATA controller device pmspcv # # SafeNet crypto driver: can be moved to the MI NOTES as soon as # it's tested on a big-endian machine # device safe # SafeNet 1141 options SAFE_DEBUG # enable debugging support: hw.safe.debug options SAFE_RNDTEST # enable rndtest support # # VirtIO support # # The virtio entry provides a generic bus for use by the device drivers. # It must be combined with an interface that communicates with the host. # Multiple such interfaces are defined by the VirtIO specification. FreeBSD # only has support for PCI. Therefore, virtio_pci must be statically # compiled in or loaded as a module for the device drivers to function. # device virtio # Generic VirtIO bus (required) device virtio_pci # VirtIO PCI Interface device vtnet # VirtIO Ethernet device device virtio_blk # VirtIO Block device device virtio_scsi # VirtIO SCSI device device virtio_balloon # VirtIO Memory Balloon device device virtio_random # VirtIO Entropy device device virtio_console # VirtIO Console device # Microsoft Hyper-V enhancement support device hyperv # HyperV drivers # Xen HVM Guest Optimizations options XENHVM # Xen HVM kernel infrastructure device xenpci # Xen HVM Hypervisor services driver ##################################################################### # # Miscellaneous hardware: # # ipmi: Intelligent Platform Management Interface # pbio: Parallel (8255 PPI) basic I/O (mode 0) port (e.g. Advantech PCL-724) # smbios: DMI/SMBIOS entry point # vpd: Vital Product Data kernel interface # asmc: Apple System Management Controller # si: Specialix International SI/XIO or SX intelligent serial card # tpm: Trusted Platform Module # Notes on the Specialix SI/XIO driver: # The host card is memory, not IO mapped. # The Rev 1 host cards use a 64K chunk, on a 32K boundary. # The Rev 2 host cards use a 32K chunk, on a 32K boundary. # The cards can use an IRQ of 11, 12 or 15. device ipmi device pbio hint.pbio.0.at="isa" hint.pbio.0.port="0x360" device smbios device vpd device asmc device si device tpm device padlock_rng # VIA Padlock RNG device rdrand_rng # Intel Bull Mountain RNG device aesni # AES-NI OpenCrypto module device ioat # Intel I/OAT DMA engine # # Laptop/Notebook options: # # # I2C Bus # # # Hardware watchdog timers: # # ichwd: Intel ICH watchdog timer # amdsbwd: AMD SB7xx watchdog timer # viawd: VIA south bridge watchdog timer # wbwd: Winbond watchdog timer # device ichwd device amdsbwd device viawd device wbwd # # Temperature sensors: # # coretemp: on-die sensor on Intel Core and newer CPUs # amdtemp: on-die sensor on AMD K8/K10/K11 CPUs # device coretemp device amdtemp # # CPU control pseudo-device. Provides access to MSRs, CPUID info and # microcode update feature. # device cpuctl # # System Management Bus (SMB) # options ENABLE_ALART # Control alarm on Intel intpm driver # # Number of initial kernel page table pages used for early bootstrap. # This number should include enough pages to map the kernel and any # modules or other data loaded with the kernel by the loader. Each # page table page maps 2MB. # options NKPT=31 # EFI Runtime Services support (not functional yet). options EFIRT ##################################################################### # ABI Emulation #XXX keep these here for now and reactivate when support for emulating #XXX these 32 bit binaries is added. # Enable 32-bit runtime support for FreeBSD/i386 binaries. options COMPAT_FREEBSD32 # Enable iBCS2 runtime support for SCO and ISC binaries #XXX#options IBCS2 # Emulate spx device for client side of SVR3 local X interface #XXX#options SPX_HACK # Enable 32-bit runtime support for CloudABI binaries. options COMPAT_CLOUDABI32 # Enable 64-bit runtime support for CloudABI binaries. options COMPAT_CLOUDABI64 # Enable Linux ABI emulation #XXX#options COMPAT_LINUX # Enable 32-bit Linux ABI emulation (requires COMPAT_43 and COMPAT_FREEBSD32) options COMPAT_LINUX32 # Enable the linux-like proc filesystem support (requires COMPAT_LINUX32 # and PSEUDOFS) options LINPROCFS #Enable the linux-like sys filesystem support (requires COMPAT_LINUX32 # and PSEUDOFS) options LINSYSFS # # SysVR4 ABI emulation # # The svr4 ABI emulator can be statically compiled into the kernel or loaded as # a KLD module. # The STREAMS network emulation code can also be compiled statically or as a # module. If loaded as a module, it must be loaded before the svr4 module # (the /usr/sbin/svr4 script does this for you). If compiling statically, # the `streams' device must be configured into any kernel which also # specifies COMPAT_SVR4. It is possible to have a statically-configured # STREAMS device and a dynamically loadable svr4 emulator; the /usr/sbin/svr4 # script understands that it doesn't need to load the `streams' module under # those circumstances. # Caveat: At this time, `options KTRACE' is required for the svr4 emulator # (whether static or dynamic). # #XXX#options COMPAT_SVR4 # build emulator statically #XXX#options DEBUG_SVR4 # enable verbose debugging #XXX#device streams # STREAMS network driver (required for svr4). ##################################################################### # VM OPTIONS # KSTACK_PAGES is the number of memory pages to assign to the kernel # stack of each thread. options KSTACK_PAGES=5 # Enable detailed accounting by the PV entry allocator. options PV_STATS ##################################################################### # More undocumented options for linting. # Note that documenting these are not considered an affront. options FB_INSTALL_CDEV # install a CDEV entry in /dev options KBDIO_DEBUG=2 options KBD_MAXRETRY=4 options KBD_MAXWAIT=6 options KBD_RESETDELAY=201 options PSM_DEBUG=1 options TIMER_FREQ=((14318182+6)/12) options VM_KMEM_SIZE options VM_KMEM_SIZE_MAX options VM_KMEM_SIZE_SCALE # Enable NDIS binary driver support options NDISAPI device ndis Index: stable/11/sys/conf/files =================================================================== --- stable/11/sys/conf/files (revision 329158) +++ stable/11/sys/conf/files (revision 329159) @@ -1,4676 +1,4635 @@ # $FreeBSD$ # # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and # dependency lines other than the first are silently ignored. # acpi_quirks.h optional acpi \ dependency "$S/tools/acpi_quirks2h.awk $S/dev/acpica/acpi_quirks" \ compile-with "${AWK} -f $S/tools/acpi_quirks2h.awk $S/dev/acpica/acpi_quirks" \ no-obj no-implicit-rule before-depend \ clean "acpi_quirks.h" bhnd_nvram_map.h optional bhnd \ dependency "$S/dev/bhnd/tools/nvram_map_gen.sh $S/dev/bhnd/tools/nvram_map_gen.awk $S/dev/bhnd/nvram/nvram_map" \ compile-with "sh $S/dev/bhnd/tools/nvram_map_gen.sh $S/dev/bhnd/nvram/nvram_map -h" \ no-obj no-implicit-rule before-depend \ clean "bhnd_nvram_map.h" bhnd_nvram_map_data.h optional bhnd \ dependency "$S/dev/bhnd/tools/nvram_map_gen.sh $S/dev/bhnd/tools/nvram_map_gen.awk $S/dev/bhnd/nvram/nvram_map" \ compile-with "sh $S/dev/bhnd/tools/nvram_map_gen.sh $S/dev/bhnd/nvram/nvram_map -d" \ no-obj no-implicit-rule before-depend \ clean "bhnd_nvram_map_data.h" # # The 'fdt_dtb_file' target covers an actual DTB file name, which is derived # from the specified source (DTS) file: .dts -> .dtb # fdt_dtb_file optional fdt fdt_dtb_static \ compile-with "sh -c 'MACHINE=${MACHINE} $S/tools/fdt/make_dtb.sh $S ${FDT_DTS_FILE} ${.CURDIR}'" \ no-obj no-implicit-rule before-depend \ clean "${FDT_DTS_FILE:R}.dtb" fdt_static_dtb.h optional fdt fdt_dtb_static \ compile-with "sh -c 'MACHINE=${MACHINE} $S/tools/fdt/make_dtbh.sh ${FDT_DTS_FILE} ${.CURDIR}'" \ dependency "fdt_dtb_file" \ no-obj no-implicit-rule before-depend \ clean "fdt_static_dtb.h" feeder_eq_gen.h optional sound \ dependency "$S/tools/sound/feeder_eq_mkfilter.awk" \ compile-with "${AWK} -f $S/tools/sound/feeder_eq_mkfilter.awk -- ${FEEDER_EQ_PRESETS} > feeder_eq_gen.h" \ no-obj no-implicit-rule before-depend \ clean "feeder_eq_gen.h" feeder_rate_gen.h optional sound \ dependency "$S/tools/sound/feeder_rate_mkfilter.awk" \ compile-with "${AWK} -f $S/tools/sound/feeder_rate_mkfilter.awk -- ${FEEDER_RATE_PRESETS} > feeder_rate_gen.h" \ no-obj no-implicit-rule before-depend \ clean "feeder_rate_gen.h" snd_fxdiv_gen.h optional sound \ dependency "$S/tools/sound/snd_fxdiv_gen.awk" \ compile-with "${AWK} -f $S/tools/sound/snd_fxdiv_gen.awk -- > snd_fxdiv_gen.h" \ no-obj no-implicit-rule before-depend \ clean "snd_fxdiv_gen.h" miidevs.h optional miibus | mii \ dependency "$S/tools/miidevs2h.awk $S/dev/mii/miidevs" \ compile-with "${AWK} -f $S/tools/miidevs2h.awk $S/dev/mii/miidevs" \ no-obj no-implicit-rule before-depend \ clean "miidevs.h" pccarddevs.h standard \ dependency "$S/tools/pccarddevs2h.awk $S/dev/pccard/pccarddevs" \ compile-with "${AWK} -f $S/tools/pccarddevs2h.awk $S/dev/pccard/pccarddevs" \ no-obj no-implicit-rule before-depend \ clean "pccarddevs.h" kbdmuxmap.h optional kbdmux_dflt_keymap \ compile-with "kbdcontrol -P ${S:S/sys$/share/}/vt/keymaps -P ${S:S/sys$/share/}/syscons/keymaps -L ${KBDMUX_DFLT_KEYMAP} | sed -e 's/^static keymap_t.* = /static keymap_t key_map = /' -e 's/^static accentmap_t.* = /static accentmap_t accent_map = /' > kbdmuxmap.h" \ no-obj no-implicit-rule before-depend \ clean "kbdmuxmap.h" teken_state.h optional sc | vt \ dependency "$S/teken/gensequences $S/teken/sequences" \ compile-with "${AWK} -f $S/teken/gensequences $S/teken/sequences > teken_state.h" \ no-obj no-implicit-rule before-depend \ clean "teken_state.h" usbdevs.h optional usb \ dependency "$S/tools/usbdevs2h.awk $S/dev/usb/usbdevs" \ compile-with "${AWK} -f $S/tools/usbdevs2h.awk $S/dev/usb/usbdevs -h" \ no-obj no-implicit-rule before-depend \ clean "usbdevs.h" usbdevs_data.h optional usb \ dependency "$S/tools/usbdevs2h.awk $S/dev/usb/usbdevs" \ compile-with "${AWK} -f $S/tools/usbdevs2h.awk $S/dev/usb/usbdevs -d" \ no-obj no-implicit-rule before-depend \ clean "usbdevs_data.h" cam/cam.c optional scbus cam/cam_compat.c optional scbus cam/cam_iosched.c optional scbus cam/cam_periph.c optional scbus cam/cam_queue.c optional scbus cam/cam_sim.c optional scbus cam/cam_xpt.c optional scbus cam/ata/ata_all.c optional scbus cam/ata/ata_xpt.c optional scbus cam/ata/ata_pmp.c optional scbus cam/nvme/nvme_all.c optional scbus nvme cam/nvme/nvme_da.c optional scbus nvme da cam/nvme/nvme_xpt.c optional scbus nvme cam/scsi/scsi_xpt.c optional scbus cam/scsi/scsi_all.c optional scbus cam/scsi/scsi_cd.c optional cd cam/scsi/scsi_ch.c optional ch cam/ata/ata_da.c optional ada | da cam/ctl/ctl.c optional ctl cam/ctl/ctl_backend.c optional ctl cam/ctl/ctl_backend_block.c optional ctl cam/ctl/ctl_backend_ramdisk.c optional ctl cam/ctl/ctl_cmd_table.c optional ctl cam/ctl/ctl_frontend.c optional ctl cam/ctl/ctl_frontend_cam_sim.c optional ctl cam/ctl/ctl_frontend_ioctl.c optional ctl cam/ctl/ctl_frontend_iscsi.c optional ctl cfiscsi cam/ctl/ctl_ha.c optional ctl cam/ctl/ctl_scsi_all.c optional ctl cam/ctl/ctl_tpc.c optional ctl cam/ctl/ctl_tpc_local.c optional ctl cam/ctl/ctl_error.c optional ctl cam/ctl/ctl_util.c optional ctl cam/ctl/scsi_ctl.c optional ctl cam/scsi/scsi_da.c optional da cam/scsi/scsi_low.c optional ct | ncv | nsp | stg cam/scsi/scsi_pass.c optional pass cam/scsi/scsi_pt.c optional pt cam/scsi/scsi_sa.c optional sa cam/scsi/scsi_enc.c optional ses cam/scsi/scsi_enc_ses.c optional ses cam/scsi/scsi_enc_safte.c optional ses cam/scsi/scsi_sg.c optional sg cam/scsi/scsi_targ_bh.c optional targbh cam/scsi/scsi_target.c optional targ cam/scsi/smp_all.c optional scbus # shared between zfs and dtrace cddl/compat/opensolaris/kern/opensolaris.c optional zfs | dtrace compile-with "${CDDL_C}" cddl/compat/opensolaris/kern/opensolaris_cmn_err.c optional zfs | dtrace compile-with "${CDDL_C}" cddl/compat/opensolaris/kern/opensolaris_kmem.c optional zfs | dtrace compile-with "${CDDL_C}" cddl/compat/opensolaris/kern/opensolaris_misc.c optional zfs | dtrace compile-with "${CDDL_C}" cddl/compat/opensolaris/kern/opensolaris_proc.c optional zfs | dtrace compile-with "${CDDL_C}" cddl/compat/opensolaris/kern/opensolaris_sunddi.c optional zfs | dtrace compile-with "${CDDL_C}" cddl/compat/opensolaris/kern/opensolaris_taskq.c optional zfs | dtrace compile-with "${CDDL_C}" # zfs specific cddl/compat/opensolaris/kern/opensolaris_acl.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_dtrace.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_kobj.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_kstat.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_lookup.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_policy.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_string.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_sysevent.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_uio.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_vfs.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_vm.c optional zfs compile-with "${ZFS_C}" cddl/compat/opensolaris/kern/opensolaris_zone.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/acl/acl_common.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/avl/avl.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/nvpair/opensolaris_fnvpair.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/nvpair/opensolaris_nvpair.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/nvpair/opensolaris_nvpair_alloc_fixed.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/unicode/u8_textprep.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zfeature_common.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zfs_comutil.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zfs_deleg.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zfs_fletcher.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zfs_ioctl_compat.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zfs_namecheck.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zfs_prop.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zpool_prop.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/common/zfs/zprop_common.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/vnode.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/abd.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/blkptr.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/bplist.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/bpobj.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/bptree.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/bqueue.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/ddt.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/ddt_zap.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_diff.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_object.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_send.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_traverse.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_zfetch.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c optional zfs compile-with "${ZFS_C}" \ warning "kernel contains CDDL licensed ZFS filesystem" cddl/contrib/opensolaris/uts/common/fs/zfs/dnode_sync.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_bookmark.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_deadlist.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_deleg.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_destroy.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_prop.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_userhold.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_synctask.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/gzip.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lz4.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lzjb.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/multilist.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/refcount.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/rrwlock.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/sa.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/sha256.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/skein_zfs.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/spa_config.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/spa_errlog.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/spa_history.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/space_map.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/space_reftree.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/trim_map.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/uberblock.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/unique.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_cache.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_file.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_geom.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_label.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_mirror.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_missing.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_queue.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_raidz.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_root.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zap.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zap_leaf.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zap_micro.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zcp.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zcp_get.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zcp_global.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zcp_iter.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zcp_synctask.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfeature.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_byteswap.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ctldir.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_debug.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_dir.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_fm.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_fuid.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_log.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_onexit.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_replay.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_rlock.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_sa.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zil.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zio_checksum.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zio_compress.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zio_inject.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zle.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zrlock.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/os/callb.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/os/fm.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/os/list.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/os/nvpair_alloc_system.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/adler32.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/deflate.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/inffast.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/inflate.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/inftrees.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/opensolaris_crc32.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/trees.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/zmod.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/zmod_subr.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/zmod/zutil.c optional zfs compile-with "${ZFS_C}" # zfs lua support cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lapi.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lauxlib.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lbaselib.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lbitlib.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lcode.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lcompat.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lcorolib.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lctype.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/ldebug.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/ldo.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/ldump.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lfunc.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lgc.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/llex.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lmem.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lobject.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lopcodes.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lparser.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lstate.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lstring.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lstrlib.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/ltable.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/ltablib.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/ltm.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lundump.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lvm.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lzio.c optional zfs compile-with "${ZFS_C}" # dtrace specific cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c optional dtrace compile-with "${DTRACE_C}" \ warning "kernel contains CDDL licensed DTRACE" cddl/dev/dtmalloc/dtmalloc.c optional dtmalloc | dtraceall compile-with "${CDDL_C}" cddl/dev/profile/profile.c optional dtrace_profile | dtraceall compile-with "${CDDL_C}" cddl/dev/sdt/sdt.c optional dtrace_sdt | dtraceall compile-with "${CDDL_C}" cddl/dev/fbt/fbt.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" cddl/dev/systrace/systrace.c optional dtrace_systrace | dtraceall compile-with "${CDDL_C}" cddl/dev/prototype.c optional dtrace_prototype | dtraceall compile-with "${CDDL_C}" fs/nfsclient/nfs_clkdtrace.c optional dtnfscl nfscl | dtraceall nfscl compile-with "${CDDL_C}" compat/cloudabi/cloudabi_clock.c optional compat_cloudabi32 | compat_cloudabi64 compat/cloudabi/cloudabi_errno.c optional compat_cloudabi32 | compat_cloudabi64 compat/cloudabi/cloudabi_fd.c optional compat_cloudabi32 | compat_cloudabi64 compat/cloudabi/cloudabi_file.c optional compat_cloudabi32 | compat_cloudabi64 compat/cloudabi/cloudabi_futex.c optional compat_cloudabi32 | compat_cloudabi64 compat/cloudabi/cloudabi_mem.c optional compat_cloudabi32 | compat_cloudabi64 compat/cloudabi/cloudabi_proc.c optional compat_cloudabi32 | compat_cloudabi64 compat/cloudabi/cloudabi_random.c optional compat_cloudabi32 | compat_cloudabi64 compat/cloudabi/cloudabi_sock.c optional compat_cloudabi32 | compat_cloudabi64 compat/cloudabi/cloudabi_thread.c optional compat_cloudabi32 | compat_cloudabi64 compat/cloudabi/cloudabi_vdso.c optional compat_cloudabi32 | compat_cloudabi64 compat/cloudabi32/cloudabi32_fd.c optional compat_cloudabi32 compat/cloudabi32/cloudabi32_module.c optional compat_cloudabi32 compat/cloudabi32/cloudabi32_poll.c optional compat_cloudabi32 compat/cloudabi32/cloudabi32_sock.c optional compat_cloudabi32 compat/cloudabi32/cloudabi32_syscalls.c optional compat_cloudabi32 compat/cloudabi32/cloudabi32_sysent.c optional compat_cloudabi32 compat/cloudabi32/cloudabi32_thread.c optional compat_cloudabi32 compat/cloudabi64/cloudabi64_fd.c optional compat_cloudabi64 compat/cloudabi64/cloudabi64_module.c optional compat_cloudabi64 compat/cloudabi64/cloudabi64_poll.c optional compat_cloudabi64 compat/cloudabi64/cloudabi64_sock.c optional compat_cloudabi64 compat/cloudabi64/cloudabi64_syscalls.c optional compat_cloudabi64 compat/cloudabi64/cloudabi64_sysent.c optional compat_cloudabi64 compat/cloudabi64/cloudabi64_thread.c optional compat_cloudabi64 compat/freebsd32/freebsd32_capability.c optional compat_freebsd32 compat/freebsd32/freebsd32_ioctl.c optional compat_freebsd32 compat/freebsd32/freebsd32_misc.c optional compat_freebsd32 compat/freebsd32/freebsd32_syscalls.c optional compat_freebsd32 compat/freebsd32/freebsd32_sysent.c optional compat_freebsd32 contrib/ck/src/ck_array.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include" contrib/ck/src/ck_barrier_centralized.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include" contrib/ck/src/ck_barrier_combining.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include" contrib/ck/src/ck_barrier_dissemination.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include" contrib/ck/src/ck_barrier_mcs.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include" contrib/ck/src/ck_barrier_tournament.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include" contrib/ck/src/ck_epoch.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include" contrib/ck/src/ck_hp.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include" contrib/ck/src/ck_hs.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include" contrib/ck/src/ck_ht.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include" contrib/ck/src/ck_rhs.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include" contrib/dev/acpica/common/ahids.c optional acpi acpi_debug contrib/dev/acpica/common/ahuuids.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbcmds.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbconvert.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbdisply.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbexec.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbhistry.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbinput.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbmethod.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbnames.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbobject.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbstats.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbtest.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbutils.c optional acpi acpi_debug contrib/dev/acpica/components/debugger/dbxface.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmbuffer.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmcstyle.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmdeferred.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmnames.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmopcode.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmresrc.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmresrcl.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmresrcl2.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmresrcs.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmutils.c optional acpi acpi_debug contrib/dev/acpica/components/disassembler/dmwalk.c optional acpi acpi_debug contrib/dev/acpica/components/dispatcher/dsargs.c optional acpi contrib/dev/acpica/components/dispatcher/dscontrol.c optional acpi contrib/dev/acpica/components/dispatcher/dsdebug.c optional acpi contrib/dev/acpica/components/dispatcher/dsfield.c optional acpi contrib/dev/acpica/components/dispatcher/dsinit.c optional acpi contrib/dev/acpica/components/dispatcher/dsmethod.c optional acpi contrib/dev/acpica/components/dispatcher/dsmthdat.c optional acpi contrib/dev/acpica/components/dispatcher/dsobject.c optional acpi contrib/dev/acpica/components/dispatcher/dsopcode.c optional acpi contrib/dev/acpica/components/dispatcher/dspkginit.c optional acpi contrib/dev/acpica/components/dispatcher/dsutils.c optional acpi contrib/dev/acpica/components/dispatcher/dswexec.c optional acpi contrib/dev/acpica/components/dispatcher/dswload.c optional acpi contrib/dev/acpica/components/dispatcher/dswload2.c optional acpi contrib/dev/acpica/components/dispatcher/dswscope.c optional acpi contrib/dev/acpica/components/dispatcher/dswstate.c optional acpi contrib/dev/acpica/components/events/evevent.c optional acpi contrib/dev/acpica/components/events/evglock.c optional acpi contrib/dev/acpica/components/events/evgpe.c optional acpi contrib/dev/acpica/components/events/evgpeblk.c optional acpi contrib/dev/acpica/components/events/evgpeinit.c optional acpi contrib/dev/acpica/components/events/evgpeutil.c optional acpi contrib/dev/acpica/components/events/evhandler.c optional acpi contrib/dev/acpica/components/events/evmisc.c optional acpi contrib/dev/acpica/components/events/evregion.c optional acpi contrib/dev/acpica/components/events/evrgnini.c optional acpi contrib/dev/acpica/components/events/evsci.c optional acpi contrib/dev/acpica/components/events/evxface.c optional acpi contrib/dev/acpica/components/events/evxfevnt.c optional acpi contrib/dev/acpica/components/events/evxfgpe.c optional acpi contrib/dev/acpica/components/events/evxfregn.c optional acpi contrib/dev/acpica/components/executer/exconcat.c optional acpi contrib/dev/acpica/components/executer/exconfig.c optional acpi contrib/dev/acpica/components/executer/exconvrt.c optional acpi contrib/dev/acpica/components/executer/excreate.c optional acpi contrib/dev/acpica/components/executer/exdebug.c optional acpi contrib/dev/acpica/components/executer/exdump.c optional acpi contrib/dev/acpica/components/executer/exfield.c optional acpi contrib/dev/acpica/components/executer/exfldio.c optional acpi contrib/dev/acpica/components/executer/exmisc.c optional acpi contrib/dev/acpica/components/executer/exmutex.c optional acpi contrib/dev/acpica/components/executer/exnames.c optional acpi contrib/dev/acpica/components/executer/exoparg1.c optional acpi contrib/dev/acpica/components/executer/exoparg2.c optional acpi contrib/dev/acpica/components/executer/exoparg3.c optional acpi contrib/dev/acpica/components/executer/exoparg6.c optional acpi contrib/dev/acpica/components/executer/exprep.c optional acpi contrib/dev/acpica/components/executer/exregion.c optional acpi contrib/dev/acpica/components/executer/exresnte.c optional acpi contrib/dev/acpica/components/executer/exresolv.c optional acpi contrib/dev/acpica/components/executer/exresop.c optional acpi contrib/dev/acpica/components/executer/exstore.c optional acpi contrib/dev/acpica/components/executer/exstoren.c optional acpi contrib/dev/acpica/components/executer/exstorob.c optional acpi contrib/dev/acpica/components/executer/exsystem.c optional acpi contrib/dev/acpica/components/executer/extrace.c optional acpi contrib/dev/acpica/components/executer/exutils.c optional acpi contrib/dev/acpica/components/hardware/hwacpi.c optional acpi contrib/dev/acpica/components/hardware/hwesleep.c optional acpi contrib/dev/acpica/components/hardware/hwgpe.c optional acpi contrib/dev/acpica/components/hardware/hwpci.c optional acpi contrib/dev/acpica/components/hardware/hwregs.c optional acpi contrib/dev/acpica/components/hardware/hwsleep.c optional acpi contrib/dev/acpica/components/hardware/hwtimer.c optional acpi contrib/dev/acpica/components/hardware/hwvalid.c optional acpi contrib/dev/acpica/components/hardware/hwxface.c optional acpi contrib/dev/acpica/components/hardware/hwxfsleep.c optional acpi contrib/dev/acpica/components/namespace/nsaccess.c optional acpi contrib/dev/acpica/components/namespace/nsalloc.c optional acpi contrib/dev/acpica/components/namespace/nsarguments.c optional acpi contrib/dev/acpica/components/namespace/nsconvert.c optional acpi contrib/dev/acpica/components/namespace/nsdump.c optional acpi contrib/dev/acpica/components/namespace/nseval.c optional acpi contrib/dev/acpica/components/namespace/nsinit.c optional acpi contrib/dev/acpica/components/namespace/nsload.c optional acpi contrib/dev/acpica/components/namespace/nsnames.c optional acpi contrib/dev/acpica/components/namespace/nsobject.c optional acpi contrib/dev/acpica/components/namespace/nsparse.c optional acpi contrib/dev/acpica/components/namespace/nspredef.c optional acpi contrib/dev/acpica/components/namespace/nsprepkg.c optional acpi contrib/dev/acpica/components/namespace/nsrepair.c optional acpi contrib/dev/acpica/components/namespace/nsrepair2.c optional acpi contrib/dev/acpica/components/namespace/nssearch.c optional acpi contrib/dev/acpica/components/namespace/nsutils.c optional acpi contrib/dev/acpica/components/namespace/nswalk.c optional acpi contrib/dev/acpica/components/namespace/nsxfeval.c optional acpi contrib/dev/acpica/components/namespace/nsxfname.c optional acpi contrib/dev/acpica/components/namespace/nsxfobj.c optional acpi contrib/dev/acpica/components/parser/psargs.c optional acpi contrib/dev/acpica/components/parser/psloop.c optional acpi contrib/dev/acpica/components/parser/psobject.c optional acpi contrib/dev/acpica/components/parser/psopcode.c optional acpi contrib/dev/acpica/components/parser/psopinfo.c optional acpi contrib/dev/acpica/components/parser/psparse.c optional acpi contrib/dev/acpica/components/parser/psscope.c optional acpi contrib/dev/acpica/components/parser/pstree.c optional acpi contrib/dev/acpica/components/parser/psutils.c optional acpi contrib/dev/acpica/components/parser/pswalk.c optional acpi contrib/dev/acpica/components/parser/psxface.c optional acpi contrib/dev/acpica/components/resources/rsaddr.c optional acpi contrib/dev/acpica/components/resources/rscalc.c optional acpi contrib/dev/acpica/components/resources/rscreate.c optional acpi contrib/dev/acpica/components/resources/rsdump.c optional acpi acpi_debug contrib/dev/acpica/components/resources/rsdumpinfo.c optional acpi contrib/dev/acpica/components/resources/rsinfo.c optional acpi contrib/dev/acpica/components/resources/rsio.c optional acpi contrib/dev/acpica/components/resources/rsirq.c optional acpi contrib/dev/acpica/components/resources/rslist.c optional acpi contrib/dev/acpica/components/resources/rsmemory.c optional acpi contrib/dev/acpica/components/resources/rsmisc.c optional acpi contrib/dev/acpica/components/resources/rsserial.c optional acpi contrib/dev/acpica/components/resources/rsutils.c optional acpi contrib/dev/acpica/components/resources/rsxface.c optional acpi contrib/dev/acpica/components/tables/tbdata.c optional acpi contrib/dev/acpica/components/tables/tbfadt.c optional acpi contrib/dev/acpica/components/tables/tbfind.c optional acpi contrib/dev/acpica/components/tables/tbinstal.c optional acpi contrib/dev/acpica/components/tables/tbprint.c optional acpi contrib/dev/acpica/components/tables/tbutils.c optional acpi contrib/dev/acpica/components/tables/tbxface.c optional acpi contrib/dev/acpica/components/tables/tbxfload.c optional acpi contrib/dev/acpica/components/tables/tbxfroot.c optional acpi contrib/dev/acpica/components/utilities/utaddress.c optional acpi contrib/dev/acpica/components/utilities/utalloc.c optional acpi contrib/dev/acpica/components/utilities/utascii.c optional acpi contrib/dev/acpica/components/utilities/utbuffer.c optional acpi contrib/dev/acpica/components/utilities/utcache.c optional acpi contrib/dev/acpica/components/utilities/utcopy.c optional acpi contrib/dev/acpica/components/utilities/utdebug.c optional acpi contrib/dev/acpica/components/utilities/utdecode.c optional acpi contrib/dev/acpica/components/utilities/utdelete.c optional acpi contrib/dev/acpica/components/utilities/uterror.c optional acpi contrib/dev/acpica/components/utilities/uteval.c optional acpi contrib/dev/acpica/components/utilities/utexcep.c optional acpi contrib/dev/acpica/components/utilities/utglobal.c optional acpi contrib/dev/acpica/components/utilities/uthex.c optional acpi contrib/dev/acpica/components/utilities/utids.c optional acpi contrib/dev/acpica/components/utilities/utinit.c optional acpi contrib/dev/acpica/components/utilities/utlock.c optional acpi contrib/dev/acpica/components/utilities/utmath.c optional acpi contrib/dev/acpica/components/utilities/utmisc.c optional acpi contrib/dev/acpica/components/utilities/utmutex.c optional acpi contrib/dev/acpica/components/utilities/utnonansi.c optional acpi contrib/dev/acpica/components/utilities/utobject.c optional acpi contrib/dev/acpica/components/utilities/utosi.c optional acpi contrib/dev/acpica/components/utilities/utownerid.c optional acpi contrib/dev/acpica/components/utilities/utpredef.c optional acpi contrib/dev/acpica/components/utilities/utresdecode.c optional acpi acpi_debug contrib/dev/acpica/components/utilities/utresrc.c optional acpi contrib/dev/acpica/components/utilities/utstate.c optional acpi contrib/dev/acpica/components/utilities/utstring.c optional acpi contrib/dev/acpica/components/utilities/utstrsuppt.c optional acpi contrib/dev/acpica/components/utilities/utstrtoul64.c optional acpi contrib/dev/acpica/components/utilities/utuuid.c optional acpi acpi_debug contrib/dev/acpica/components/utilities/utxface.c optional acpi contrib/dev/acpica/components/utilities/utxferror.c optional acpi contrib/dev/acpica/components/utilities/utxfinit.c optional acpi contrib/dev/acpica/os_specific/service_layers/osgendbg.c optional acpi acpi_debug contrib/ipfilter/netinet/fil.c optional ipfilter inet \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_auth.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_fil_freebsd.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_frag.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_log.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_nat.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_proxy.c optional ipfilter inet \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_state.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_lookup.c optional ipfilter inet \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN} -Wno-unused -Wno-error -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_pool.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_htable.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_sync.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/mlfk_ipl.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_nat6.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_rules.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_scan.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_dstlist.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/radix_ipf.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/libfdt/fdt.c optional fdt contrib/libfdt/fdt_ro.c optional fdt contrib/libfdt/fdt_rw.c optional fdt contrib/libfdt/fdt_strerror.c optional fdt contrib/libfdt/fdt_sw.c optional fdt contrib/libfdt/fdt_wip.c optional fdt contrib/libnv/dnvlist.c standard contrib/libnv/nvlist.c standard contrib/libnv/nvpair.c standard contrib/ngatm/netnatm/api/cc_conn.c optional ngatm_ccatm \ compile-with "${NORMAL_C_NOWERROR} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_data.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_dump.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_port.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_sig.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_user.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/unisap.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/misc/straddr.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/misc/unimsg_common.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/msg/traffic.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/msg/uni_ie.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/msg/uni_msg.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/saal/saal_sscfu.c optional ngatm_sscfu \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/saal/saal_sscop.c optional ngatm_sscop \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_call.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_coord.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_party.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_print.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_reset.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_uni.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_unimsgcpy.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_verify.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" crypto/blowfish/bf_ecb.c optional ipsec | ipsec_support crypto/blowfish/bf_skey.c optional crypto | ipsec | ipsec_support crypto/camellia/camellia.c optional crypto | ipsec | ipsec_support crypto/camellia/camellia-api.c optional crypto | ipsec | ipsec_support crypto/des/des_ecb.c optional crypto | ipsec | ipsec_support | netsmb crypto/des/des_setkey.c optional crypto | ipsec | ipsec_support | netsmb crypto/rc4/rc4.c optional netgraph_mppc_encryption | kgssapi crypto/rijndael/rijndael-alg-fst.c optional crypto | geom_bde | \ ipsec | ipsec_support | random !random_loadable | wlan_ccmp crypto/rijndael/rijndael-api-fst.c optional geom_bde | random !random_loadable crypto/rijndael/rijndael-api.c optional crypto | ipsec | ipsec_support | \ wlan_ccmp crypto/sha1.c optional carp | crypto | ipsec | \ ipsec_support | netgraph_mppc_encryption | sctp crypto/sha2/sha256c.c optional crypto | geom_bde | ipsec | \ ipsec_support | random !random_loadable | sctp | zfs crypto/sha2/sha512c.c optional crypto | geom_bde | ipsec | \ ipsec_support | zfs crypto/skein/skein.c optional crypto | zfs crypto/skein/skein_block.c optional crypto | zfs crypto/siphash/siphash.c optional inet | inet6 crypto/siphash/siphash_test.c optional inet | inet6 ddb/db_access.c optional ddb ddb/db_break.c optional ddb ddb/db_capture.c optional ddb ddb/db_command.c optional ddb ddb/db_examine.c optional ddb ddb/db_expr.c optional ddb ddb/db_input.c optional ddb ddb/db_lex.c optional ddb ddb/db_main.c optional ddb ddb/db_output.c optional ddb ddb/db_print.c optional ddb ddb/db_ps.c optional ddb ddb/db_run.c optional ddb ddb/db_script.c optional ddb ddb/db_sym.c optional ddb ddb/db_thread.c optional ddb ddb/db_textdump.c optional ddb ddb/db_variables.c optional ddb ddb/db_watch.c optional ddb ddb/db_write_cmd.c optional ddb dev/aac/aac.c optional aac dev/aac/aac_cam.c optional aacp aac dev/aac/aac_debug.c optional aac dev/aac/aac_disk.c optional aac dev/aac/aac_linux.c optional aac compat_linux dev/aac/aac_pci.c optional aac pci dev/aacraid/aacraid.c optional aacraid dev/aacraid/aacraid_cam.c optional aacraid scbus dev/aacraid/aacraid_debug.c optional aacraid dev/aacraid/aacraid_linux.c optional aacraid compat_linux dev/aacraid/aacraid_pci.c optional aacraid pci dev/acpi_support/acpi_wmi.c optional acpi_wmi acpi dev/acpi_support/acpi_asus.c optional acpi_asus acpi dev/acpi_support/acpi_asus_wmi.c optional acpi_asus_wmi acpi dev/acpi_support/acpi_fujitsu.c optional acpi_fujitsu acpi dev/acpi_support/acpi_hp.c optional acpi_hp acpi dev/acpi_support/acpi_ibm.c optional acpi_ibm acpi dev/acpi_support/acpi_panasonic.c optional acpi_panasonic acpi dev/acpi_support/acpi_sony.c optional acpi_sony acpi dev/acpi_support/acpi_toshiba.c optional acpi_toshiba acpi dev/acpi_support/atk0110.c optional aibs acpi dev/acpica/Osd/OsdDebug.c optional acpi dev/acpica/Osd/OsdHardware.c optional acpi dev/acpica/Osd/OsdInterrupt.c optional acpi dev/acpica/Osd/OsdMemory.c optional acpi dev/acpica/Osd/OsdSchedule.c optional acpi dev/acpica/Osd/OsdStream.c optional acpi dev/acpica/Osd/OsdSynch.c optional acpi dev/acpica/Osd/OsdTable.c optional acpi dev/acpica/acpi.c optional acpi dev/acpica/acpi_acad.c optional acpi dev/acpica/acpi_battery.c optional acpi dev/acpica/acpi_button.c optional acpi dev/acpica/acpi_cmbat.c optional acpi dev/acpica/acpi_cpu.c optional acpi dev/acpica/acpi_ec.c optional acpi dev/acpica/acpi_isab.c optional acpi isa dev/acpica/acpi_lid.c optional acpi dev/acpica/acpi_package.c optional acpi dev/acpica/acpi_pci.c optional acpi pci dev/acpica/acpi_pci_link.c optional acpi pci dev/acpica/acpi_pcib.c optional acpi pci dev/acpica/acpi_pcib_acpi.c optional acpi pci dev/acpica/acpi_pcib_pci.c optional acpi pci dev/acpica/acpi_perf.c optional acpi dev/acpica/acpi_powerres.c optional acpi dev/acpica/acpi_quirk.c optional acpi dev/acpica/acpi_resource.c optional acpi dev/acpica/acpi_container.c optional acpi dev/acpica/acpi_smbat.c optional acpi dev/acpica/acpi_thermal.c optional acpi dev/acpica/acpi_throttle.c optional acpi dev/acpica/acpi_timer.c optional acpi dev/acpica/acpi_video.c optional acpi_video acpi dev/acpica/acpi_dock.c optional acpi_dock acpi dev/adlink/adlink.c optional adlink dev/advansys/adv_eisa.c optional adv eisa dev/advansys/adv_pci.c optional adv pci dev/advansys/advansys.c optional adv dev/advansys/advlib.c optional adv dev/advansys/advmcode.c optional adv dev/advansys/adw_pci.c optional adw pci dev/advansys/adwcam.c optional adw dev/advansys/adwlib.c optional adw dev/advansys/adwmcode.c optional adw dev/ae/if_ae.c optional ae pci dev/age/if_age.c optional age pci dev/agp/agp.c optional agp pci dev/agp/agp_if.m optional agp pci dev/aha/aha.c optional aha dev/aha/aha_isa.c optional aha isa dev/aha/aha_mca.c optional aha mca dev/ahb/ahb.c optional ahb eisa dev/ahci/ahci.c optional ahci dev/ahci/ahciem.c optional ahci dev/ahci/ahci_pci.c optional ahci pci dev/aic/aic.c optional aic dev/aic/aic_pccard.c optional aic pccard dev/aic7xxx/ahc_eisa.c optional ahc eisa dev/aic7xxx/ahc_isa.c optional ahc isa dev/aic7xxx/ahc_pci.c optional ahc pci \ compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}" dev/aic7xxx/ahd_pci.c optional ahd pci \ compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}" dev/aic7xxx/aic7770.c optional ahc dev/aic7xxx/aic79xx.c optional ahd pci dev/aic7xxx/aic79xx_osm.c optional ahd pci dev/aic7xxx/aic79xx_pci.c optional ahd pci dev/aic7xxx/aic79xx_reg_print.c optional ahd pci ahd_reg_pretty_print dev/aic7xxx/aic7xxx.c optional ahc dev/aic7xxx/aic7xxx_93cx6.c optional ahc dev/aic7xxx/aic7xxx_osm.c optional ahc dev/aic7xxx/aic7xxx_pci.c optional ahc pci dev/aic7xxx/aic7xxx_reg_print.c optional ahc ahc_reg_pretty_print dev/alc/if_alc.c optional alc pci dev/ale/if_ale.c optional ale pci dev/alpm/alpm.c optional alpm pci dev/altera/avgen/altera_avgen.c optional altera_avgen dev/altera/avgen/altera_avgen_fdt.c optional altera_avgen fdt dev/altera/avgen/altera_avgen_nexus.c optional altera_avgen dev/altera/sdcard/altera_sdcard.c optional altera_sdcard dev/altera/sdcard/altera_sdcard_disk.c optional altera_sdcard dev/altera/sdcard/altera_sdcard_io.c optional altera_sdcard dev/altera/sdcard/altera_sdcard_fdt.c optional altera_sdcard fdt dev/altera/sdcard/altera_sdcard_nexus.c optional altera_sdcard dev/altera/pio/pio.c optional altera_pio dev/altera/pio/pio_if.m optional altera_pio dev/amdpm/amdpm.c optional amdpm pci | nfpm pci dev/amdsmb/amdsmb.c optional amdsmb pci dev/amr/amr.c optional amr dev/amr/amr_cam.c optional amrp amr dev/amr/amr_disk.c optional amr dev/amr/amr_linux.c optional amr compat_linux dev/amr/amr_pci.c optional amr pci dev/an/if_an.c optional an dev/an/if_an_isa.c optional an isa dev/an/if_an_pccard.c optional an pccard dev/an/if_an_pci.c optional an pci # dev/ata/ata_if.m optional ata | atacore dev/ata/ata-all.c optional ata | atacore dev/ata/ata-dma.c optional ata | atacore dev/ata/ata-lowlevel.c optional ata | atacore dev/ata/ata-sata.c optional ata | atacore dev/ata/ata-card.c optional ata pccard | atapccard dev/ata/ata-cbus.c optional ata pc98 | atapc98 dev/ata/ata-isa.c optional ata isa | ataisa dev/ata/ata-pci.c optional ata pci | atapci dev/ata/chipsets/ata-acard.c optional ata pci | ataacard dev/ata/chipsets/ata-acerlabs.c optional ata pci | ataacerlabs dev/ata/chipsets/ata-amd.c optional ata pci | ataamd dev/ata/chipsets/ata-ati.c optional ata pci | ataati dev/ata/chipsets/ata-cenatek.c optional ata pci | atacenatek dev/ata/chipsets/ata-cypress.c optional ata pci | atacypress dev/ata/chipsets/ata-cyrix.c optional ata pci | atacyrix dev/ata/chipsets/ata-highpoint.c optional ata pci | atahighpoint dev/ata/chipsets/ata-intel.c optional ata pci | ataintel dev/ata/chipsets/ata-ite.c optional ata pci | ataite dev/ata/chipsets/ata-jmicron.c optional ata pci | atajmicron dev/ata/chipsets/ata-marvell.c optional ata pci | atamarvell dev/ata/chipsets/ata-micron.c optional ata pci | atamicron dev/ata/chipsets/ata-national.c optional ata pci | atanational dev/ata/chipsets/ata-netcell.c optional ata pci | atanetcell dev/ata/chipsets/ata-nvidia.c optional ata pci | atanvidia dev/ata/chipsets/ata-promise.c optional ata pci | atapromise dev/ata/chipsets/ata-serverworks.c optional ata pci | ataserverworks dev/ata/chipsets/ata-siliconimage.c optional ata pci | atasiliconimage | ataati dev/ata/chipsets/ata-sis.c optional ata pci | atasis dev/ata/chipsets/ata-via.c optional ata pci | atavia # dev/ath/if_ath_pci.c optional ath_pci pci \ compile-with "${NORMAL_C} -I$S/dev/ath" # dev/ath/if_ath_ahb.c optional ath_ahb \ compile-with "${NORMAL_C} -I$S/dev/ath" # dev/ath/if_ath.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_alq.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_beacon.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_btcoex.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_btcoex_mci.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_debug.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_descdma.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_keycache.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_ioctl.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_led.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_lna_div.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_tx.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_tx_edma.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_tx_ht.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_tdma.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_sysctl.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_rx.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_rx_edma.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_spectral.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ah_osdep.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" # dev/ath/ath_hal/ah.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_hal/ah_eeprom_v1.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_hal/ah_eeprom_v3.c optional ath_hal | ath_ar5211 | ath_ar5212 \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_hal/ah_eeprom_v14.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_hal/ah_eeprom_v4k.c \ optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_hal/ah_eeprom_9287.c \ optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_hal/ah_regdomain.c optional ath \ compile-with "${NORMAL_C} ${NO_WSHIFT_COUNT_NEGATIVE} ${NO_WSHIFT_COUNT_OVERFLOW} -I$S/dev/ath" # ar5210 dev/ath/ath_hal/ar5210/ar5210_attach.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_beacon.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_interrupts.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_keycache.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_misc.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_phy.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_power.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_recv.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_reset.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_xmit.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar5211 dev/ath/ath_hal/ar5211/ar5211_attach.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_beacon.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_interrupts.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_keycache.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_misc.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_phy.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_power.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_recv.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_reset.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_xmit.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar5212 dev/ath/ath_hal/ar5212/ar5212_ani.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_attach.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_beacon.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_eeprom.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_gpio.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_interrupts.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_keycache.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_misc.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_phy.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_power.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_recv.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_reset.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_rfgain.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_xmit.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar5416 (depends on ar5212) dev/ath/ath_hal/ar5416/ar5416_ani.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_attach.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_beacon.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_btcoex.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_cal.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_cal_iq.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_cal_adcgain.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_cal_adcdc.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_eeprom.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_gpio.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_interrupts.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_keycache.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_misc.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_phy.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_power.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_radar.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_recv.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_reset.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_spectral.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_xmit.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9130 (depends upon ar5416) - also requires AH_SUPPORT_AR9130 # # Since this is an embedded MAC SoC, there's no need to compile it into the # default HAL. dev/ath/ath_hal/ar9001/ar9130_attach.c optional ath_ar9130 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9001/ar9130_phy.c optional ath_ar9130 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9001/ar9130_eeprom.c optional ath_ar9130 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9160 (depends on ar5416) dev/ath/ath_hal/ar9001/ar9160_attach.c optional ath_hal | ath_ar9160 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9280 (depends on ar5416) dev/ath/ath_hal/ar9002/ar9280_attach.c optional ath_hal | ath_ar9280 | \ ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9280_olc.c optional ath_hal | ath_ar9280 | \ ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9285 (depends on ar5416 and ar9280) dev/ath/ath_hal/ar9002/ar9285_attach.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285_btcoex.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285_reset.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285_cal.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285_phy.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285_diversity.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9287 (depends on ar5416) dev/ath/ath_hal/ar9002/ar9287_attach.c optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9287_reset.c optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9287_cal.c optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9287_olc.c optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9300 contrib/dev/ath/ath_hal/ar9300/ar9300_ani.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_attach.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_beacon.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_eeprom.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal ${NO_WCONSTANT_CONVERSION}" contrib/dev/ath/ath_hal/ar9300/ar9300_freebsd.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_gpio.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_interrupts.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_keycache.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_mci.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_misc.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_paprd.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_phy.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_power.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_radar.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_radio.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_recv.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_recv_ds.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_reset.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal ${NO_WSOMETIMES_UNINITIALIZED} -Wno-unused-function" contrib/dev/ath/ath_hal/ar9300/ar9300_stub.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_stub_funcs.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_spectral.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_timer.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_xmit.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" contrib/dev/ath/ath_hal/ar9300/ar9300_xmit_ds.c optional ath_hal | ath_ar9300 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal" # rf backends dev/ath/ath_hal/ar5212/ar2316.c optional ath_rf2316 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar2317.c optional ath_rf2317 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar2413.c optional ath_hal | ath_rf2413 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar2425.c optional ath_hal | ath_rf2425 | ath_rf2417 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5111.c optional ath_hal | ath_rf5111 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5112.c optional ath_hal | ath_rf5112 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5413.c optional ath_hal | ath_rf5413 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar2133.c optional ath_hal | ath_ar5416 | \ ath_ar9130 | ath_ar9160 | ath_ar9280 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9280.c optional ath_hal | ath_ar9280 | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9287.c optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ath rate control algorithms dev/ath/ath_rate/amrr/amrr.c optional ath_rate_amrr \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_rate/onoe/onoe.c optional ath_rate_onoe \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_rate/sample/sample.c optional ath_rate_sample \ compile-with "${NORMAL_C} -I$S/dev/ath" # ath DFS modules dev/ath/ath_dfs/null/dfs_null.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" # dev/bce/if_bce.c optional bce dev/bfe/if_bfe.c optional bfe dev/bge/if_bge.c optional bge dev/bhnd/bhnd.c optional bhnd dev/bhnd/bhnd_nexus.c optional bhnd siba_nexus | \ bhnd bcma_nexus dev/bhnd/bhnd_subr.c optional bhnd dev/bhnd/bhnd_bus_if.m optional bhnd dev/bhnd/bhndb/bhnd_bhndb.c optional bhndb bhnd dev/bhnd/bhndb/bhndb.c optional bhndb bhnd dev/bhnd/bhndb/bhndb_bus_if.m optional bhndb bhnd dev/bhnd/bhndb/bhndb_hwdata.c optional bhndb bhnd dev/bhnd/bhndb/bhndb_if.m optional bhndb bhnd dev/bhnd/bhndb/bhndb_pci.c optional bhndb bhnd pci dev/bhnd/bhndb/bhndb_pci_hwdata.c optional bhndb bhnd pci dev/bhnd/bhndb/bhndb_pci_sprom.c optional bhndb bhnd pci dev/bhnd/bhndb/bhndb_subr.c optional bhndb bhnd dev/bhnd/bcma/bcma.c optional bcma bhnd dev/bhnd/bcma/bcma_bhndb.c optional bcma bhnd bhndb dev/bhnd/bcma/bcma_erom.c optional bcma bhnd dev/bhnd/bcma/bcma_nexus.c optional bcma_nexus bcma bhnd dev/bhnd/bcma/bcma_subr.c optional bcma bhnd dev/bhnd/cores/chipc/chipc.c optional bhnd dev/bhnd/cores/chipc/chipc_cfi.c optional bhnd cfi dev/bhnd/cores/chipc/chipc_slicer.c optional bhnd cfi | bhnd spibus dev/bhnd/cores/chipc/chipc_spi.c optional bhnd spibus dev/bhnd/cores/chipc/chipc_subr.c optional bhnd dev/bhnd/cores/chipc/bhnd_chipc_if.m optional bhnd dev/bhnd/cores/chipc/bhnd_sprom_chipc.c optional bhnd dev/bhnd/cores/pci/bhnd_pci.c optional bhnd pci dev/bhnd/cores/pci/bhnd_pci_hostb.c optional bhndb bhnd pci dev/bhnd/cores/pci/bhnd_pcib.c optional bhnd_pcib bhnd pci dev/bhnd/cores/pcie2/bhnd_pcie2.c optional bhnd pci dev/bhnd/cores/pcie2/bhnd_pcie2_hostb.c optional bhndb bhnd pci dev/bhnd/cores/pcie2/bhnd_pcie2b.c optional bhnd_pcie2b bhnd pci dev/bhnd/nvram/bhnd_nvram_if.m optional bhnd dev/bhnd/nvram/bhnd_sprom.c optional bhnd dev/bhnd/nvram/bhnd_sprom_subr.c optional bhnd dev/bhnd/nvram/nvram_subr.c optional bhnd dev/bhnd/siba/siba.c optional siba bhnd dev/bhnd/siba/siba_bhndb.c optional siba bhnd bhndb dev/bhnd/siba/siba_nexus.c optional siba_nexus siba bhnd dev/bhnd/siba/siba_subr.c optional siba bhnd # dev/bktr/bktr_audio.c optional bktr pci dev/bktr/bktr_card.c optional bktr pci dev/bktr/bktr_core.c optional bktr pci dev/bktr/bktr_i2c.c optional bktr pci smbus dev/bktr/bktr_os.c optional bktr pci dev/bktr/bktr_tuner.c optional bktr pci dev/bktr/msp34xx.c optional bktr pci dev/bnxt/bnxt_hwrm.c optional bnxt iflib pci dev/bnxt/bnxt_sysctl.c optional bnxt iflib pci dev/bnxt/bnxt_txrx.c optional bnxt iflib pci dev/bnxt/if_bnxt.c optional bnxt iflib pci dev/buslogic/bt.c optional bt dev/buslogic/bt_eisa.c optional bt eisa dev/buslogic/bt_isa.c optional bt isa dev/buslogic/bt_mca.c optional bt mca dev/buslogic/bt_pci.c optional bt pci dev/bwi/bwimac.c optional bwi dev/bwi/bwiphy.c optional bwi dev/bwi/bwirf.c optional bwi dev/bwi/if_bwi.c optional bwi dev/bwi/if_bwi_pci.c optional bwi pci # XXX Work around clang warnings, until maintainer approves fix. dev/bwn/if_bwn.c optional bwn siba_bwn \ compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}" dev/bwn/if_bwn_pci.c optional bwn pci bhnd dev/bwn/if_bwn_phy_common.c optional bwn siba_bwn dev/bwn/if_bwn_phy_g.c optional bwn siba_bwn \ compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED} ${NO_WCONSTANT_CONVERSION}" dev/bwn/if_bwn_phy_lp.c optional bwn siba_bwn \ compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}" dev/bwn/if_bwn_phy_n.c optional bwn siba_bwn dev/bwn/if_bwn_util.c optional bwn siba_bwn dev/bwn/bwn_mac.c optional bwn bhnd dev/cardbus/cardbus.c optional cardbus dev/cardbus/cardbus_cis.c optional cardbus dev/cardbus/cardbus_device.c optional cardbus dev/cas/if_cas.c optional cas dev/cfi/cfi_bus_fdt.c optional cfi fdt dev/cfi/cfi_bus_nexus.c optional cfi dev/cfi/cfi_core.c optional cfi dev/cfi/cfi_dev.c optional cfi dev/cfi/cfi_disk.c optional cfid dev/chromebook_platform/chromebook_platform.c optional chromebook_platform dev/ciss/ciss.c optional ciss dev/cm/smc90cx6.c optional cm dev/cmx/cmx.c optional cmx dev/cmx/cmx_pccard.c optional cmx pccard dev/cpufreq/ichss.c optional cpufreq dev/cs/if_cs.c optional cs dev/cs/if_cs_isa.c optional cs isa dev/cs/if_cs_pccard.c optional cs pccard dev/cxgb/cxgb_main.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/cxgb_sge.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_mc5.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_vsc7323.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_vsc8211.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_ael1002.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_aq100x.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_mv88e1xxx.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_xgmac.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_t3_hw.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_tn1010.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/sys/uipc_mvec.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/cxgb_t3fw.c optional cxgb cxgb_t3fw \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgbe/t4_if.m optional cxgbe pci dev/cxgbe/t4_iov.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/t4_mp_ring.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/t4_main.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/t4_netmap.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/t4_sched.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/t4_sge.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/t4_l2t.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/t4_tracer.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/t4_vf.c optional cxgbev pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/common/t4_hw.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/common/t4vf_hw.c optional cxgbev pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" t4fw_cfg.c optional cxgbe \ compile-with "${AWK} -f $S/tools/fw_stub.awk t4fw_cfg.fw:t4fw_cfg t4fw_cfg_uwire.fw:t4fw_cfg_uwire t4fw.fw:t4fw -mt4fw_cfg -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "t4fw_cfg.c" t4fw_cfg.fwo optional cxgbe \ dependency "t4fw_cfg.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t4fw_cfg.fwo" t4fw_cfg.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t4fw_cfg.txt" \ compile-with "${CP} ${.ALLSRC} ${.TARGET}" \ no-obj no-implicit-rule \ clean "t4fw_cfg.fw" t4fw_cfg_uwire.fwo optional cxgbe \ dependency "t4fw_cfg_uwire.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t4fw_cfg_uwire.fwo" t4fw_cfg_uwire.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t4fw_cfg_uwire.txt" \ compile-with "${CP} ${.ALLSRC} ${.TARGET}" \ no-obj no-implicit-rule \ clean "t4fw_cfg_uwire.fw" t4fw.fwo optional cxgbe \ dependency "t4fw.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t4fw.fwo" t4fw.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t4fw-1.16.45.0.bin.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "t4fw.fw" t5fw_cfg.c optional cxgbe \ compile-with "${AWK} -f $S/tools/fw_stub.awk t5fw_cfg.fw:t5fw_cfg t5fw.fw:t5fw -mt5fw_cfg -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "t5fw_cfg.c" t5fw_cfg.fwo optional cxgbe \ dependency "t5fw_cfg.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t5fw_cfg.fwo" t5fw_cfg.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t5fw_cfg.txt" \ compile-with "${CP} ${.ALLSRC} ${.TARGET}" \ no-obj no-implicit-rule \ clean "t5fw_cfg.fw" t5fw.fwo optional cxgbe \ dependency "t5fw.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t5fw.fwo" t5fw.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t5fw-1.16.45.0.bin.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "t5fw.fw" t6fw_cfg.c optional cxgbe \ compile-with "${AWK} -f $S/tools/fw_stub.awk t6fw_cfg.fw:t6fw_cfg t6fw.fw:t6fw -mt6fw_cfg -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "t6fw_cfg.c" t6fw_cfg.fwo optional cxgbe \ dependency "t6fw_cfg.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t6fw_cfg.fwo" t6fw_cfg.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t6fw_cfg.txt" \ compile-with "${CP} ${.ALLSRC} ${.TARGET}" \ no-obj no-implicit-rule \ clean "t6fw_cfg.fw" t6fw.fwo optional cxgbe \ dependency "t6fw.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t6fw.fwo" t6fw.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t6fw-1.16.45.0.bin.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "t6fw.fw" dev/cy/cy.c optional cy dev/cy/cy_isa.c optional cy isa dev/cy/cy_pci.c optional cy pci dev/cyapa/cyapa.c optional cyapa iicbus dev/dc/if_dc.c optional dc pci dev/dc/dcphy.c optional dc pci dev/dc/pnphy.c optional dc pci dev/dcons/dcons.c optional dcons dev/dcons/dcons_crom.c optional dcons_crom dev/dcons/dcons_os.c optional dcons dev/de/if_de.c optional de pci dev/digi/CX.c optional digi_CX dev/digi/CX_PCI.c optional digi_CX_PCI dev/digi/EPCX.c optional digi_EPCX dev/digi/EPCX_PCI.c optional digi_EPCX_PCI dev/digi/Xe.c optional digi_Xe dev/digi/Xem.c optional digi_Xem dev/digi/Xr.c optional digi_Xr dev/digi/digi.c optional digi dev/digi/digi_isa.c optional digi isa dev/digi/digi_pci.c optional digi pci dev/dpt/dpt_eisa.c optional dpt eisa dev/dpt/dpt_pci.c optional dpt pci dev/dpt/dpt_scsi.c optional dpt dev/drm/ati_pcigart.c optional drm dev/drm/drm_agpsupport.c optional drm dev/drm/drm_auth.c optional drm dev/drm/drm_bufs.c optional drm dev/drm/drm_context.c optional drm dev/drm/drm_dma.c optional drm dev/drm/drm_drawable.c optional drm dev/drm/drm_drv.c optional drm dev/drm/drm_fops.c optional drm dev/drm/drm_hashtab.c optional drm dev/drm/drm_ioctl.c optional drm dev/drm/drm_irq.c optional drm dev/drm/drm_lock.c optional drm dev/drm/drm_memory.c optional drm dev/drm/drm_mm.c optional drm dev/drm/drm_pci.c optional drm dev/drm/drm_scatter.c optional drm dev/drm/drm_sman.c optional drm dev/drm/drm_sysctl.c optional drm dev/drm/drm_vm.c optional drm dev/drm/i915_dma.c optional i915drm dev/drm/i915_drv.c optional i915drm dev/drm/i915_irq.c optional i915drm dev/drm/i915_mem.c optional i915drm dev/drm/i915_suspend.c optional i915drm dev/drm/mach64_dma.c optional mach64drm dev/drm/mach64_drv.c optional mach64drm dev/drm/mach64_irq.c optional mach64drm dev/drm/mach64_state.c optional mach64drm dev/drm/mga_dma.c optional mgadrm dev/drm/mga_drv.c optional mgadrm dev/drm/mga_irq.c optional mgadrm dev/drm/mga_state.c optional mgadrm dev/drm/mga_warp.c optional mgadrm dev/drm/r128_cce.c optional r128drm \ compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}" dev/drm/r128_drv.c optional r128drm dev/drm/r128_irq.c optional r128drm dev/drm/r128_state.c optional r128drm dev/drm/r300_cmdbuf.c optional radeondrm dev/drm/r600_blit.c optional radeondrm dev/drm/r600_cp.c optional radeondrm \ compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}" dev/drm/radeon_cp.c optional radeondrm \ compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}" dev/drm/radeon_cs.c optional radeondrm dev/drm/radeon_drv.c optional radeondrm dev/drm/radeon_irq.c optional radeondrm dev/drm/radeon_mem.c optional radeondrm dev/drm/radeon_state.c optional radeondrm dev/drm/savage_bci.c optional savagedrm dev/drm/savage_drv.c optional savagedrm dev/drm/savage_state.c optional savagedrm dev/drm/sis_drv.c optional sisdrm dev/drm/sis_ds.c optional sisdrm dev/drm/sis_mm.c optional sisdrm dev/drm/tdfx_drv.c optional tdfxdrm dev/drm/via_dma.c optional viadrm dev/drm/via_dmablit.c optional viadrm dev/drm/via_drv.c optional viadrm dev/drm/via_irq.c optional viadrm dev/drm/via_map.c optional viadrm dev/drm/via_mm.c optional viadrm dev/drm/via_verifier.c optional viadrm dev/drm/via_video.c optional viadrm dev/drm2/drm_agpsupport.c optional drm2 dev/drm2/drm_auth.c optional drm2 dev/drm2/drm_bufs.c optional drm2 dev/drm2/drm_buffer.c optional drm2 dev/drm2/drm_context.c optional drm2 dev/drm2/drm_crtc.c optional drm2 dev/drm2/drm_crtc_helper.c optional drm2 dev/drm2/drm_dma.c optional drm2 dev/drm2/drm_dp_helper.c optional drm2 dev/drm2/drm_dp_iic_helper.c optional drm2 dev/drm2/drm_drv.c optional drm2 dev/drm2/drm_edid.c optional drm2 dev/drm2/drm_fb_helper.c optional drm2 dev/drm2/drm_fops.c optional drm2 dev/drm2/drm_gem.c optional drm2 dev/drm2/drm_gem_names.c optional drm2 dev/drm2/drm_global.c optional drm2 dev/drm2/drm_hashtab.c optional drm2 dev/drm2/drm_ioctl.c optional drm2 dev/drm2/drm_irq.c optional drm2 dev/drm2/drm_linux_list_sort.c optional drm2 dev/drm2/drm_lock.c optional drm2 dev/drm2/drm_memory.c optional drm2 dev/drm2/drm_mm.c optional drm2 dev/drm2/drm_modes.c optional drm2 dev/drm2/drm_pci.c optional drm2 dev/drm2/drm_platform.c optional drm2 dev/drm2/drm_scatter.c optional drm2 dev/drm2/drm_stub.c optional drm2 dev/drm2/drm_sysctl.c optional drm2 dev/drm2/drm_vm.c optional drm2 dev/drm2/drm_os_freebsd.c optional drm2 dev/drm2/ttm/ttm_agp_backend.c optional drm2 dev/drm2/ttm/ttm_lock.c optional drm2 dev/drm2/ttm/ttm_object.c optional drm2 dev/drm2/ttm/ttm_tt.c optional drm2 dev/drm2/ttm/ttm_bo_util.c optional drm2 dev/drm2/ttm/ttm_bo.c optional drm2 dev/drm2/ttm/ttm_bo_manager.c optional drm2 dev/drm2/ttm/ttm_execbuf_util.c optional drm2 dev/drm2/ttm/ttm_memory.c optional drm2 dev/drm2/ttm/ttm_page_alloc.c optional drm2 dev/drm2/ttm/ttm_bo_vm.c optional drm2 dev/drm2/ati_pcigart.c optional drm2 agp pci dev/ed/if_ed.c optional ed dev/ed/if_ed_novell.c optional ed dev/ed/if_ed_rtl80x9.c optional ed dev/ed/if_ed_pccard.c optional ed pccard dev/ed/if_ed_pci.c optional ed pci dev/efidev/efidev.c optional efirt dev/eisa/eisa_if.m standard dev/eisa/eisaconf.c optional eisa dev/e1000/if_em.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/if_lem.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/if_igb.c optional igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_80003es2lan.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82540.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82541.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82542.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82543.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82571.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82575.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_ich8lan.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_i210.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_api.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_mac.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_manage.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_nvm.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_phy.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_vf.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_mbx.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_osdep.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/et/if_et.c optional et dev/ena/ena.c optional ena \ compile-with "${NORMAL_C} -I$S/contrib" dev/ena/ena_sysctl.c optional ena \ compile-with "${NORMAL_C} -I$S/contrib" contrib/ena-com/ena_com.c optional ena contrib/ena-com/ena_eth_com.c optional ena dev/en/if_en_pci.c optional en pci dev/en/midway.c optional en dev/ep/if_ep.c optional ep dev/ep/if_ep_eisa.c optional ep eisa dev/ep/if_ep_isa.c optional ep isa dev/ep/if_ep_mca.c optional ep mca dev/ep/if_ep_pccard.c optional ep pccard dev/esp/esp_pci.c optional esp pci dev/esp/ncr53c9x.c optional esp dev/etherswitch/arswitch/arswitch.c optional arswitch dev/etherswitch/arswitch/arswitch_reg.c optional arswitch dev/etherswitch/arswitch/arswitch_phy.c optional arswitch dev/etherswitch/arswitch/arswitch_8216.c optional arswitch dev/etherswitch/arswitch/arswitch_8226.c optional arswitch dev/etherswitch/arswitch/arswitch_8316.c optional arswitch dev/etherswitch/arswitch/arswitch_8327.c optional arswitch dev/etherswitch/arswitch/arswitch_7240.c optional arswitch dev/etherswitch/arswitch/arswitch_9340.c optional arswitch dev/etherswitch/arswitch/arswitch_vlans.c optional arswitch dev/etherswitch/etherswitch.c optional etherswitch dev/etherswitch/etherswitch_if.m optional etherswitch dev/etherswitch/ip17x/ip17x.c optional ip17x dev/etherswitch/ip17x/ip175c.c optional ip17x dev/etherswitch/ip17x/ip175d.c optional ip17x dev/etherswitch/ip17x/ip17x_phy.c optional ip17x dev/etherswitch/ip17x/ip17x_vlans.c optional ip17x dev/etherswitch/miiproxy.c optional miiproxy dev/etherswitch/rtl8366/rtl8366rb.c optional rtl8366rb dev/etherswitch/ukswitch/ukswitch.c optional ukswitch dev/evdev/cdev.c optional evdev dev/evdev/evdev.c optional evdev dev/evdev/evdev_mt.c optional evdev dev/evdev/evdev_utils.c optional evdev dev/evdev/uinput.c optional evdev uinput dev/ex/if_ex.c optional ex dev/ex/if_ex_isa.c optional ex isa dev/ex/if_ex_pccard.c optional ex pccard dev/exca/exca.c optional cbb dev/extres/clk/clk.c optional ext_resources clk dev/extres/clk/clkdev_if.m optional ext_resources clk dev/extres/clk/clknode_if.m optional ext_resources clk dev/extres/clk/clk_bus.c optional ext_resources clk fdt dev/extres/clk/clk_div.c optional ext_resources clk dev/extres/clk/clk_fixed.c optional ext_resources clk dev/extres/clk/clk_gate.c optional ext_resources clk dev/extres/clk/clk_mux.c optional ext_resources clk dev/extres/phy/phy.c optional ext_resources phy dev/extres/phy/phy_if.m optional ext_resources phy dev/extres/hwreset/hwreset.c optional ext_resources hwreset dev/extres/hwreset/hwreset_if.m optional ext_resources hwreset dev/extres/regulator/regdev_if.m optional ext_resources regulator dev/extres/regulator/regnode_if.m optional ext_resources regulator dev/extres/regulator/regulator.c optional ext_resources regulator dev/extres/regulator/regulator_bus.c optional ext_resources regulator fdt dev/extres/regulator/regulator_fixed.c optional ext_resources regulator dev/fatm/if_fatm.c optional fatm pci dev/fb/fbd.c optional fbd | vt dev/fb/fb_if.m standard dev/fb/splash.c optional sc splash dev/fdt/fdt_clock.c optional fdt fdt_clock dev/fdt/fdt_clock_if.m optional fdt fdt_clock dev/fdt/fdt_common.c optional fdt dev/fdt/fdt_pinctrl.c optional fdt fdt_pinctrl dev/fdt/fdt_pinctrl_if.m optional fdt fdt_pinctrl dev/fdt/fdt_slicer.c optional fdt cfi | fdt nand | fdt mx25l dev/fdt/fdt_static_dtb.S optional fdt fdt_dtb_static \ dependency "fdt_dtb_file" dev/fdt/simplebus.c optional fdt dev/fe/if_fe.c optional fe dev/fe/if_fe_pccard.c optional fe pccard dev/filemon/filemon.c optional filemon dev/firewire/firewire.c optional firewire dev/firewire/fwcrom.c optional firewire dev/firewire/fwdev.c optional firewire dev/firewire/fwdma.c optional firewire dev/firewire/fwmem.c optional firewire dev/firewire/fwohci.c optional firewire dev/firewire/fwohci_pci.c optional firewire pci dev/firewire/if_fwe.c optional fwe dev/firewire/if_fwip.c optional fwip dev/firewire/sbp.c optional sbp dev/firewire/sbp_targ.c optional sbp_targ dev/flash/at45d.c optional at45d dev/flash/mx25l.c optional mx25l dev/fxp/if_fxp.c optional fxp dev/fxp/inphy.c optional fxp dev/gem/if_gem.c optional gem dev/gem/if_gem_pci.c optional gem pci dev/gem/if_gem_sbus.c optional gem sbus dev/gpio/gpiobacklight.c optional gpiobacklight fdt dev/gpio/gpiokeys.c optional gpiokeys fdt dev/gpio/gpiokeys_codes.c optional gpiokeys fdt dev/gpio/gpiobus.c optional gpio \ dependency "gpiobus_if.h" dev/gpio/gpioc.c optional gpio \ dependency "gpio_if.h" dev/gpio/gpioiic.c optional gpioiic dev/gpio/gpioled.c optional gpioled !fdt dev/gpio/gpioled_fdt.c optional gpioled fdt dev/gpio/gpiospi.c optional gpiospi dev/gpio/gpio_if.m optional gpio dev/gpio/gpiobus_if.m optional gpio dev/gpio/gpiopps.c optional gpiopps dev/gpio/ofw_gpiobus.c optional fdt gpio dev/hatm/if_hatm.c optional hatm pci dev/hatm/if_hatm_intr.c optional hatm pci dev/hatm/if_hatm_ioctl.c optional hatm pci dev/hatm/if_hatm_rx.c optional hatm pci dev/hatm/if_hatm_tx.c optional hatm pci dev/hifn/hifn7751.c optional hifn dev/hme/if_hme.c optional hme dev/hme/if_hme_pci.c optional hme pci dev/hme/if_hme_sbus.c optional hme sbus dev/hptiop/hptiop.c optional hptiop scbus dev/hwpmc/hwpmc_logging.c optional hwpmc dev/hwpmc/hwpmc_mod.c optional hwpmc dev/hwpmc/hwpmc_soft.c optional hwpmc dev/ichiic/ig4_acpi.c optional ig4 acpi iicbus dev/ichiic/ig4_iic.c optional ig4 iicbus dev/ichiic/ig4_pci.c optional ig4 pci iicbus dev/ichsmb/ichsmb.c optional ichsmb dev/ichsmb/ichsmb_pci.c optional ichsmb pci dev/ida/ida.c optional ida dev/ida/ida_disk.c optional ida dev/ida/ida_eisa.c optional ida eisa dev/ida/ida_pci.c optional ida pci dev/ie/if_ie.c optional ie isa nowerror dev/ie/if_ie_isa.c optional ie isa dev/iicbus/ad7418.c optional ad7418 dev/iicbus/ds1307.c optional ds1307 dev/iicbus/ds13rtc.c optional ds13rtc | ds133x | ds1374 dev/iicbus/ds1672.c optional ds1672 dev/iicbus/ds3231.c optional ds3231 dev/iicbus/icee.c optional icee dev/iicbus/if_ic.c optional ic dev/iicbus/iic.c optional iic dev/iicbus/iic_recover_bus.c optional iicbus dev/iicbus/iicbb.c optional iicbb dev/iicbus/iicbb_if.m optional iicbb dev/iicbus/iicbus.c optional iicbus dev/iicbus/iicbus_if.m optional iicbus dev/iicbus/iiconf.c optional iicbus dev/iicbus/iicsmb.c optional iicsmb \ dependency "iicbus_if.h" dev/iicbus/iicoc.c optional iicoc dev/iicbus/isl12xx.c optional isl12xx dev/iicbus/lm75.c optional lm75 dev/iicbus/nxprtc.c optional nxprtc | pcf8563 dev/iicbus/ofw_iicbus.c optional fdt iicbus dev/iicbus/s35390a.c optional s35390a dev/iir/iir.c optional iir dev/iir/iir_ctrl.c optional iir dev/iir/iir_pci.c optional iir pci dev/intpm/intpm.c optional intpm pci # XXX Work around clang warning, until maintainer approves fix. dev/ips/ips.c optional ips \ compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}" dev/ips/ips_commands.c optional ips dev/ips/ips_disk.c optional ips dev/ips/ips_ioctl.c optional ips dev/ips/ips_pci.c optional ips pci dev/ipw/if_ipw.c optional ipw ipwbssfw.c optional ipwbssfw | ipwfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk ipw_bss.fw:ipw_bss:130 -lintel_ipw -mipw_bss -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "ipwbssfw.c" ipw_bss.fwo optional ipwbssfw | ipwfw \ dependency "ipw_bss.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "ipw_bss.fwo" ipw_bss.fw optional ipwbssfw | ipwfw \ dependency "$S/contrib/dev/ipw/ipw2100-1.3.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "ipw_bss.fw" ipwibssfw.c optional ipwibssfw | ipwfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk ipw_ibss.fw:ipw_ibss:130 -lintel_ipw -mipw_ibss -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "ipwibssfw.c" ipw_ibss.fwo optional ipwibssfw | ipwfw \ dependency "ipw_ibss.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "ipw_ibss.fwo" ipw_ibss.fw optional ipwibssfw | ipwfw \ dependency "$S/contrib/dev/ipw/ipw2100-1.3-i.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "ipw_ibss.fw" ipwmonitorfw.c optional ipwmonitorfw | ipwfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk ipw_monitor.fw:ipw_monitor:130 -lintel_ipw -mipw_monitor -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "ipwmonitorfw.c" ipw_monitor.fwo optional ipwmonitorfw | ipwfw \ dependency "ipw_monitor.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "ipw_monitor.fwo" ipw_monitor.fw optional ipwmonitorfw | ipwfw \ dependency "$S/contrib/dev/ipw/ipw2100-1.3-p.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "ipw_monitor.fw" dev/iscsi/icl.c optional iscsi dev/iscsi/icl_conn_if.m optional cfiscsi | iscsi dev/iscsi/icl_soft.c optional iscsi dev/iscsi/icl_soft_proxy.c optional iscsi dev/iscsi/iscsi.c optional iscsi scbus dev/iscsi_initiator/iscsi.c optional iscsi_initiator scbus dev/iscsi_initiator/iscsi_subr.c optional iscsi_initiator scbus dev/iscsi_initiator/isc_cam.c optional iscsi_initiator scbus dev/iscsi_initiator/isc_soc.c optional iscsi_initiator scbus dev/iscsi_initiator/isc_sm.c optional iscsi_initiator scbus dev/iscsi_initiator/isc_subr.c optional iscsi_initiator scbus dev/ismt/ismt.c optional ismt dev/isl/isl.c optional isl iicbus dev/isp/isp.c optional isp dev/isp/isp_freebsd.c optional isp dev/isp/isp_library.c optional isp dev/isp/isp_pci.c optional isp pci dev/isp/isp_sbus.c optional isp sbus dev/isp/isp_target.c optional isp dev/ispfw/ispfw.c optional ispfw dev/iwi/if_iwi.c optional iwi iwibssfw.c optional iwibssfw | iwifw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwi_bss.fw:iwi_bss:300 -lintel_iwi -miwi_bss -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwibssfw.c" iwi_bss.fwo optional iwibssfw | iwifw \ dependency "iwi_bss.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwi_bss.fwo" iwi_bss.fw optional iwibssfw | iwifw \ dependency "$S/contrib/dev/iwi/ipw2200-bss.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwi_bss.fw" iwiibssfw.c optional iwiibssfw | iwifw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwi_ibss.fw:iwi_ibss:300 -lintel_iwi -miwi_ibss -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwiibssfw.c" iwi_ibss.fwo optional iwiibssfw | iwifw \ dependency "iwi_ibss.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwi_ibss.fwo" iwi_ibss.fw optional iwiibssfw | iwifw \ dependency "$S/contrib/dev/iwi/ipw2200-ibss.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwi_ibss.fw" iwimonitorfw.c optional iwimonitorfw | iwifw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwi_monitor.fw:iwi_monitor:300 -lintel_iwi -miwi_monitor -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwimonitorfw.c" iwi_monitor.fwo optional iwimonitorfw | iwifw \ dependency "iwi_monitor.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwi_monitor.fwo" iwi_monitor.fw optional iwimonitorfw | iwifw \ dependency "$S/contrib/dev/iwi/ipw2200-sniffer.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwi_monitor.fw" dev/iwm/if_iwm.c optional iwm dev/iwm/if_iwm_binding.c optional iwm dev/iwm/if_iwm_led.c optional iwm dev/iwm/if_iwm_mac_ctxt.c optional iwm dev/iwm/if_iwm_pcie_trans.c optional iwm dev/iwm/if_iwm_phy_ctxt.c optional iwm dev/iwm/if_iwm_phy_db.c optional iwm dev/iwm/if_iwm_power.c optional iwm dev/iwm/if_iwm_scan.c optional iwm dev/iwm/if_iwm_time_event.c optional iwm dev/iwm/if_iwm_util.c optional iwm iwm3160fw.c optional iwm3160fw | iwmfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwm3160.fw:iwm3160fw -miwm3160fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwm3160fw.c" iwm3160fw.fwo optional iwm3160fw | iwmfw \ dependency "iwm3160.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwm3160fw.fwo" iwm3160.fw optional iwm3160fw | iwmfw \ dependency "$S/contrib/dev/iwm/iwm-3160-16.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwm3160.fw" iwm7260fw.c optional iwm7260fw | iwmfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwm7260.fw:iwm7260fw -miwm7260fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwm7260fw.c" iwm7260fw.fwo optional iwm7260fw | iwmfw \ dependency "iwm7260.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwm7260fw.fwo" iwm7260.fw optional iwm7260fw | iwmfw \ dependency "$S/contrib/dev/iwm/iwm-7260-16.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwm7260.fw" iwm7265fw.c optional iwm7265fw | iwmfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwm7265.fw:iwm7265fw -miwm7265fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwm7265fw.c" iwm7265fw.fwo optional iwm7265fw | iwmfw \ dependency "iwm7265.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwm7265fw.fwo" iwm7265.fw optional iwm7265fw | iwmfw \ dependency "$S/contrib/dev/iwm/iwm-7265-16.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwm7265.fw" iwm8000Cfw.c optional iwm8000Cfw | iwmfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwm8000C.fw:iwm8000Cfw -miwm8000Cfw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwm8000Cfw.c" iwm8000Cfw.fwo optional iwm8000Cfw | iwmfw \ dependency "iwm8000C.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwm8000Cfw.fwo" iwm8000C.fw optional iwm8000Cfw | iwmfw \ dependency "$S/contrib/dev/iwm/iwm-8000C-16.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwm8000C.fw" dev/iwn/if_iwn.c optional iwn iwn1000fw.c optional iwn1000fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn1000.fw:iwn1000fw -miwn1000fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn1000fw.c" iwn1000fw.fwo optional iwn1000fw | iwnfw \ dependency "iwn1000.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn1000fw.fwo" iwn1000.fw optional iwn1000fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-1000-39.31.5.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn1000.fw" iwn100fw.c optional iwn100fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn100.fw:iwn100fw -miwn100fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn100fw.c" iwn100fw.fwo optional iwn100fw | iwnfw \ dependency "iwn100.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn100fw.fwo" iwn100.fw optional iwn100fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-100-39.31.5.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn100.fw" iwn105fw.c optional iwn105fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn105.fw:iwn105fw -miwn105fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn105fw.c" iwn105fw.fwo optional iwn105fw | iwnfw \ dependency "iwn105.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn105fw.fwo" iwn105.fw optional iwn105fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-105-6-18.168.6.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn105.fw" iwn135fw.c optional iwn135fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn135.fw:iwn135fw -miwn135fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn135fw.c" iwn135fw.fwo optional iwn135fw | iwnfw \ dependency "iwn135.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn135fw.fwo" iwn135.fw optional iwn135fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-135-6-18.168.6.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn135.fw" iwn2000fw.c optional iwn2000fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn2000.fw:iwn2000fw -miwn2000fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn2000fw.c" iwn2000fw.fwo optional iwn2000fw | iwnfw \ dependency "iwn2000.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn2000fw.fwo" iwn2000.fw optional iwn2000fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-2000-18.168.6.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn2000.fw" iwn2030fw.c optional iwn2030fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn2030.fw:iwn2030fw -miwn2030fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn2030fw.c" iwn2030fw.fwo optional iwn2030fw | iwnfw \ dependency "iwn2030.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn2030fw.fwo" iwn2030.fw optional iwn2030fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwnwifi-2030-18.168.6.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn2030.fw" iwn4965fw.c optional iwn4965fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn4965.fw:iwn4965fw -miwn4965fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn4965fw.c" iwn4965fw.fwo optional iwn4965fw | iwnfw \ dependency "iwn4965.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn4965fw.fwo" iwn4965.fw optional iwn4965fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-4965-228.61.2.24.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn4965.fw" iwn5000fw.c optional iwn5000fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn5000.fw:iwn5000fw -miwn5000fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn5000fw.c" iwn5000fw.fwo optional iwn5000fw | iwnfw \ dependency "iwn5000.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn5000fw.fwo" iwn5000.fw optional iwn5000fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-5000-8.83.5.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn5000.fw" iwn5150fw.c optional iwn5150fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn5150.fw:iwn5150fw -miwn5150fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn5150fw.c" iwn5150fw.fwo optional iwn5150fw | iwnfw \ dependency "iwn5150.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn5150fw.fwo" iwn5150.fw optional iwn5150fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-5150-8.24.2.2.fw.uu"\ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn5150.fw" iwn6000fw.c optional iwn6000fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn6000.fw:iwn6000fw -miwn6000fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn6000fw.c" iwn6000fw.fwo optional iwn6000fw | iwnfw \ dependency "iwn6000.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn6000fw.fwo" iwn6000.fw optional iwn6000fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-6000-9.221.4.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn6000.fw" iwn6000g2afw.c optional iwn6000g2afw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn6000g2a.fw:iwn6000g2afw -miwn6000g2afw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn6000g2afw.c" iwn6000g2afw.fwo optional iwn6000g2afw | iwnfw \ dependency "iwn6000g2a.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn6000g2afw.fwo" iwn6000g2a.fw optional iwn6000g2afw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-6000g2a-18.168.6.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn6000g2a.fw" iwn6000g2bfw.c optional iwn6000g2bfw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn6000g2b.fw:iwn6000g2bfw -miwn6000g2bfw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn6000g2bfw.c" iwn6000g2bfw.fwo optional iwn6000g2bfw | iwnfw \ dependency "iwn6000g2b.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn6000g2bfw.fwo" iwn6000g2b.fw optional iwn6000g2bfw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-6000g2b-18.168.6.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn6000g2b.fw" iwn6050fw.c optional iwn6050fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn6050.fw:iwn6050fw -miwn6050fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn6050fw.c" iwn6050fw.fwo optional iwn6050fw | iwnfw \ dependency "iwn6050.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn6050fw.fwo" iwn6050.fw optional iwn6050fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-6050-41.28.5.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn6050.fw" dev/ixgb/if_ixgb.c optional ixgb dev/ixgb/ixgb_ee.c optional ixgb dev/ixgb/ixgb_hw.c optional ixgb dev/ixgbe/if_ix.c optional ix inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe -DSMP" dev/ixgbe/if_ixv.c optional ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe -DSMP" dev/ixgbe/if_bypass.c optional ix inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_netmap.c optional ix inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/if_fdir.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/if_sriov.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ix_txrx.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_osdep.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_phy.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_api.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_common.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_mbx.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_vf.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_82598.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_82599.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_x540.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_x550.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_dcb.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_dcb_82598.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_dcb_82599.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/jedec_ts/jedec_ts.c optional jedec_ts smbus dev/jme/if_jme.c optional jme pci dev/joy/joy.c optional joy dev/joy/joy_isa.c optional joy isa dev/kbd/kbd.c optional atkbd | pckbd | sc | ukbd | vt dev/kbdmux/kbdmux.c optional kbdmux dev/ksyms/ksyms.c optional ksyms dev/le/am7990.c optional le dev/le/am79900.c optional le dev/le/if_le_pci.c optional le pci dev/le/lance.c optional le dev/led/led.c standard dev/lge/if_lge.c optional lge dev/liquidio/base/cn23xx_pf_device.c optional lio \ compile-with "${NORMAL_C} \ -I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP" dev/liquidio/base/lio_console.c optional lio \ compile-with "${NORMAL_C} \ -I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP" dev/liquidio/base/lio_ctrl.c optional lio \ compile-with "${NORMAL_C} \ -I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP" dev/liquidio/base/lio_device.c optional lio \ compile-with "${NORMAL_C} \ -I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP" dev/liquidio/base/lio_droq.c optional lio \ compile-with "${NORMAL_C} \ -I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP" dev/liquidio/base/lio_mem_ops.c optional lio \ compile-with "${NORMAL_C} \ -I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP" dev/liquidio/base/lio_request_manager.c optional lio \ compile-with "${NORMAL_C} \ -I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP" dev/liquidio/base/lio_response_manager.c optional lio \ compile-with "${NORMAL_C} \ -I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP" dev/liquidio/lio_core.c optional lio \ compile-with "${NORMAL_C} \ -I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP" dev/liquidio/lio_ioctl.c optional lio \ compile-with "${NORMAL_C} \ -I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP" dev/liquidio/lio_main.c optional lio \ compile-with "${NORMAL_C} \ -I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP" dev/liquidio/lio_rss.c optional lio \ compile-with "${NORMAL_C} \ -I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP" dev/liquidio/lio_rxtx.c optional lio \ compile-with "${NORMAL_C} \ -I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP" dev/liquidio/lio_sysctl.c optional lio \ compile-with "${NORMAL_C} \ -I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP" lio.c optional lio \ compile-with "${AWK} -f $S/tools/fw_stub.awk lio_23xx_nic.bin.fw:lio_23xx_nic.bin -mlio_23xx_nic.bin -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "lio.c" lio_23xx_nic.bin.fw.fwo optional lio \ dependency "lio_23xx_nic.bin.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "lio_23xx_nic.bin.fw.fwo" lio_23xx_nic.bin.fw optional lio \ dependency "$S/contrib/dev/liquidio/lio_23xx_nic.bin.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "lio_23xx_nic.bin.fw" dev/lmc/if_lmc.c optional lmc dev/malo/if_malo.c optional malo dev/malo/if_malohal.c optional malo dev/malo/if_malo_pci.c optional malo pci dev/mc146818/mc146818.c optional mc146818 dev/mca/mca_bus.c optional mca dev/mcd/mcd.c optional mcd isa nowerror dev/mcd/mcd_isa.c optional mcd isa nowerror dev/md/md.c optional md dev/mdio/mdio_if.m optional miiproxy | mdio dev/mdio/mdio.c optional miiproxy | mdio dev/mem/memdev.c optional mem dev/mem/memutil.c optional mem dev/mfi/mfi.c optional mfi dev/mfi/mfi_debug.c optional mfi dev/mfi/mfi_pci.c optional mfi pci dev/mfi/mfi_disk.c optional mfi dev/mfi/mfi_syspd.c optional mfi dev/mfi/mfi_tbolt.c optional mfi dev/mfi/mfi_linux.c optional mfi compat_linux dev/mfi/mfi_cam.c optional mfip scbus dev/mii/acphy.c optional miibus | acphy dev/mii/amphy.c optional miibus | amphy dev/mii/atphy.c optional miibus | atphy dev/mii/axphy.c optional miibus | axphy dev/mii/bmtphy.c optional miibus | bmtphy dev/mii/brgphy.c optional miibus | brgphy dev/mii/ciphy.c optional miibus | ciphy dev/mii/e1000phy.c optional miibus | e1000phy dev/mii/gentbi.c optional miibus | gentbi dev/mii/icsphy.c optional miibus | icsphy dev/mii/ip1000phy.c optional miibus | ip1000phy dev/mii/jmphy.c optional miibus | jmphy dev/mii/lxtphy.c optional miibus | lxtphy dev/mii/micphy.c optional miibus fdt | micphy fdt dev/mii/mii.c optional miibus | mii dev/mii/mii_bitbang.c optional miibus | mii_bitbang dev/mii/mii_physubr.c optional miibus | mii dev/mii/mii_fdt.c optional miibus fdt | mii fdt dev/mii/miibus_if.m optional miibus | mii dev/mii/mlphy.c optional miibus | mlphy dev/mii/nsgphy.c optional miibus | nsgphy dev/mii/nsphy.c optional miibus | nsphy dev/mii/nsphyter.c optional miibus | nsphyter dev/mii/pnaphy.c optional miibus | pnaphy dev/mii/qsphy.c optional miibus | qsphy dev/mii/rdcphy.c optional miibus | rdcphy dev/mii/rgephy.c optional miibus | rgephy dev/mii/rlphy.c optional miibus | rlphy dev/mii/rlswitch.c optional rlswitch dev/mii/smcphy.c optional miibus | smcphy dev/mii/smscphy.c optional miibus | smscphy dev/mii/tdkphy.c optional miibus | tdkphy dev/mii/tlphy.c optional miibus | tlphy dev/mii/truephy.c optional miibus | truephy dev/mii/ukphy.c optional miibus | mii dev/mii/ukphy_subr.c optional miibus | mii dev/mii/vscphy.c optional miibus | vscphy dev/mii/xmphy.c optional miibus | xmphy dev/mk48txx/mk48txx.c optional mk48txx dev/mlx/mlx.c optional mlx dev/mlx/mlx_disk.c optional mlx dev/mlx/mlx_pci.c optional mlx pci dev/mly/mly.c optional mly dev/mmc/mmc_subr.c optional mmc | mmcsd dev/mmc/mmc.c optional mmc dev/mmc/mmcbr_if.m standard dev/mmc/mmcbus_if.m standard dev/mmc/mmcsd.c optional mmcsd dev/mn/if_mn.c optional mn pci dev/mpr/mpr.c optional mpr dev/mpr/mpr_config.c optional mpr # XXX Work around clang warning, until maintainer approves fix. dev/mpr/mpr_mapping.c optional mpr \ compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}" dev/mpr/mpr_pci.c optional mpr pci dev/mpr/mpr_sas.c optional mpr \ compile-with "${NORMAL_C} ${NO_WUNNEEDED_INTERNAL_DECL}" dev/mpr/mpr_sas_lsi.c optional mpr dev/mpr/mpr_table.c optional mpr dev/mpr/mpr_user.c optional mpr dev/mps/mps.c optional mps dev/mps/mps_config.c optional mps # XXX Work around clang warning, until maintainer approves fix. dev/mps/mps_mapping.c optional mps \ compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}" dev/mps/mps_pci.c optional mps pci dev/mps/mps_sas.c optional mps \ compile-with "${NORMAL_C} ${NO_WUNNEEDED_INTERNAL_DECL}" dev/mps/mps_sas_lsi.c optional mps dev/mps/mps_table.c optional mps dev/mps/mps_user.c optional mps dev/mpt/mpt.c optional mpt dev/mpt/mpt_cam.c optional mpt dev/mpt/mpt_debug.c optional mpt dev/mpt/mpt_pci.c optional mpt pci dev/mpt/mpt_raid.c optional mpt dev/mpt/mpt_user.c optional mpt dev/mrsas/mrsas.c optional mrsas dev/mrsas/mrsas_cam.c optional mrsas dev/mrsas/mrsas_ioctl.c optional mrsas dev/mrsas/mrsas_fp.c optional mrsas dev/msk/if_msk.c optional msk dev/mvs/mvs.c optional mvs dev/mvs/mvs_if.m optional mvs dev/mvs/mvs_pci.c optional mvs pci dev/mwl/if_mwl.c optional mwl dev/mwl/if_mwl_pci.c optional mwl pci dev/mwl/mwlhal.c optional mwl mwlfw.c optional mwlfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk mw88W8363.fw:mw88W8363fw mwlboot.fw:mwlboot -mmwl -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "mwlfw.c" mw88W8363.fwo optional mwlfw \ dependency "mw88W8363.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "mw88W8363.fwo" mw88W8363.fw optional mwlfw \ dependency "$S/contrib/dev/mwl/mw88W8363.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "mw88W8363.fw" mwlboot.fwo optional mwlfw \ dependency "mwlboot.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "mwlboot.fwo" mwlboot.fw optional mwlfw \ dependency "$S/contrib/dev/mwl/mwlboot.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "mwlboot.fw" dev/mxge/if_mxge.c optional mxge pci dev/mxge/mxge_eth_z8e.c optional mxge pci dev/mxge/mxge_ethp_z8e.c optional mxge pci dev/mxge/mxge_rss_eth_z8e.c optional mxge pci dev/mxge/mxge_rss_ethp_z8e.c optional mxge pci dev/my/if_my.c optional my dev/nand/nand.c optional nand dev/nand/nand_bbt.c optional nand dev/nand/nand_cdev.c optional nand dev/nand/nand_generic.c optional nand dev/nand/nand_geom.c optional nand dev/nand/nand_id.c optional nand dev/nand/nandbus.c optional nand dev/nand/nandbus_if.m optional nand dev/nand/nand_if.m optional nand dev/nand/nandsim.c optional nandsim nand dev/nand/nandsim_chip.c optional nandsim nand dev/nand/nandsim_ctrl.c optional nandsim nand dev/nand/nandsim_log.c optional nandsim nand dev/nand/nandsim_swap.c optional nandsim nand dev/nand/nfc_if.m optional nand dev/ncr/ncr.c optional ncr pci dev/ncv/ncr53c500.c optional ncv dev/ncv/ncr53c500_pccard.c optional ncv pccard dev/netmap/netmap.c optional netmap dev/netmap/netmap_freebsd.c optional netmap dev/netmap/netmap_generic.c optional netmap dev/netmap/netmap_mbq.c optional netmap dev/netmap/netmap_mem2.c optional netmap dev/netmap/netmap_monitor.c optional netmap dev/netmap/netmap_offloadings.c optional netmap dev/netmap/netmap_pipe.c optional netmap dev/netmap/netmap_vale.c optional netmap # compile-with "${NORMAL_C} -Wconversion -Wextra" dev/nfsmb/nfsmb.c optional nfsmb pci dev/nge/if_nge.c optional nge dev/nxge/if_nxge.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-device.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-mm.c optional nxge dev/nxge/xgehal/xge-queue.c optional nxge dev/nxge/xgehal/xgehal-driver.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-ring.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-channel.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-fifo.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-stats.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-config.c optional nxge dev/nxge/xgehal/xgehal-mgmt.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nmdm/nmdm.c optional nmdm dev/nsp/nsp.c optional nsp dev/nsp/nsp_pccard.c optional nsp pccard dev/null/null.c standard dev/oce/oce_hw.c optional oce pci dev/oce/oce_if.c optional oce pci dev/oce/oce_mbox.c optional oce pci dev/oce/oce_queue.c optional oce pci dev/oce/oce_sysctl.c optional oce pci dev/oce/oce_util.c optional oce pci dev/ofw/ofw_bus_if.m optional fdt dev/ofw/ofw_bus_subr.c optional fdt dev/ofw/ofw_fdt.c optional fdt dev/ofw/ofw_if.m optional fdt dev/ofw/ofw_subr.c optional fdt dev/ofw/ofwbus.c optional fdt dev/ofw/openfirm.c optional fdt dev/ofw/openfirmio.c optional fdt dev/ow/ow.c optional ow \ dependency "owll_if.h" \ dependency "own_if.h" dev/ow/owll_if.m optional ow dev/ow/own_if.m optional ow dev/ow/ow_temp.c optional ow_temp dev/ow/owc_gpiobus.c optional owc gpio dev/patm/if_patm.c optional patm pci dev/patm/if_patm_attach.c optional patm pci dev/patm/if_patm_intr.c optional patm pci dev/patm/if_patm_ioctl.c optional patm pci dev/patm/if_patm_rtables.c optional patm pci dev/patm/if_patm_rx.c optional patm pci dev/patm/if_patm_tx.c optional patm pci dev/pbio/pbio.c optional pbio isa dev/pccard/card_if.m standard dev/pccard/pccard.c optional pccard dev/pccard/pccard_cis.c optional pccard dev/pccard/pccard_cis_quirks.c optional pccard dev/pccard/pccard_device.c optional pccard dev/pccard/power_if.m standard dev/pccbb/pccbb.c optional cbb dev/pccbb/pccbb_isa.c optional cbb isa dev/pccbb/pccbb_pci.c optional cbb pci dev/pcf/pcf.c optional pcf dev/pci/eisa_pci.c optional pci eisa dev/pci/fixup_pci.c optional pci dev/pci/hostb_pci.c optional pci dev/pci/ignore_pci.c optional pci dev/pci/isa_pci.c optional pci isa dev/pci/pci.c optional pci dev/pci/pci_if.m standard dev/pci/pci_iov.c optional pci pci_iov dev/pci/pci_iov_if.m standard dev/pci/pci_iov_schema.c optional pci pci_iov dev/pci/pci_pci.c optional pci dev/pci/pci_subr.c optional pci dev/pci/pci_user.c optional pci dev/pci/pcib_if.m standard dev/pci/pcib_support.c standard dev/pci/vga_pci.c optional pci dev/pcn/if_pcn.c optional pcn pci dev/pdq/if_fea.c optional fea eisa dev/pdq/if_fpa.c optional fpa pci dev/pdq/pdq.c optional nowerror fea eisa | fpa pci dev/pdq/pdq_ifsubr.c optional nowerror fea eisa | fpa pci dev/pms/freebsd/driver/ini/src/agtiapi.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/sadisc.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/mpi.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/saframe.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/sahw.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/sainit.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/saint.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/sampicmd.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/sampirsp.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/saphy.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/saport.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/sasata.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/sasmp.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/sassp.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/satimer.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/sautil.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/saioctlcmd.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sallsdk/spc/mpidebug.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/discovery/dm/dminit.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/discovery/dm/dmsmp.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/discovery/dm/dmdisc.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/discovery/dm/dmport.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/discovery/dm/dmtimer.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/discovery/dm/dmmisc.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sat/src/sminit.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sat/src/smmisc.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sat/src/smsat.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sat/src/smsatcb.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sat/src/smsathw.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/sat/src/smtimer.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/common/tdinit.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/common/tdmisc.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/common/tdesgl.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/common/tdport.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/common/tdint.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/common/tdioctl.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/common/tdhw.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/common/ossacmnapi.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/common/tddmcmnapi.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/common/tdsmcmnapi.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/common/tdtimers.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/sas/ini/itdio.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/sas/ini/itdcb.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/sas/ini/itdinit.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/sas/ini/itddisc.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/sata/host/sat.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/sata/host/ossasat.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/pms/RefTisa/tisa/sassata/sata/host/sathw.c optional pmspcv \ compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w" dev/ppbus/if_plip.c optional plip dev/ppbus/immio.c optional vpo dev/ppbus/lpbb.c optional lpbb dev/ppbus/lpt.c optional lpt dev/ppbus/pcfclock.c optional pcfclock dev/ppbus/ppb_1284.c optional ppbus dev/ppbus/ppb_base.c optional ppbus dev/ppbus/ppb_msq.c optional ppbus dev/ppbus/ppbconf.c optional ppbus dev/ppbus/ppbus_if.m optional ppbus dev/ppbus/ppi.c optional ppi dev/ppbus/pps.c optional pps dev/ppbus/vpo.c optional vpo dev/ppbus/vpoio.c optional vpo dev/ppc/ppc.c optional ppc dev/ppc/ppc_acpi.c optional ppc acpi dev/ppc/ppc_isa.c optional ppc isa dev/ppc/ppc_pci.c optional ppc pci dev/ppc/ppc_puc.c optional ppc puc dev/proto/proto_bus_isa.c optional proto acpi | proto isa dev/proto/proto_bus_pci.c optional proto pci dev/proto/proto_busdma.c optional proto dev/proto/proto_core.c optional proto dev/pst/pst-iop.c optional pst dev/pst/pst-pci.c optional pst pci dev/pst/pst-raid.c optional pst dev/pty/pty.c optional pty dev/puc/puc.c optional puc dev/puc/puc_cfg.c optional puc dev/puc/puc_pccard.c optional puc pccard dev/puc/puc_pci.c optional puc pci dev/puc/pucdata.c optional puc pci dev/quicc/quicc_core.c optional quicc dev/ral/rt2560.c optional ral dev/ral/rt2661.c optional ral dev/ral/rt2860.c optional ral dev/ral/if_ral_pci.c optional ral pci rt2561fw.c optional rt2561fw | ralfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk rt2561.fw:rt2561fw -mrt2561 -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "rt2561fw.c" rt2561fw.fwo optional rt2561fw | ralfw \ dependency "rt2561.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "rt2561fw.fwo" rt2561.fw optional rt2561fw | ralfw \ dependency "$S/contrib/dev/ral/rt2561.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "rt2561.fw" rt2561sfw.c optional rt2561sfw | ralfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk rt2561s.fw:rt2561sfw -mrt2561s -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "rt2561sfw.c" rt2561sfw.fwo optional rt2561sfw | ralfw \ dependency "rt2561s.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "rt2561sfw.fwo" rt2561s.fw optional rt2561sfw | ralfw \ dependency "$S/contrib/dev/ral/rt2561s.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "rt2561s.fw" rt2661fw.c optional rt2661fw | ralfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk rt2661.fw:rt2661fw -mrt2661 -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "rt2661fw.c" rt2661fw.fwo optional rt2661fw | ralfw \ dependency "rt2661.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "rt2661fw.fwo" rt2661.fw optional rt2661fw | ralfw \ dependency "$S/contrib/dev/ral/rt2661.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "rt2661.fw" rt2860fw.c optional rt2860fw | ralfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk rt2860.fw:rt2860fw -mrt2860 -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "rt2860fw.c" rt2860fw.fwo optional rt2860fw | ralfw \ dependency "rt2860.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "rt2860fw.fwo" rt2860.fw optional rt2860fw | ralfw \ dependency "$S/contrib/dev/ral/rt2860.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "rt2860.fw" dev/random/random_infra.c optional random dev/random/random_harvestq.c optional random dev/random/randomdev.c optional random random_yarrow | \ random !random_yarrow !random_loadable dev/random/yarrow.c optional random random_yarrow dev/random/fortuna.c optional random !random_yarrow !random_loadable dev/random/hash.c optional random random_yarrow | \ random !random_yarrow !random_loadable dev/rc/rc.c optional rc dev/rccgpio/rccgpio.c optional rccgpio gpio dev/re/if_re.c optional re dev/rl/if_rl.c optional rl pci dev/rndtest/rndtest.c optional rndtest dev/rp/rp.c optional rp dev/rp/rp_isa.c optional rp isa dev/rp/rp_pci.c optional rp pci dev/rtwn/if_rtwn.c optional rtwn rtwn-rtl8192cfwU.c optional rtwn-rtl8192cfwU | rtwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk rtwn-rtl8192cfwU.fw:rtwn-rtl8192cfwU:111 -mrtwn-rtl8192cfwU -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "rtwn-rtl8192cfwU.c" rtwn-rtl8192cfwU.fwo optional rtwn-rtl8192cfwU | rtwnfw \ dependency "rtwn-rtl8192cfwU.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "rtwn-rtl8192cfwU.fwo" rtwn-rtl8192cfwU.fw optional rtwn-rtl8192cfwU | rtwnfw \ dependency "$S/contrib/dev/rtwn/rtwn-rtl8192cfwU.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "rtwn-rtl8192cfwU.fw" rtwn-rtl8192cfwU_B.c optional rtwn-rtl8192cfwU_B | rtwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk rtwn-rtl8192cfwU_B.fw:rtwn-rtl8192cfwU_B:111 -mrtwn-rtl8192cfwU_B -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "rtwn-rtl8192cfwU_B.c" rtwn-rtl8192cfwU_B.fwo optional rtwn-rtl8192cfwU_B | rtwnfw \ dependency "rtwn-rtl8192cfwU_B.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "rtwn-rtl8192cfwU_B.fwo" rtwn-rtl8192cfwU_B.fw optional rtwn-rtl8192cfwU_B | rtwnfw \ dependency "$S/contrib/dev/rtwn/rtwn-rtl8192cfwU_B.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "rtwn-rtl8192cfwU_B.fw" dev/safe/safe.c optional safe dev/scc/scc_if.m optional scc dev/scc/scc_bfe_ebus.c optional scc ebus dev/scc/scc_bfe_quicc.c optional scc quicc dev/scc/scc_bfe_sbus.c optional scc fhc | scc sbus dev/scc/scc_core.c optional scc dev/scc/scc_dev_quicc.c optional scc quicc dev/scc/scc_dev_sab82532.c optional scc dev/scc/scc_dev_z8530.c optional scc dev/scd/scd.c optional scd isa dev/scd/scd_isa.c optional scd isa dev/sdhci/sdhci.c optional sdhci dev/sdhci/sdhci_fdt_gpio.c optional sdhci fdt gpio dev/sdhci/sdhci_if.m optional sdhci dev/sdhci/sdhci_acpi.c optional sdhci acpi dev/sdhci/sdhci_pci.c optional sdhci pci dev/sf/if_sf.c optional sf pci dev/sge/if_sge.c optional sge pci dev/si/si.c optional si \ compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}" dev/si/si2_z280.c optional si dev/si/si3_t225.c optional si dev/si/si_eisa.c optional si eisa dev/si/si_isa.c optional si isa dev/si/si_pci.c optional si pci dev/siba/siba_bwn.c optional siba_bwn pci dev/siba/siba_core.c optional siba_bwn pci dev/siis/siis.c optional siis pci dev/sis/if_sis.c optional sis pci dev/sk/if_sk.c optional sk pci dev/smbus/smb.c optional smb dev/smbus/smbconf.c optional smbus dev/smbus/smbus.c optional smbus dev/smbus/smbus_if.m optional smbus dev/smc/if_smc.c optional smc dev/smc/if_smc_fdt.c optional smc fdt dev/sn/if_sn.c optional sn dev/sn/if_sn_isa.c optional sn isa dev/sn/if_sn_pccard.c optional sn pccard dev/snp/snp.c optional snp dev/sound/clone.c optional sound dev/sound/unit.c optional sound dev/sound/isa/ad1816.c optional snd_ad1816 isa dev/sound/isa/ess.c optional snd_ess isa dev/sound/isa/gusc.c optional snd_gusc isa dev/sound/isa/mss.c optional snd_mss isa dev/sound/isa/sb16.c optional snd_sb16 isa dev/sound/isa/sb8.c optional snd_sb8 isa dev/sound/isa/sbc.c optional snd_sbc isa dev/sound/isa/sndbuf_dma.c optional sound isa dev/sound/pci/als4000.c optional snd_als4000 pci dev/sound/pci/atiixp.c optional snd_atiixp pci dev/sound/pci/cmi.c optional snd_cmi pci dev/sound/pci/cs4281.c optional snd_cs4281 pci dev/sound/pci/csa.c optional snd_csa pci dev/sound/pci/csapcm.c optional snd_csa pci dev/sound/pci/ds1.c optional snd_ds1 pci dev/sound/pci/emu10k1.c optional snd_emu10k1 pci dev/sound/pci/emu10kx.c optional snd_emu10kx pci dev/sound/pci/emu10kx-pcm.c optional snd_emu10kx pci dev/sound/pci/emu10kx-midi.c optional snd_emu10kx pci dev/sound/pci/envy24.c optional snd_envy24 pci dev/sound/pci/envy24ht.c optional snd_envy24ht pci dev/sound/pci/es137x.c optional snd_es137x pci dev/sound/pci/fm801.c optional snd_fm801 pci dev/sound/pci/ich.c optional snd_ich pci dev/sound/pci/maestro.c optional snd_maestro pci dev/sound/pci/maestro3.c optional snd_maestro3 pci dev/sound/pci/neomagic.c optional snd_neomagic pci dev/sound/pci/solo.c optional snd_solo pci dev/sound/pci/spicds.c optional snd_spicds pci dev/sound/pci/t4dwave.c optional snd_t4dwave pci dev/sound/pci/via8233.c optional snd_via8233 pci dev/sound/pci/via82c686.c optional snd_via82c686 pci dev/sound/pci/vibes.c optional snd_vibes pci dev/sound/pci/hda/hdaa.c optional snd_hda pci dev/sound/pci/hda/hdaa_patches.c optional snd_hda pci dev/sound/pci/hda/hdac.c optional snd_hda pci dev/sound/pci/hda/hdac_if.m optional snd_hda pci dev/sound/pci/hda/hdacc.c optional snd_hda pci dev/sound/pci/hdspe.c optional snd_hdspe pci dev/sound/pci/hdspe-pcm.c optional snd_hdspe pci dev/sound/pcm/ac97.c optional sound dev/sound/pcm/ac97_if.m optional sound dev/sound/pcm/ac97_patch.c optional sound dev/sound/pcm/buffer.c optional sound \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/channel.c optional sound dev/sound/pcm/channel_if.m optional sound dev/sound/pcm/dsp.c optional sound dev/sound/pcm/feeder.c optional sound dev/sound/pcm/feeder_chain.c optional sound dev/sound/pcm/feeder_eq.c optional sound \ dependency "feeder_eq_gen.h" \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/feeder_if.m optional sound dev/sound/pcm/feeder_format.c optional sound \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/feeder_matrix.c optional sound \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/feeder_mixer.c optional sound \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/feeder_rate.c optional sound \ dependency "feeder_rate_gen.h" \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/feeder_volume.c optional sound \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/mixer.c optional sound dev/sound/pcm/mixer_if.m optional sound dev/sound/pcm/sndstat.c optional sound dev/sound/pcm/sound.c optional sound dev/sound/pcm/vchan.c optional sound dev/sound/usb/uaudio.c optional snd_uaudio usb dev/sound/usb/uaudio_pcm.c optional snd_uaudio usb dev/sound/midi/midi.c optional sound dev/sound/midi/mpu401.c optional sound dev/sound/midi/mpu_if.m optional sound dev/sound/midi/mpufoi_if.m optional sound dev/sound/midi/sequencer.c optional sound dev/sound/midi/synth_if.m optional sound dev/spibus/ofw_spibus.c optional fdt spibus dev/spibus/spibus.c optional spibus \ dependency "spibus_if.h" dev/spibus/spigen.c optional spigen dev/spibus/spibus_if.m optional spibus dev/ste/if_ste.c optional ste pci dev/stg/tmc18c30.c optional stg dev/stg/tmc18c30_isa.c optional stg isa dev/stg/tmc18c30_pccard.c optional stg pccard dev/stg/tmc18c30_pci.c optional stg pci dev/stg/tmc18c30_subr.c optional stg dev/stge/if_stge.c optional stge dev/streams/streams.c optional streams dev/sym/sym_hipd.c optional sym \ dependency "$S/dev/sym/sym_{conf,defs}.h" dev/syscons/blank/blank_saver.c optional blank_saver dev/syscons/daemon/daemon_saver.c optional daemon_saver dev/syscons/dragon/dragon_saver.c optional dragon_saver dev/syscons/fade/fade_saver.c optional fade_saver dev/syscons/fire/fire_saver.c optional fire_saver dev/syscons/green/green_saver.c optional green_saver dev/syscons/logo/logo.c optional logo_saver dev/syscons/logo/logo_saver.c optional logo_saver dev/syscons/rain/rain_saver.c optional rain_saver dev/syscons/schistory.c optional sc dev/syscons/scmouse.c optional sc dev/syscons/scterm.c optional sc dev/syscons/scvidctl.c optional sc dev/syscons/snake/snake_saver.c optional snake_saver dev/syscons/star/star_saver.c optional star_saver dev/syscons/syscons.c optional sc dev/syscons/sysmouse.c optional sc dev/syscons/warp/warp_saver.c optional warp_saver dev/tdfx/tdfx_linux.c optional tdfx_linux tdfx compat_linux dev/tdfx/tdfx_pci.c optional tdfx pci dev/ti/if_ti.c optional ti pci dev/tl/if_tl.c optional tl pci dev/trm/trm.c optional trm dev/twa/tw_cl_init.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_cl_intr.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_cl_io.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_cl_misc.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_osl_cam.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_osl_freebsd.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twe/twe.c optional twe dev/twe/twe_freebsd.c optional twe dev/tws/tws.c optional tws dev/tws/tws_cam.c optional tws dev/tws/tws_hdm.c optional tws dev/tws/tws_services.c optional tws dev/tws/tws_user.c optional tws dev/tx/if_tx.c optional tx dev/txp/if_txp.c optional txp dev/uart/uart_bus_acpi.c optional uart acpi dev/uart/uart_bus_ebus.c optional uart ebus dev/uart/uart_bus_fdt.c optional uart fdt dev/uart/uart_bus_isa.c optional uart isa dev/uart/uart_bus_pccard.c optional uart pccard dev/uart/uart_bus_pci.c optional uart pci dev/uart/uart_bus_puc.c optional uart puc dev/uart/uart_bus_scc.c optional uart scc dev/uart/uart_core.c optional uart dev/uart/uart_dbg.c optional uart gdb dev/uart/uart_dev_ns8250.c optional uart uart_ns8250 | uart uart_snps dev/uart/uart_dev_pl011.c optional uart pl011 dev/uart/uart_dev_quicc.c optional uart quicc dev/uart/uart_dev_sab82532.c optional uart uart_sab82532 dev/uart/uart_dev_sab82532.c optional uart scc dev/uart/uart_dev_snps.c optional uart uart_snps dev/uart/uart_dev_z8530.c optional uart uart_z8530 dev/uart/uart_dev_z8530.c optional uart scc dev/uart/uart_if.m optional uart dev/uart/uart_subr.c optional uart dev/uart/uart_tty.c optional uart dev/ubsec/ubsec.c optional ubsec # # USB controller drivers # dev/usb/controller/at91dci.c optional at91dci dev/usb/controller/at91dci_atmelarm.c optional at91dci at91rm9200 dev/usb/controller/musb_otg.c optional musb dev/usb/controller/musb_otg_atmelarm.c optional musb at91rm9200 dev/usb/controller/dwc_otg.c optional dwcotg dev/usb/controller/dwc_otg_fdt.c optional dwcotg fdt dev/usb/controller/ehci.c optional ehci dev/usb/controller/ehci_pci.c optional ehci pci dev/usb/controller/ohci.c optional ohci dev/usb/controller/ohci_pci.c optional ohci pci dev/usb/controller/uhci.c optional uhci dev/usb/controller/uhci_pci.c optional uhci pci dev/usb/controller/xhci.c optional xhci dev/usb/controller/xhci_pci.c optional xhci pci dev/usb/controller/saf1761_otg.c optional saf1761otg dev/usb/controller/saf1761_otg_fdt.c optional saf1761otg fdt dev/usb/controller/uss820dci.c optional uss820dci dev/usb/controller/uss820dci_atmelarm.c optional uss820dci at91rm9200 dev/usb/controller/usb_controller.c optional usb # # USB storage drivers # dev/usb/storage/cfumass.c optional cfumass ctl dev/usb/storage/umass.c optional umass dev/usb/storage/urio.c optional urio dev/usb/storage/ustorage_fs.c optional usfs # # USB core # dev/usb/usb_busdma.c optional usb dev/usb/usb_core.c optional usb dev/usb/usb_debug.c optional usb dev/usb/usb_dev.c optional usb dev/usb/usb_device.c optional usb dev/usb/usb_dynamic.c optional usb dev/usb/usb_error.c optional usb dev/usb/usb_generic.c optional usb dev/usb/usb_handle_request.c optional usb dev/usb/usb_hid.c optional usb dev/usb/usb_hub.c optional usb dev/usb/usb_if.m optional usb dev/usb/usb_lookup.c optional usb dev/usb/usb_mbuf.c optional usb dev/usb/usb_msctest.c optional usb dev/usb/usb_parse.c optional usb dev/usb/usb_pf.c optional usb dev/usb/usb_process.c optional usb dev/usb/usb_request.c optional usb dev/usb/usb_transfer.c optional usb dev/usb/usb_util.c optional usb # # USB network drivers # dev/usb/net/if_aue.c optional aue dev/usb/net/if_axe.c optional axe dev/usb/net/if_axge.c optional axge dev/usb/net/if_cdce.c optional cdce dev/usb/net/if_cue.c optional cue dev/usb/net/if_ipheth.c optional ipheth dev/usb/net/if_kue.c optional kue dev/usb/net/if_mos.c optional mos dev/usb/net/if_rue.c optional rue dev/usb/net/if_smsc.c optional smsc dev/usb/net/if_udav.c optional udav dev/usb/net/if_ure.c optional ure dev/usb/net/if_usie.c optional usie dev/usb/net/if_urndis.c optional urndis dev/usb/net/ruephy.c optional rue dev/usb/net/usb_ethernet.c optional uether | aue | axe | axge | cdce | \ cue | ipheth | kue | mos | rue | \ smsc | udav | ure | urndis dev/usb/net/uhso.c optional uhso # # USB WLAN drivers # dev/usb/wlan/if_rsu.c optional rsu rsu-rtl8712fw.c optional rsu-rtl8712fw | rsufw \ compile-with "${AWK} -f $S/tools/fw_stub.awk rsu-rtl8712fw.fw:rsu-rtl8712fw:120 -mrsu-rtl8712fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "rsu-rtl8712fw.c" rsu-rtl8712fw.fwo optional rsu-rtl8712fw | rsufw \ dependency "rsu-rtl8712fw.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "rsu-rtl8712fw.fwo" rsu-rtl8712fw.fw optional rsu-rtl8712.fw | rsufw \ dependency "$S/contrib/dev/rsu/rsu-rtl8712fw.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "rsu-rtl8712fw.fw" dev/usb/wlan/if_rum.c optional rum dev/usb/wlan/if_run.c optional run runfw.c optional runfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk run.fw:runfw -mrunfw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "runfw.c" runfw.fwo optional runfw \ dependency "run.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "runfw.fwo" run.fw optional runfw \ dependency "$S/contrib/dev/run/rt2870.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "run.fw" dev/usb/wlan/if_uath.c optional uath dev/usb/wlan/if_upgt.c optional upgt dev/usb/wlan/if_ural.c optional ural dev/usb/wlan/if_urtw.c optional urtw dev/usb/wlan/if_zyd.c optional zyd # # USB serial and parallel port drivers # dev/usb/serial/u3g.c optional u3g dev/usb/serial/uark.c optional uark dev/usb/serial/ubsa.c optional ubsa dev/usb/serial/ubser.c optional ubser dev/usb/serial/uchcom.c optional uchcom dev/usb/serial/ucycom.c optional ucycom dev/usb/serial/ufoma.c optional ufoma dev/usb/serial/uftdi.c optional uftdi dev/usb/serial/ugensa.c optional ugensa dev/usb/serial/uipaq.c optional uipaq dev/usb/serial/ulpt.c optional ulpt dev/usb/serial/umcs.c optional umcs dev/usb/serial/umct.c optional umct dev/usb/serial/umodem.c optional umodem dev/usb/serial/umoscom.c optional umoscom dev/usb/serial/uplcom.c optional uplcom dev/usb/serial/uslcom.c optional uslcom dev/usb/serial/uvisor.c optional uvisor dev/usb/serial/uvscom.c optional uvscom dev/usb/serial/usb_serial.c optional ucom | u3g | uark | ubsa | ubser | \ uchcom | ucycom | ufoma | uftdi | \ ugensa | uipaq | umcs | umct | \ umodem | umoscom | uplcom | usie | \ uslcom | uvisor | uvscom # # USB misc drivers # dev/usb/misc/ufm.c optional ufm dev/usb/misc/udbp.c optional udbp dev/usb/misc/ugold.c optional ugold dev/usb/misc/uled.c optional uled # # USB input drivers # dev/usb/input/atp.c optional atp dev/usb/input/uep.c optional uep dev/usb/input/uhid.c optional uhid dev/usb/input/ukbd.c optional ukbd dev/usb/input/ums.c optional ums dev/usb/input/wmt.c optional wmt dev/usb/input/wsp.c optional wsp # # USB quirks # dev/usb/quirk/usb_quirk.c optional usb # # USB templates # dev/usb/template/usb_template.c optional usb_template dev/usb/template/usb_template_audio.c optional usb_template dev/usb/template/usb_template_cdce.c optional usb_template dev/usb/template/usb_template_kbd.c optional usb_template dev/usb/template/usb_template_modem.c optional usb_template dev/usb/template/usb_template_mouse.c optional usb_template dev/usb/template/usb_template_msc.c optional usb_template dev/usb/template/usb_template_mtp.c optional usb_template dev/usb/template/usb_template_phone.c optional usb_template dev/usb/template/usb_template_serialnet.c optional usb_template dev/usb/template/usb_template_midi.c optional usb_template # # USB video drivers # dev/usb/video/udl.c optional udl # # USB END # dev/videomode/videomode.c optional videomode dev/videomode/edid.c optional videomode dev/videomode/pickmode.c optional videomode dev/videomode/vesagtf.c optional videomode dev/utopia/idtphy.c optional utopia dev/utopia/suni.c optional utopia dev/utopia/utopia.c optional utopia dev/vge/if_vge.c optional vge dev/viapm/viapm.c optional viapm pci dev/virtio/virtio.c optional virtio dev/virtio/virtqueue.c optional virtio dev/virtio/virtio_bus_if.m optional virtio dev/virtio/virtio_if.m optional virtio dev/virtio/pci/virtio_pci.c optional virtio_pci dev/virtio/mmio/virtio_mmio.c optional virtio_mmio dev/virtio/mmio/virtio_mmio_if.m optional virtio_mmio dev/virtio/network/if_vtnet.c optional vtnet dev/virtio/block/virtio_blk.c optional virtio_blk dev/virtio/balloon/virtio_balloon.c optional virtio_balloon dev/virtio/scsi/virtio_scsi.c optional virtio_scsi dev/virtio/random/virtio_random.c optional virtio_random dev/virtio/console/virtio_console.c optional virtio_console dev/vkbd/vkbd.c optional vkbd dev/vr/if_vr.c optional vr pci dev/vt/colors/vt_termcolors.c optional vt dev/vt/font/vt_font_default.c optional vt dev/vt/font/vt_mouse_cursor.c optional vt dev/vt/hw/efifb/efifb.c optional vt_efifb dev/vt/hw/fb/vt_fb.c optional vt dev/vt/hw/vga/vt_vga.c optional vt vt_vga dev/vt/logo/logo_freebsd.c optional vt splash dev/vt/logo/logo_beastie.c optional vt splash dev/vt/vt_buf.c optional vt dev/vt/vt_consolectl.c optional vt dev/vt/vt_core.c optional vt dev/vt/vt_cpulogos.c optional vt splash dev/vt/vt_font.c optional vt dev/vt/vt_sysmouse.c optional vt dev/vte/if_vte.c optional vte pci dev/vx/if_vx.c optional vx dev/vx/if_vx_eisa.c optional vx eisa dev/vx/if_vx_pci.c optional vx pci dev/vxge/vxge.c optional vxge dev/vxge/vxgehal/vxgehal-ifmsg.c optional vxge dev/vxge/vxgehal/vxgehal-mrpcim.c optional vxge dev/vxge/vxgehal/vxge-queue.c optional vxge dev/vxge/vxgehal/vxgehal-ring.c optional vxge dev/vxge/vxgehal/vxgehal-swapper.c optional vxge dev/vxge/vxgehal/vxgehal-mgmt.c optional vxge dev/vxge/vxgehal/vxgehal-srpcim.c optional vxge dev/vxge/vxgehal/vxgehal-config.c optional vxge dev/vxge/vxgehal/vxgehal-blockpool.c optional vxge dev/vxge/vxgehal/vxgehal-doorbells.c optional vxge dev/vxge/vxgehal/vxgehal-mgmtaux.c optional vxge dev/vxge/vxgehal/vxgehal-device.c optional vxge dev/vxge/vxgehal/vxgehal-mm.c optional vxge dev/vxge/vxgehal/vxgehal-driver.c optional vxge dev/vxge/vxgehal/vxgehal-virtualpath.c optional vxge dev/vxge/vxgehal/vxgehal-channel.c optional vxge dev/vxge/vxgehal/vxgehal-fifo.c optional vxge dev/watchdog/watchdog.c standard dev/wb/if_wb.c optional wb pci dev/wds/wd7000.c optional wds isa dev/wi/if_wi.c optional wi dev/wi/if_wi_pccard.c optional wi pccard dev/wi/if_wi_pci.c optional wi pci dev/wl/if_wl.c optional wl isa dev/wpi/if_wpi.c optional wpi pci wpifw.c optional wpifw \ compile-with "${AWK} -f $S/tools/fw_stub.awk wpi.fw:wpifw:153229 -mwpi -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "wpifw.c" wpifw.fwo optional wpifw \ dependency "wpi.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "wpifw.fwo" wpi.fw optional wpifw \ dependency "$S/contrib/dev/wpi/iwlwifi-3945-15.32.2.9.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "wpi.fw" dev/xe/if_xe.c optional xe dev/xe/if_xe_pccard.c optional xe pccard dev/xen/balloon/balloon.c optional xenhvm dev/xen/blkfront/blkfront.c optional xenhvm dev/xen/blkback/blkback.c optional xenhvm dev/xen/console/xen_console.c optional xenhvm dev/xen/control/control.c optional xenhvm dev/xen/grant_table/grant_table.c optional xenhvm dev/xen/netback/netback.c optional xenhvm dev/xen/netfront/netfront.c optional xenhvm dev/xen/xenpci/xenpci.c optional xenpci dev/xen/timer/timer.c optional xenhvm dev/xen/pvcpu/pvcpu.c optional xenhvm dev/xen/xenstore/xenstore.c optional xenhvm dev/xen/xenstore/xenstore_dev.c optional xenhvm dev/xen/xenstore/xenstored_dev.c optional xenhvm dev/xen/evtchn/evtchn_dev.c optional xenhvm dev/xen/privcmd/privcmd.c optional xenhvm dev/xen/debug/debug.c optional xenhvm dev/xl/if_xl.c optional xl pci dev/xl/xlphy.c optional xl pci fs/autofs/autofs.c optional autofs fs/autofs/autofs_vfsops.c optional autofs fs/autofs/autofs_vnops.c optional autofs fs/deadfs/dead_vnops.c standard fs/devfs/devfs_devs.c standard fs/devfs/devfs_dir.c standard fs/devfs/devfs_rule.c standard fs/devfs/devfs_vfsops.c standard fs/devfs/devfs_vnops.c standard fs/fdescfs/fdesc_vfsops.c optional fdescfs fs/fdescfs/fdesc_vnops.c optional fdescfs fs/fifofs/fifo_vnops.c standard fs/cuse/cuse.c optional cuse fs/fuse/fuse_device.c optional fuse fs/fuse/fuse_file.c optional fuse fs/fuse/fuse_internal.c optional fuse fs/fuse/fuse_io.c optional fuse fs/fuse/fuse_ipc.c optional fuse fs/fuse/fuse_main.c optional fuse fs/fuse/fuse_node.c optional fuse fs/fuse/fuse_vfsops.c optional fuse fs/fuse/fuse_vnops.c optional fuse fs/msdosfs/msdosfs_conv.c optional msdosfs fs/msdosfs/msdosfs_denode.c optional msdosfs fs/msdosfs/msdosfs_fat.c optional msdosfs fs/msdosfs/msdosfs_fileno.c optional msdosfs fs/msdosfs/msdosfs_iconv.c optional msdosfs_iconv fs/msdosfs/msdosfs_lookup.c optional msdosfs fs/msdosfs/msdosfs_vfsops.c optional msdosfs fs/msdosfs/msdosfs_vnops.c optional msdosfs fs/nandfs/bmap.c optional nandfs fs/nandfs/nandfs_alloc.c optional nandfs fs/nandfs/nandfs_bmap.c optional nandfs fs/nandfs/nandfs_buffer.c optional nandfs fs/nandfs/nandfs_cleaner.c optional nandfs fs/nandfs/nandfs_cpfile.c optional nandfs fs/nandfs/nandfs_dat.c optional nandfs fs/nandfs/nandfs_dir.c optional nandfs fs/nandfs/nandfs_ifile.c optional nandfs fs/nandfs/nandfs_segment.c optional nandfs fs/nandfs/nandfs_subr.c optional nandfs fs/nandfs/nandfs_sufile.c optional nandfs fs/nandfs/nandfs_vfsops.c optional nandfs fs/nandfs/nandfs_vnops.c optional nandfs fs/nfs/nfs_commonkrpc.c optional nfscl | nfsd fs/nfs/nfs_commonsubs.c optional nfscl | nfsd fs/nfs/nfs_commonport.c optional nfscl | nfsd fs/nfs/nfs_commonacl.c optional nfscl | nfsd fs/nfsclient/nfs_clcomsubs.c optional nfscl fs/nfsclient/nfs_clsubs.c optional nfscl fs/nfsclient/nfs_clstate.c optional nfscl fs/nfsclient/nfs_clkrpc.c optional nfscl fs/nfsclient/nfs_clrpcops.c optional nfscl fs/nfsclient/nfs_clvnops.c optional nfscl fs/nfsclient/nfs_clnode.c optional nfscl fs/nfsclient/nfs_clvfsops.c optional nfscl fs/nfsclient/nfs_clport.c optional nfscl fs/nfsclient/nfs_clbio.c optional nfscl fs/nfsclient/nfs_clnfsiod.c optional nfscl fs/nfsserver/nfs_fha_new.c optional nfsd inet fs/nfsserver/nfs_nfsdsocket.c optional nfsd inet fs/nfsserver/nfs_nfsdsubs.c optional nfsd inet fs/nfsserver/nfs_nfsdstate.c optional nfsd inet fs/nfsserver/nfs_nfsdkrpc.c optional nfsd inet fs/nfsserver/nfs_nfsdserv.c optional nfsd inet fs/nfsserver/nfs_nfsdport.c optional nfsd inet fs/nfsserver/nfs_nfsdcache.c optional nfsd inet fs/nullfs/null_subr.c optional nullfs fs/nullfs/null_vfsops.c optional nullfs fs/nullfs/null_vnops.c optional nullfs fs/procfs/procfs.c optional procfs fs/procfs/procfs_ctl.c optional procfs fs/procfs/procfs_dbregs.c optional procfs fs/procfs/procfs_fpregs.c optional procfs fs/procfs/procfs_ioctl.c optional procfs fs/procfs/procfs_map.c optional procfs fs/procfs/procfs_mem.c optional procfs fs/procfs/procfs_note.c optional procfs fs/procfs/procfs_osrel.c optional procfs fs/procfs/procfs_regs.c optional procfs fs/procfs/procfs_rlimit.c optional procfs fs/procfs/procfs_status.c optional procfs fs/procfs/procfs_type.c optional procfs fs/pseudofs/pseudofs.c optional pseudofs fs/pseudofs/pseudofs_fileno.c optional pseudofs fs/pseudofs/pseudofs_vncache.c optional pseudofs fs/pseudofs/pseudofs_vnops.c optional pseudofs fs/smbfs/smbfs_io.c optional smbfs fs/smbfs/smbfs_node.c optional smbfs fs/smbfs/smbfs_smb.c optional smbfs fs/smbfs/smbfs_subr.c optional smbfs fs/smbfs/smbfs_vfsops.c optional smbfs fs/smbfs/smbfs_vnops.c optional smbfs fs/udf/osta.c optional udf fs/udf/udf_iconv.c optional udf_iconv fs/udf/udf_vfsops.c optional udf fs/udf/udf_vnops.c optional udf fs/unionfs/union_subr.c optional unionfs fs/unionfs/union_vfsops.c optional unionfs fs/unionfs/union_vnops.c optional unionfs fs/tmpfs/tmpfs_vnops.c optional tmpfs fs/tmpfs/tmpfs_fifoops.c optional tmpfs fs/tmpfs/tmpfs_vfsops.c optional tmpfs fs/tmpfs/tmpfs_subr.c optional tmpfs gdb/gdb_cons.c optional gdb gdb/gdb_main.c optional gdb gdb/gdb_packet.c optional gdb geom/bde/g_bde.c optional geom_bde geom/bde/g_bde_crypt.c optional geom_bde geom/bde/g_bde_lock.c optional geom_bde geom/bde/g_bde_work.c optional geom_bde geom/cache/g_cache.c optional geom_cache geom/concat/g_concat.c optional geom_concat geom/eli/g_eli.c optional geom_eli geom/eli/g_eli_crypto.c optional geom_eli geom/eli/g_eli_ctl.c optional geom_eli geom/eli/g_eli_hmac.c optional geom_eli geom/eli/g_eli_integrity.c optional geom_eli geom/eli/g_eli_key.c optional geom_eli geom/eli/g_eli_key_cache.c optional geom_eli geom/eli/g_eli_privacy.c optional geom_eli geom/eli/pkcs5v2.c optional geom_eli geom/gate/g_gate.c optional geom_gate geom/geom_aes.c optional geom_aes geom/geom_bsd.c optional geom_bsd geom/geom_bsd_enc.c optional geom_bsd | geom_part_bsd geom/geom_ccd.c optional ccd | geom_ccd geom/geom_ctl.c standard geom/geom_dev.c standard geom/geom_disk.c standard geom/geom_dump.c standard geom/geom_event.c standard geom/geom_fox.c optional geom_fox geom/geom_flashmap.c optional fdt cfi | fdt nand | fdt mx25l | mmcsd geom/geom_io.c standard geom/geom_kern.c standard geom/geom_map.c optional geom_map geom/geom_mbr.c optional geom_mbr geom/geom_mbr_enc.c optional geom_mbr geom/geom_pc98.c optional geom_pc98 geom/geom_pc98_enc.c optional geom_pc98 geom/geom_redboot.c optional geom_redboot geom/geom_slice.c standard geom/geom_subr.c standard geom/geom_sunlabel.c optional geom_sunlabel geom/geom_sunlabel_enc.c optional geom_sunlabel geom/geom_vfs.c standard geom/geom_vol_ffs.c optional geom_vol geom/journal/g_journal.c optional geom_journal geom/journal/g_journal_ufs.c optional geom_journal geom/label/g_label.c optional geom_label | geom_label_gpt geom/label/g_label_ext2fs.c optional geom_label geom/label/g_label_iso9660.c optional geom_label geom/label/g_label_msdosfs.c optional geom_label geom/label/g_label_ntfs.c optional geom_label geom/label/g_label_reiserfs.c optional geom_label geom/label/g_label_ufs.c optional geom_label geom/label/g_label_gpt.c optional geom_label | geom_label_gpt geom/label/g_label_disk_ident.c optional geom_label geom/linux_lvm/g_linux_lvm.c optional geom_linux_lvm geom/mirror/g_mirror.c optional geom_mirror geom/mirror/g_mirror_ctl.c optional geom_mirror geom/mountver/g_mountver.c optional geom_mountver geom/multipath/g_multipath.c optional geom_multipath geom/nop/g_nop.c optional geom_nop geom/part/g_part.c standard geom/part/g_part_if.m standard geom/part/g_part_apm.c optional geom_part_apm geom/part/g_part_bsd.c optional geom_part_bsd geom/part/g_part_bsd64.c optional geom_part_bsd64 geom/part/g_part_ebr.c optional geom_part_ebr geom/part/g_part_gpt.c optional geom_part_gpt geom/part/g_part_ldm.c optional geom_part_ldm geom/part/g_part_mbr.c optional geom_part_mbr geom/part/g_part_pc98.c optional geom_part_pc98 geom/part/g_part_vtoc8.c optional geom_part_vtoc8 geom/raid/g_raid.c optional geom_raid geom/raid/g_raid_ctl.c optional geom_raid geom/raid/g_raid_md_if.m optional geom_raid geom/raid/g_raid_tr_if.m optional geom_raid geom/raid/md_ddf.c optional geom_raid geom/raid/md_intel.c optional geom_raid geom/raid/md_jmicron.c optional geom_raid geom/raid/md_nvidia.c optional geom_raid geom/raid/md_promise.c optional geom_raid geom/raid/md_sii.c optional geom_raid geom/raid/tr_concat.c optional geom_raid geom/raid/tr_raid0.c optional geom_raid geom/raid/tr_raid1.c optional geom_raid geom/raid/tr_raid1e.c optional geom_raid geom/raid/tr_raid5.c optional geom_raid geom/raid3/g_raid3.c optional geom_raid3 geom/raid3/g_raid3_ctl.c optional geom_raid3 geom/shsec/g_shsec.c optional geom_shsec geom/stripe/g_stripe.c optional geom_stripe contrib/xz-embedded/freebsd/xz_malloc.c \ optional xz_embedded | geom_uzip \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_crc32.c \ optional xz_embedded | geom_uzip \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_dec_bcj.c \ optional xz_embedded | geom_uzip \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_dec_lzma2.c \ optional xz_embedded | geom_uzip \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_dec_stream.c \ optional xz_embedded | geom_uzip \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" geom/uzip/g_uzip.c optional geom_uzip geom/uzip/g_uzip_lzma.c optional geom_uzip geom/uzip/g_uzip_wrkthr.c optional geom_uzip geom/uzip/g_uzip_zlib.c optional geom_uzip geom/vinum/geom_vinum.c optional geom_vinum geom/vinum/geom_vinum_create.c optional geom_vinum geom/vinum/geom_vinum_drive.c optional geom_vinum geom/vinum/geom_vinum_plex.c optional geom_vinum geom/vinum/geom_vinum_volume.c optional geom_vinum geom/vinum/geom_vinum_subr.c optional geom_vinum geom/vinum/geom_vinum_raid5.c optional geom_vinum geom/vinum/geom_vinum_share.c optional geom_vinum geom/vinum/geom_vinum_list.c optional geom_vinum geom/vinum/geom_vinum_rm.c optional geom_vinum geom/vinum/geom_vinum_init.c optional geom_vinum geom/vinum/geom_vinum_state.c optional geom_vinum geom/vinum/geom_vinum_rename.c optional geom_vinum geom/vinum/geom_vinum_move.c optional geom_vinum geom/vinum/geom_vinum_events.c optional geom_vinum geom/virstor/binstream.c optional geom_virstor geom/virstor/g_virstor.c optional geom_virstor geom/virstor/g_virstor_md.c optional geom_virstor geom/zero/g_zero.c optional geom_zero fs/ext2fs/ext2_acl.c optional ext2fs fs/ext2fs/ext2_alloc.c optional ext2fs fs/ext2fs/ext2_balloc.c optional ext2fs fs/ext2fs/ext2_bmap.c optional ext2fs fs/ext2fs/ext2_csum.c optional ext2fs fs/ext2fs/ext2_extattr.c optional ext2fs fs/ext2fs/ext2_extents.c optional ext2fs fs/ext2fs/ext2_inode.c optional ext2fs fs/ext2fs/ext2_inode_cnv.c optional ext2fs fs/ext2fs/ext2_hash.c optional ext2fs fs/ext2fs/ext2_htree.c optional ext2fs fs/ext2fs/ext2_lookup.c optional ext2fs fs/ext2fs/ext2_subr.c optional ext2fs fs/ext2fs/ext2_vfsops.c optional ext2fs fs/ext2fs/ext2_vnops.c optional ext2fs # isa/isa_if.m standard isa/isa_common.c optional isa isa/isahint.c optional isa isa/pnp.c optional isa isapnp isa/pnpparse.c optional isa isapnp fs/cd9660/cd9660_bmap.c optional cd9660 fs/cd9660/cd9660_lookup.c optional cd9660 fs/cd9660/cd9660_node.c optional cd9660 fs/cd9660/cd9660_rrip.c optional cd9660 fs/cd9660/cd9660_util.c optional cd9660 fs/cd9660/cd9660_vfsops.c optional cd9660 fs/cd9660/cd9660_vnops.c optional cd9660 fs/cd9660/cd9660_iconv.c optional cd9660_iconv kern/bus_if.m standard kern/clock_if.m standard kern/cpufreq_if.m standard kern/device_if.m standard kern/imgact_binmisc.c optional imagact_binmisc kern/imgact_elf.c standard kern/imgact_elf32.c optional compat_freebsd32 kern/imgact_shell.c standard kern/inflate.c optional gzip kern/init_main.c standard kern/init_sysent.c standard kern/ksched.c optional _kposix_priority_scheduling kern/kern_acct.c standard kern/kern_alq.c optional alq kern/kern_clock.c standard kern/kern_condvar.c standard kern/kern_conf.c standard kern/kern_cons.c standard kern/kern_cpu.c standard kern/kern_cpuset.c standard kern/kern_context.c standard kern/kern_descrip.c standard kern/kern_dtrace.c optional kdtrace_hooks kern/kern_dump.c standard kern/kern_environment.c standard kern/kern_et.c standard kern/kern_event.c standard kern/kern_exec.c standard kern/kern_exit.c standard kern/kern_fail.c standard kern/kern_ffclock.c standard kern/kern_fork.c standard kern/kern_gzio.c optional gzio kern/kern_hhook.c standard kern/kern_idle.c standard kern/kern_intr.c standard kern/kern_jail.c standard kern/kern_khelp.c standard kern/kern_kthread.c standard kern/kern_ktr.c optional ktr kern/kern_ktrace.c standard kern/kern_linker.c standard kern/kern_lock.c standard kern/kern_lockf.c standard kern/kern_lockstat.c optional kdtrace_hooks kern/kern_loginclass.c standard kern/kern_malloc.c standard kern/kern_mbuf.c standard kern/kern_mib.c standard kern/kern_module.c standard kern/kern_mtxpool.c standard kern/kern_mutex.c standard kern/kern_ntptime.c standard kern/kern_numa.c standard kern/kern_osd.c standard kern/kern_physio.c standard kern/kern_pmc.c standard kern/kern_poll.c optional device_polling kern/kern_priv.c standard kern/kern_proc.c standard kern/kern_procctl.c standard kern/kern_prot.c standard kern/kern_racct.c standard kern/kern_rangelock.c standard kern/kern_rctl.c standard kern/kern_resource.c standard kern/kern_rmlock.c standard kern/kern_rwlock.c standard kern/kern_sdt.c optional kdtrace_hooks kern/kern_sema.c standard kern/kern_sendfile.c standard kern/kern_sharedpage.c standard kern/kern_shutdown.c standard kern/kern_sig.c standard kern/kern_switch.c standard kern/kern_sx.c standard kern/kern_synch.c standard kern/kern_syscalls.c standard kern/kern_sysctl.c standard kern/kern_tc.c standard kern/kern_thr.c standard kern/kern_thread.c standard kern/kern_time.c standard kern/kern_timeout.c standard kern/kern_umtx.c standard kern/kern_uuid.c standard kern/kern_xxx.c standard kern/link_elf.c standard kern/linker_if.m standard kern/md4c.c optional netsmb kern/md5c.c standard kern/p1003_1b.c standard kern/posix4_mib.c standard kern/sched_4bsd.c optional sched_4bsd kern/sched_ule.c optional sched_ule kern/serdev_if.m standard kern/stack_protector.c standard \ compile-with "${NORMAL_C:N-fstack-protector*}" kern/subr_acl_nfs4.c optional ufs_acl | zfs kern/subr_acl_posix1e.c optional ufs_acl kern/subr_autoconf.c standard kern/subr_blist.c standard kern/subr_bus.c standard kern/subr_bus_dma.c standard kern/subr_bufring.c standard kern/subr_capability.c standard kern/subr_clock.c standard kern/subr_counter.c standard kern/subr_devstat.c standard kern/subr_disk.c standard kern/subr_eventhandler.c standard kern/subr_fattime.c standard kern/subr_firmware.c optional firmware kern/subr_gtaskqueue.c standard kern/subr_hash.c standard kern/subr_hints.c standard kern/subr_kdb.c standard kern/subr_kobj.c standard kern/subr_lock.c standard kern/subr_log.c standard kern/subr_mbpool.c optional libmbpool kern/subr_mchain.c optional libmchain kern/subr_module.c standard kern/subr_msgbuf.c standard kern/subr_param.c standard kern/subr_pcpu.c standard kern/subr_pctrie.c standard kern/subr_power.c standard kern/subr_prf.c standard kern/subr_prof.c standard kern/subr_rman.c standard kern/subr_rtc.c standard kern/subr_sbuf.c standard kern/subr_scanf.c standard kern/subr_sglist.c standard kern/subr_sleepqueue.c standard kern/subr_smp.c standard kern/subr_stack.c optional ddb | stack | ktr kern/subr_taskqueue.c standard kern/subr_terminal.c optional vt kern/subr_trap.c standard kern/subr_turnstile.c standard kern/subr_uio.c standard kern/subr_unit.c standard kern/subr_vmem.c standard kern/subr_witness.c optional witness kern/sys_capability.c standard kern/sys_generic.c standard kern/sys_pipe.c standard kern/sys_procdesc.c standard kern/sys_process.c standard kern/sys_socket.c standard kern/syscalls.c standard kern/sysv_ipc.c standard kern/sysv_msg.c optional sysvmsg kern/sysv_sem.c optional sysvsem kern/sysv_shm.c optional sysvshm kern/tty.c standard kern/tty_compat.c optional compat_43tty kern/tty_info.c standard kern/tty_inq.c standard kern/tty_outq.c standard kern/tty_pts.c standard kern/tty_tty.c standard kern/tty_ttydisc.c standard kern/uipc_accf.c standard kern/uipc_debug.c optional ddb kern/uipc_domain.c standard kern/uipc_mbuf.c standard kern/uipc_mbuf2.c standard kern/uipc_mbufhash.c standard kern/uipc_mqueue.c optional p1003_1b_mqueue kern/uipc_sem.c optional p1003_1b_semaphores kern/uipc_shm.c standard kern/uipc_sockbuf.c standard kern/uipc_socket.c standard kern/uipc_syscalls.c standard kern/uipc_usrreq.c standard kern/vfs_acl.c standard kern/vfs_aio.c standard kern/vfs_bio.c standard kern/vfs_cache.c standard kern/vfs_cluster.c standard kern/vfs_default.c standard kern/vfs_export.c standard kern/vfs_extattr.c standard kern/vfs_hash.c standard kern/vfs_init.c standard kern/vfs_lookup.c standard kern/vfs_mount.c standard kern/vfs_mountroot.c standard kern/vfs_subr.c standard kern/vfs_syscalls.c standard kern/vfs_vnops.c standard # # Kernel GSS-API # gssd.h optional kgssapi \ dependency "$S/kgssapi/gssd.x" \ compile-with "RPCGEN_CPP='${CPP}' rpcgen -hM $S/kgssapi/gssd.x | grep -v pthread.h > gssd.h" \ no-obj no-implicit-rule before-depend local \ clean "gssd.h" gssd_xdr.c optional kgssapi \ dependency "$S/kgssapi/gssd.x gssd.h" \ compile-with "RPCGEN_CPP='${CPP}' rpcgen -c $S/kgssapi/gssd.x -o gssd_xdr.c" \ no-implicit-rule before-depend local \ clean "gssd_xdr.c" gssd_clnt.c optional kgssapi \ dependency "$S/kgssapi/gssd.x gssd.h" \ compile-with "RPCGEN_CPP='${CPP}' rpcgen -lM $S/kgssapi/gssd.x | grep -v string.h > gssd_clnt.c" \ no-implicit-rule before-depend local \ clean "gssd_clnt.c" kgssapi/gss_accept_sec_context.c optional kgssapi kgssapi/gss_add_oid_set_member.c optional kgssapi kgssapi/gss_acquire_cred.c optional kgssapi kgssapi/gss_canonicalize_name.c optional kgssapi kgssapi/gss_create_empty_oid_set.c optional kgssapi kgssapi/gss_delete_sec_context.c optional kgssapi kgssapi/gss_display_status.c optional kgssapi kgssapi/gss_export_name.c optional kgssapi kgssapi/gss_get_mic.c optional kgssapi kgssapi/gss_init_sec_context.c optional kgssapi kgssapi/gss_impl.c optional kgssapi kgssapi/gss_import_name.c optional kgssapi kgssapi/gss_names.c optional kgssapi kgssapi/gss_pname_to_uid.c optional kgssapi kgssapi/gss_release_buffer.c optional kgssapi kgssapi/gss_release_cred.c optional kgssapi kgssapi/gss_release_name.c optional kgssapi kgssapi/gss_release_oid_set.c optional kgssapi kgssapi/gss_set_cred_option.c optional kgssapi kgssapi/gss_test_oid_set_member.c optional kgssapi kgssapi/gss_unwrap.c optional kgssapi kgssapi/gss_verify_mic.c optional kgssapi kgssapi/gss_wrap.c optional kgssapi kgssapi/gss_wrap_size_limit.c optional kgssapi kgssapi/gssd_prot.c optional kgssapi kgssapi/krb5/krb5_mech.c optional kgssapi kgssapi/krb5/kcrypto.c optional kgssapi kgssapi/krb5/kcrypto_aes.c optional kgssapi kgssapi/krb5/kcrypto_arcfour.c optional kgssapi kgssapi/krb5/kcrypto_des.c optional kgssapi kgssapi/krb5/kcrypto_des3.c optional kgssapi kgssapi/kgss_if.m optional kgssapi kgssapi/gsstest.c optional kgssapi_debug # These files in libkern/ are those needed by all architectures. Some # of the files in libkern/ are only needed on some architectures, e.g., # libkern/divdi3.c is needed by i386 but not alpha. Also, some of these # routines may be optimized for a particular platform. In either case, # the file should be moved to conf/files. from here. # libkern/arc4random.c standard libkern/asprintf.c standard libkern/bcd.c standard libkern/bsearch.c standard libkern/crc32.c standard libkern/explicit_bzero.c standard libkern/fnmatch.c standard libkern/iconv.c optional libiconv libkern/iconv_converter_if.m optional libiconv libkern/iconv_ucs.c optional libiconv libkern/iconv_xlat.c optional libiconv libkern/iconv_xlat16.c optional libiconv libkern/inet_aton.c standard libkern/inet_ntoa.c standard libkern/inet_ntop.c standard libkern/inet_pton.c standard libkern/jenkins_hash.c standard libkern/murmur3_32.c standard libkern/mcount.c optional profiling-routine libkern/memcchr.c standard libkern/memchr.c standard libkern/memcmp.c standard libkern/memmem.c optional gdb libkern/qsort.c standard libkern/qsort_r.c standard libkern/random.c standard libkern/scanc.c standard libkern/strcasecmp.c standard libkern/strcat.c standard libkern/strchr.c standard libkern/strcmp.c standard libkern/strcpy.c standard libkern/strcspn.c standard libkern/strdup.c standard libkern/strndup.c standard libkern/strlcat.c standard libkern/strlcpy.c standard libkern/strlen.c standard libkern/strncat.c standard libkern/strncmp.c standard libkern/strncpy.c standard libkern/strnlen.c standard libkern/strrchr.c standard libkern/strsep.c standard libkern/strspn.c standard libkern/strstr.c standard libkern/strtol.c standard libkern/strtoq.c standard libkern/strtoul.c standard libkern/strtouq.c standard libkern/strvalid.c standard libkern/timingsafe_bcmp.c standard libkern/zlib.c optional crypto | geom_uzip | ipsec | \ ipsec_support | mxge | netgraph_deflate | ddb_ctf | gzio net/altq/altq_cbq.c optional altq net/altq/altq_cdnr.c optional altq net/altq/altq_codel.c optional altq net/altq/altq_hfsc.c optional altq net/altq/altq_fairq.c optional altq net/altq/altq_priq.c optional altq net/altq/altq_red.c optional altq net/altq/altq_rio.c optional altq net/altq/altq_rmclass.c optional altq net/altq/altq_subr.c optional altq net/bpf.c standard net/bpf_buffer.c optional bpf net/bpf_jitter.c optional bpf_jitter net/bpf_filter.c optional bpf | netgraph_bpf net/bpf_zerocopy.c optional bpf net/bridgestp.c optional bridge | if_bridge net/flowtable.c optional flowtable inet | flowtable inet6 net/ieee8023ad_lacp.c optional lagg net/if.c standard net/if_arcsubr.c optional arcnet net/if_atmsubr.c optional atm net/if_bridge.c optional bridge inet | if_bridge inet net/if_clone.c standard net/if_dead.c standard net/if_debug.c optional ddb net/if_disc.c optional disc net/if_edsc.c optional edsc net/if_enc.c optional enc inet | enc inet6 net/if_epair.c optional epair net/if_ethersubr.c optional ether net/if_fddisubr.c optional fddi net/if_fwsubr.c optional fwip net/if_gif.c optional gif inet | gif inet6 | \ netgraph_gif inet | netgraph_gif inet6 net/if_gre.c optional gre inet | gre inet6 net/if_ipsec.c optional inet ipsec | inet6 ipsec net/if_iso88025subr.c optional token net/if_lagg.c optional lagg net/if_loop.c optional loop net/if_llatbl.c standard net/if_me.c optional me inet net/if_media.c standard net/if_mib.c standard net/if_spppfr.c optional sppp | netgraph_sppp net/if_spppsubr.c optional sppp | netgraph_sppp net/if_stf.c optional stf inet inet6 net/if_tun.c optional tun net/if_tap.c optional tap net/if_vlan.c optional vlan net/if_vxlan.c optional vxlan inet | vxlan inet6 net/ifdi_if.m optional ether pci net/iflib.c optional ether pci net/mp_ring.c optional ether net/mppcc.c optional netgraph_mppc_compression net/mppcd.c optional netgraph_mppc_compression net/netisr.c standard net/pfil.c optional ether | inet net/radix.c standard net/radix_mpath.c standard net/raw_cb.c standard net/raw_usrreq.c standard net/route.c standard net/rss_config.c optional inet rss | inet6 rss net/rtsock.c standard net/slcompress.c optional netgraph_vjc | sppp | \ netgraph_sppp net/toeplitz.c optional inet rss | inet6 rss net/vnet.c optional vimage net80211/ieee80211.c optional wlan net80211/ieee80211_acl.c optional wlan wlan_acl net80211/ieee80211_action.c optional wlan net80211/ieee80211_ageq.c optional wlan net80211/ieee80211_adhoc.c optional wlan \ compile-with "${NORMAL_C} -Wno-unused-function" net80211/ieee80211_ageq.c optional wlan net80211/ieee80211_amrr.c optional wlan | wlan_amrr net80211/ieee80211_crypto.c optional wlan \ compile-with "${NORMAL_C} -Wno-unused-function" net80211/ieee80211_crypto_ccmp.c optional wlan wlan_ccmp net80211/ieee80211_crypto_none.c optional wlan net80211/ieee80211_crypto_tkip.c optional wlan wlan_tkip net80211/ieee80211_crypto_wep.c optional wlan wlan_wep net80211/ieee80211_ddb.c optional wlan ddb net80211/ieee80211_dfs.c optional wlan net80211/ieee80211_freebsd.c optional wlan net80211/ieee80211_hostap.c optional wlan \ compile-with "${NORMAL_C} -Wno-unused-function" net80211/ieee80211_ht.c optional wlan net80211/ieee80211_hwmp.c optional wlan ieee80211_support_mesh net80211/ieee80211_input.c optional wlan net80211/ieee80211_ioctl.c optional wlan net80211/ieee80211_mesh.c optional wlan ieee80211_support_mesh \ compile-with "${NORMAL_C} -Wno-unused-function" net80211/ieee80211_monitor.c optional wlan net80211/ieee80211_node.c optional wlan net80211/ieee80211_output.c optional wlan net80211/ieee80211_phy.c optional wlan net80211/ieee80211_power.c optional wlan net80211/ieee80211_proto.c optional wlan net80211/ieee80211_radiotap.c optional wlan net80211/ieee80211_ratectl.c optional wlan net80211/ieee80211_ratectl_none.c optional wlan net80211/ieee80211_regdomain.c optional wlan net80211/ieee80211_rssadapt.c optional wlan wlan_rssadapt net80211/ieee80211_scan.c optional wlan net80211/ieee80211_scan_sta.c optional wlan net80211/ieee80211_sta.c optional wlan \ compile-with "${NORMAL_C} -Wno-unused-function" net80211/ieee80211_superg.c optional wlan ieee80211_support_superg net80211/ieee80211_scan_sw.c optional wlan net80211/ieee80211_tdma.c optional wlan ieee80211_support_tdma net80211/ieee80211_wds.c optional wlan net80211/ieee80211_xauth.c optional wlan wlan_xauth net80211/ieee80211_alq.c optional wlan ieee80211_alq netgraph/atm/ccatm/ng_ccatm.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/atm/ng_atm.c optional ngatm_atm netgraph/atm/ngatmbase.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/atm/sscfu/ng_sscfu.c optional ngatm_sscfu \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/atm/sscop/ng_sscop.c optional ngatm_sscop \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/atm/uni/ng_uni.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/bluetooth/common/ng_bluetooth.c optional netgraph_bluetooth netgraph/bluetooth/drivers/bt3c/ng_bt3c_pccard.c optional netgraph_bluetooth_bt3c netgraph/bluetooth/drivers/h4/ng_h4.c optional netgraph_bluetooth_h4 netgraph/bluetooth/drivers/ubt/ng_ubt.c optional netgraph_bluetooth_ubt usb netgraph/bluetooth/drivers/ubtbcmfw/ubtbcmfw.c optional netgraph_bluetooth_ubtbcmfw usb netgraph/bluetooth/hci/ng_hci_cmds.c optional netgraph_bluetooth_hci netgraph/bluetooth/hci/ng_hci_evnt.c optional netgraph_bluetooth_hci netgraph/bluetooth/hci/ng_hci_main.c optional netgraph_bluetooth_hci netgraph/bluetooth/hci/ng_hci_misc.c optional netgraph_bluetooth_hci netgraph/bluetooth/hci/ng_hci_ulpi.c optional netgraph_bluetooth_hci netgraph/bluetooth/l2cap/ng_l2cap_cmds.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_evnt.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_llpi.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_main.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_misc.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_ulpi.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/socket/ng_btsocket.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_hci_raw.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_l2cap.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_l2cap_raw.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_rfcomm.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_sco.c optional netgraph_bluetooth_socket netgraph/netflow/netflow.c optional netgraph_netflow netgraph/netflow/netflow_v9.c optional netgraph_netflow netgraph/netflow/ng_netflow.c optional netgraph_netflow netgraph/ng_UI.c optional netgraph_UI netgraph/ng_async.c optional netgraph_async netgraph/ng_atmllc.c optional netgraph_atmllc netgraph/ng_base.c optional netgraph netgraph/ng_bpf.c optional netgraph_bpf netgraph/ng_bridge.c optional netgraph_bridge netgraph/ng_car.c optional netgraph_car netgraph/ng_cisco.c optional netgraph_cisco netgraph/ng_deflate.c optional netgraph_deflate netgraph/ng_device.c optional netgraph_device netgraph/ng_echo.c optional netgraph_echo netgraph/ng_eiface.c optional netgraph_eiface netgraph/ng_ether.c optional netgraph_ether netgraph/ng_ether_echo.c optional netgraph_ether_echo netgraph/ng_frame_relay.c optional netgraph_frame_relay netgraph/ng_gif.c optional netgraph_gif inet6 | netgraph_gif inet netgraph/ng_gif_demux.c optional netgraph_gif_demux netgraph/ng_hole.c optional netgraph_hole netgraph/ng_iface.c optional netgraph_iface netgraph/ng_ip_input.c optional netgraph_ip_input netgraph/ng_ipfw.c optional netgraph_ipfw inet ipfirewall netgraph/ng_ksocket.c optional netgraph_ksocket netgraph/ng_l2tp.c optional netgraph_l2tp netgraph/ng_lmi.c optional netgraph_lmi netgraph/ng_mppc.c optional netgraph_mppc_compression | \ netgraph_mppc_encryption netgraph/ng_nat.c optional netgraph_nat inet libalias netgraph/ng_one2many.c optional netgraph_one2many netgraph/ng_parse.c optional netgraph netgraph/ng_patch.c optional netgraph_patch netgraph/ng_pipe.c optional netgraph_pipe netgraph/ng_ppp.c optional netgraph_ppp netgraph/ng_pppoe.c optional netgraph_pppoe netgraph/ng_pptpgre.c optional netgraph_pptpgre netgraph/ng_pred1.c optional netgraph_pred1 netgraph/ng_rfc1490.c optional netgraph_rfc1490 netgraph/ng_socket.c optional netgraph_socket netgraph/ng_split.c optional netgraph_split netgraph/ng_sppp.c optional netgraph_sppp netgraph/ng_tag.c optional netgraph_tag netgraph/ng_tcpmss.c optional netgraph_tcpmss netgraph/ng_tee.c optional netgraph_tee netgraph/ng_tty.c optional netgraph_tty netgraph/ng_vjc.c optional netgraph_vjc netgraph/ng_vlan.c optional netgraph_vlan netinet/accf_data.c optional accept_filter_data inet netinet/accf_dns.c optional accept_filter_dns inet netinet/accf_http.c optional accept_filter_http inet netinet/if_atm.c optional atm netinet/if_ether.c optional inet ether netinet/igmp.c optional inet netinet/in.c optional inet netinet/in_debug.c optional inet ddb netinet/in_kdtrace.c optional inet | inet6 netinet/ip_carp.c optional inet carp | inet6 carp netinet/in_fib.c optional inet netinet/in_gif.c optional gif inet | netgraph_gif inet netinet/ip_gre.c optional gre inet netinet/ip_id.c optional inet netinet/in_jail.c optional inet netinet/in_mcast.c optional inet netinet/in_pcb.c optional inet | inet6 netinet/in_pcbgroup.c optional inet pcbgroup | inet6 pcbgroup netinet/in_proto.c optional inet | inet6 netinet/in_rmx.c optional inet netinet/in_rss.c optional inet rss netinet/ip_divert.c optional inet ipdivert ipfirewall netinet/ip_ecn.c optional inet | inet6 netinet/ip_encap.c optional inet | inet6 netinet/ip_fastfwd.c optional inet netinet/ip_icmp.c optional inet | inet6 netinet/ip_input.c optional inet netinet/ip_mroute.c optional mrouting inet netinet/ip_options.c optional inet netinet/ip_output.c optional inet netinet/ip_reass.c optional inet netinet/raw_ip.c optional inet | inet6 netinet/cc/cc.c optional inet | inet6 netinet/cc/cc_newreno.c optional inet | inet6 netinet/sctp_asconf.c optional inet sctp | inet6 sctp netinet/sctp_auth.c optional inet sctp | inet6 sctp netinet/sctp_bsd_addr.c optional inet sctp | inet6 sctp netinet/sctp_cc_functions.c optional inet sctp | inet6 sctp netinet/sctp_crc32.c optional inet sctp | inet6 sctp netinet/sctp_indata.c optional inet sctp | inet6 sctp netinet/sctp_input.c optional inet sctp | inet6 sctp netinet/sctp_output.c optional inet sctp | inet6 sctp netinet/sctp_pcb.c optional inet sctp | inet6 sctp netinet/sctp_peeloff.c optional inet sctp | inet6 sctp netinet/sctp_ss_functions.c optional inet sctp | inet6 sctp netinet/sctp_syscalls.c optional inet sctp | inet6 sctp netinet/sctp_sysctl.c optional inet sctp | inet6 sctp netinet/sctp_timer.c optional inet sctp | inet6 sctp netinet/sctp_usrreq.c optional inet sctp | inet6 sctp netinet/sctputil.c optional inet sctp | inet6 sctp netinet/siftr.c optional inet siftr alq | inet6 siftr alq netinet/tcp_debug.c optional tcpdebug netinet/tcp_fastopen.c optional inet tcp_rfc7413 | inet6 tcp_rfc7413 netinet/tcp_hostcache.c optional inet | inet6 netinet/tcp_input.c optional inet | inet6 netinet/tcp_lro.c optional inet | inet6 netinet/tcp_output.c optional inet | inet6 netinet/tcp_offload.c optional tcp_offload inet | tcp_offload inet6 netinet/tcp_pcap.c optional inet tcppcap | inet6 tcppcap netinet/tcp_reass.c optional inet | inet6 netinet/tcp_sack.c optional inet | inet6 netinet/tcp_subr.c optional inet | inet6 netinet/tcp_syncache.c optional inet | inet6 netinet/tcp_timer.c optional inet | inet6 netinet/tcp_timewait.c optional inet | inet6 netinet/tcp_usrreq.c optional inet | inet6 netinet/udp_usrreq.c optional inet | inet6 netinet/libalias/alias.c optional libalias inet | netgraph_nat inet netinet/libalias/alias_db.c optional libalias inet | netgraph_nat inet netinet/libalias/alias_mod.c optional libalias | netgraph_nat netinet/libalias/alias_proxy.c optional libalias inet | netgraph_nat inet netinet/libalias/alias_util.c optional libalias inet | netgraph_nat inet netinet/libalias/alias_sctp.c optional libalias inet | netgraph_nat inet netinet6/dest6.c optional inet6 netinet6/frag6.c optional inet6 netinet6/icmp6.c optional inet6 netinet6/in6.c optional inet6 netinet6/in6_cksum.c optional inet6 netinet6/in6_fib.c optional inet6 netinet6/in6_gif.c optional gif inet6 | netgraph_gif inet6 netinet6/in6_ifattach.c optional inet6 netinet6/in6_jail.c optional inet6 netinet6/in6_mcast.c optional inet6 netinet6/in6_pcb.c optional inet6 netinet6/in6_pcbgroup.c optional inet6 pcbgroup netinet6/in6_proto.c optional inet6 netinet6/in6_rmx.c optional inet6 netinet6/in6_rss.c optional inet6 rss netinet6/in6_src.c optional inet6 netinet6/ip6_fastfwd.c optional inet6 netinet6/ip6_forward.c optional inet6 netinet6/ip6_gre.c optional gre inet6 netinet6/ip6_id.c optional inet6 netinet6/ip6_input.c optional inet6 netinet6/ip6_mroute.c optional mrouting inet6 netinet6/ip6_output.c optional inet6 netinet6/mld6.c optional inet6 netinet6/nd6.c optional inet6 netinet6/nd6_nbr.c optional inet6 netinet6/nd6_rtr.c optional inet6 netinet6/raw_ip6.c optional inet6 netinet6/route6.c optional inet6 netinet6/scope6.c optional inet6 netinet6/sctp6_usrreq.c optional inet6 sctp netinet6/udp6_usrreq.c optional inet6 netipsec/ipsec.c optional ipsec inet | ipsec inet6 netipsec/ipsec_input.c optional ipsec inet | ipsec inet6 netipsec/ipsec_mbuf.c optional ipsec inet | ipsec inet6 netipsec/ipsec_mod.c optional ipsec inet | ipsec inet6 netipsec/ipsec_output.c optional ipsec inet | ipsec inet6 netipsec/ipsec_pcb.c optional ipsec inet | ipsec inet6 | \ ipsec_support inet | ipsec_support inet6 netipsec/key.c optional ipsec inet | ipsec inet6 | \ ipsec_support inet | ipsec_support inet6 netipsec/key_debug.c optional ipsec inet | ipsec inet6 | \ ipsec_support inet | ipsec_support inet6 netipsec/keysock.c optional ipsec inet | ipsec inet6 | \ ipsec_support inet | ipsec_support inet6 netipsec/subr_ipsec.c optional ipsec inet | ipsec inet6 | \ ipsec_support inet | ipsec_support inet6 netipsec/udpencap.c optional ipsec inet netipsec/xform_ah.c optional ipsec inet | ipsec inet6 netipsec/xform_esp.c optional ipsec inet | ipsec inet6 netipsec/xform_ipcomp.c optional ipsec inet | ipsec inet6 netipsec/xform_tcp.c optional ipsec inet tcp_signature | \ ipsec inet6 tcp_signature | ipsec_support inet tcp_signature | \ ipsec_support inet6 tcp_signature netnatm/natm.c optional natm netnatm/natm_pcb.c optional natm netnatm/natm_proto.c optional natm netpfil/ipfw/dn_aqm_codel.c optional inet dummynet netpfil/ipfw/dn_aqm_pie.c optional inet dummynet netpfil/ipfw/dn_heap.c optional inet dummynet netpfil/ipfw/dn_sched_fifo.c optional inet dummynet netpfil/ipfw/dn_sched_fq_codel.c optional inet dummynet netpfil/ipfw/dn_sched_fq_pie.c optional inet dummynet netpfil/ipfw/dn_sched_prio.c optional inet dummynet netpfil/ipfw/dn_sched_qfq.c optional inet dummynet netpfil/ipfw/dn_sched_rr.c optional inet dummynet netpfil/ipfw/dn_sched_wf2q.c optional inet dummynet netpfil/ipfw/ip_dummynet.c optional inet dummynet netpfil/ipfw/ip_dn_io.c optional inet dummynet netpfil/ipfw/ip_dn_glue.c optional inet dummynet netpfil/ipfw/ip_fw2.c optional inet ipfirewall netpfil/ipfw/ip_fw_bpf.c optional inet ipfirewall netpfil/ipfw/ip_fw_dynamic.c optional inet ipfirewall netpfil/ipfw/ip_fw_eaction.c optional inet ipfirewall netpfil/ipfw/ip_fw_log.c optional inet ipfirewall netpfil/ipfw/ip_fw_pfil.c optional inet ipfirewall netpfil/ipfw/ip_fw_sockopt.c optional inet ipfirewall netpfil/ipfw/ip_fw_table.c optional inet ipfirewall netpfil/ipfw/ip_fw_table_algo.c optional inet ipfirewall netpfil/ipfw/ip_fw_table_value.c optional inet ipfirewall netpfil/ipfw/ip_fw_iface.c optional inet ipfirewall netpfil/ipfw/ip_fw_nat.c optional inet ipfirewall_nat netpfil/ipfw/nat64/ip_fw_nat64.c optional inet inet6 ipfirewall \ ipfirewall_nat64 netpfil/ipfw/nat64/nat64lsn.c optional inet inet6 ipfirewall \ ipfirewall_nat64 netpfil/ipfw/nat64/nat64lsn_control.c optional inet inet6 ipfirewall \ ipfirewall_nat64 netpfil/ipfw/nat64/nat64stl.c optional inet inet6 ipfirewall \ ipfirewall_nat64 netpfil/ipfw/nat64/nat64stl_control.c optional inet inet6 ipfirewall \ ipfirewall_nat64 netpfil/ipfw/nat64/nat64_translate.c optional inet inet6 ipfirewall \ ipfirewall_nat64 netpfil/ipfw/nptv6/ip_fw_nptv6.c optional inet inet6 ipfirewall \ ipfirewall_nptv6 netpfil/ipfw/nptv6/nptv6.c optional inet inet6 ipfirewall \ ipfirewall_nptv6 netpfil/ipfw/pmod/ip_fw_pmod.c optional inet ipfirewall_pmod netpfil/ipfw/pmod/tcpmod.c optional inet ipfirewall_pmod netpfil/pf/if_pflog.c optional pflog pf inet netpfil/pf/if_pfsync.c optional pfsync pf inet netpfil/pf/pf.c optional pf inet netpfil/pf/pf_if.c optional pf inet netpfil/pf/pf_ioctl.c optional pf inet netpfil/pf/pf_lb.c optional pf inet netpfil/pf/pf_norm.c optional pf inet netpfil/pf/pf_osfp.c optional pf inet netpfil/pf/pf_ruleset.c optional pf inet netpfil/pf/pf_table.c optional pf inet netpfil/pf/in4_cksum.c optional pf inet netsmb/smb_conn.c optional netsmb netsmb/smb_crypt.c optional netsmb netsmb/smb_dev.c optional netsmb netsmb/smb_iod.c optional netsmb netsmb/smb_rq.c optional netsmb netsmb/smb_smb.c optional netsmb netsmb/smb_subr.c optional netsmb netsmb/smb_trantcp.c optional netsmb netsmb/smb_usr.c optional netsmb nfs/bootp_subr.c optional bootp nfscl nfs/krpc_subr.c optional bootp nfscl nfs/nfs_diskless.c optional nfscl nfs_root nfs/nfs_fha.c optional nfsd nfs/nfs_lock.c optional nfscl | nfslockd | nfsd nfs/nfs_nfssvc.c optional nfscl | nfsd nlm/nlm_advlock.c optional nfslockd | nfsd nlm/nlm_prot_clnt.c optional nfslockd | nfsd nlm/nlm_prot_impl.c optional nfslockd | nfsd nlm/nlm_prot_server.c optional nfslockd | nfsd nlm/nlm_prot_svc.c optional nfslockd | nfsd nlm/nlm_prot_xdr.c optional nfslockd | nfsd nlm/sm_inter_xdr.c optional nfslockd | nfsd # Linux Kernel Programming Interface compat/linuxkpi/common/src/linux_kmod.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_compat.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_current.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_hrtimer.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_kthread.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_lock.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_page.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_pci.c optional compat_linuxkpi pci \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_tasklet.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_idr.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_radix.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_rcu.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C} -I$S/contrib/ck/include" compat/linuxkpi/common/src/linux_schedule.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_slab.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_usb.c optional compat_linuxkpi usb \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_work.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" # OpenFabrics Enterprise Distribution (Infiniband) ofed/drivers/infiniband/core/addr.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/agent.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/cache.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" # XXX Mad.c must be ordered before cm.c for sysinit sets to occur in # the correct order. ofed/drivers/infiniband/core/mad.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/cm.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/ -Wno-unused-function" ofed/drivers/infiniband/core/cma.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/device.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/fmr_pool.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/iwcm.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/mad_rmpp.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/multicast.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/packer.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/peer_mem.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/sa_query.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/smi.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/sysfs.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/ucm.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/ucma.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/ud_header.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/umem.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/user_mad.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/uverbs_cmd.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/uverbs_main.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/uverbs_marshall.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/verbs.c optional ofed \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/ulp/ipoib/ipoib_cm.c optional ipoib \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" #ofed/drivers/infiniband/ulp/ipoib/ipoib_fs.c optional ipoib \ # compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" ofed/drivers/infiniband/ulp/ipoib/ipoib_ib.c optional ipoib \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c optional ipoib \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" ofed/drivers/infiniband/ulp/ipoib/ipoib_multicast.c optional ipoib \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" ofed/drivers/infiniband/ulp/ipoib/ipoib_verbs.c optional ipoib \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" #ofed/drivers/infiniband/ulp/ipoib/ipoib_vlan.c optional ipoib \ # compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" ofed/drivers/infiniband/ulp/sdp/sdp_bcopy.c optional sdp inet \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/" ofed/drivers/infiniband/ulp/sdp/sdp_main.c optional sdp inet \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/" ofed/drivers/infiniband/ulp/sdp/sdp_rx.c optional sdp inet \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/" ofed/drivers/infiniband/ulp/sdp/sdp_cma.c optional sdp inet \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/" ofed/drivers/infiniband/ulp/sdp/sdp_tx.c optional sdp inet \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/" -ofed/drivers/infiniband/hw/mlx4/alias_GUID.c optional mlx4ib \ - no-depend obj-prefix "mlx4ib_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" -ofed/drivers/infiniband/hw/mlx4/mcg.c optional mlx4ib \ - no-depend obj-prefix "mlx4ib_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" -ofed/drivers/infiniband/hw/mlx4/sysfs.c optional mlx4ib \ - no-depend obj-prefix "mlx4ib_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" -ofed/drivers/infiniband/hw/mlx4/cm.c optional mlx4ib \ - no-depend obj-prefix "mlx4ib_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" -ofed/drivers/infiniband/hw/mlx4/ah.c optional mlx4ib \ - no-depend obj-prefix "mlx4ib_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" -ofed/drivers/infiniband/hw/mlx4/cq.c optional mlx4ib \ - no-depend obj-prefix "mlx4ib_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" -ofed/drivers/infiniband/hw/mlx4/doorbell.c optional mlx4ib \ - no-depend obj-prefix "mlx4ib_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" -ofed/drivers/infiniband/hw/mlx4/mad.c optional mlx4ib \ - no-depend obj-prefix "mlx4ib_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" -ofed/drivers/infiniband/hw/mlx4/main.c optional mlx4ib \ - no-depend obj-prefix "mlx4ib_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" -ofed/drivers/infiniband/hw/mlx4/mlx4_exp.c optional mlx4ib \ - no-depend obj-prefix "mlx4ib_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" -ofed/drivers/infiniband/hw/mlx4/mr.c optional mlx4ib \ - no-depend obj-prefix "mlx4ib_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" -ofed/drivers/infiniband/hw/mlx4/qp.c optional mlx4ib \ - no-depend obj-prefix "mlx4ib_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" -ofed/drivers/infiniband/hw/mlx4/srq.c optional mlx4ib \ - no-depend obj-prefix "mlx4ib_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" -ofed/drivers/infiniband/hw/mlx4/wc.c optional mlx4ib \ - no-depend obj-prefix "mlx4ib_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" +dev/mlx4/mlx4_ib/mlx4_ib_alias_GUID.c optional mlx4ib pci ofed \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_ib/mlx4_ib_mcg.c optional mlx4ib pci ofed \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_ib/mlx4_ib_sysfs.c optional mlx4ib pci ofed \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_ib/mlx4_ib_cm.c optional mlx4ib pci ofed \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_ib/mlx4_ib_ah.c optional mlx4ib pci ofed \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_ib/mlx4_ib_cq.c optional mlx4ib pci ofed \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_ib/mlx4_ib_doorbell.c optional mlx4ib pci ofed \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_ib/mlx4_ib_mad.c optional mlx4ib pci ofed \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_ib/mlx4_ib_main.c optional mlx4ib pci ofed \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_ib/mlx4_ib_exp.c optional mlx4ib pci ofed \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_ib/mlx4_ib_mr.c optional mlx4ib pci ofed \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_ib/mlx4_ib_qp.c optional mlx4ib pci ofed \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_ib/mlx4_ib_srq.c optional mlx4ib pci ofed \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_ib/mlx4_ib_wc.c optional mlx4ib pci ofed \ + compile-with "${OFED_C}" -ofed/drivers/net/mlx4/alloc.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/catas.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/cmd.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/cq.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/eq.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/fw.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/icm.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/intf.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/main.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/mcg.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/ -Wno-unused" -ofed/drivers/net/mlx4/mr.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/pd.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/port.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/profile.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/qp.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/reset.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/sense.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/srq.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/resource_tracker.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/sys_tune.c optional mlx4ib | mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" +dev/mlx4/mlx4_core/mlx4_alloc.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_catas.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_cmd.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_cq.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_eq.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_fw.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_fw_qos.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_icm.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_intf.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_main.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_mcg.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_mr.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_pd.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_port.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_profile.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_qp.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_reset.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_sense.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_srq.c optional mlx4 pci \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_core/mlx4_resource_tracker.c optional mlx4 pci \ + compile-with "${OFED_C}" -ofed/drivers/net/mlx4/en_cq.c optional mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/en_main.c optional mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/en_netdev.c optional mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/en_port.c optional mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/en_resources.c optional mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/en_rx.c optional mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/en_tx.c optional mlxen \ - no-depend obj-prefix "mlx4_" \ - compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" +dev/mlx4/mlx4_en/mlx4_en_cq.c optional mlx4en pci inet inet6 \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_en/mlx4_en_main.c optional mlx4en pci inet inet6 \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_en/mlx4_en_netdev.c optional mlx4en pci inet inet6 \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_en/mlx4_en_port.c optional mlx4en pci inet inet6 \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_en/mlx4_en_resources.c optional mlx4en pci inet inet6 \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_en/mlx4_en_rx.c optional mlx4en pci inet inet6 \ + compile-with "${OFED_C}" +dev/mlx4/mlx4_en/mlx4_en_tx.c optional mlx4en pci inet inet6 \ + compile-with "${OFED_C}" dev/mlx5/mlx5_ib/mlx5_ib_ah.c optional mlx5ib pci ofed \ compile-with "${OFED_C}" dev/mlx5/mlx5_ib/mlx5_ib_cq.c optional mlx5ib pci ofed \ compile-with "${OFED_C}" dev/mlx5/mlx5_ib/mlx5_ib_doorbell.c optional mlx5ib pci ofed \ compile-with "${OFED_C}" dev/mlx5/mlx5_ib/mlx5_ib_mad.c optional mlx5ib pci ofed \ compile-with "${OFED_C}" dev/mlx5/mlx5_ib/mlx5_ib_main.c optional mlx5ib pci ofed \ compile-with "${OFED_C}" dev/mlx5/mlx5_ib/mlx5_ib_mem.c optional mlx5ib pci ofed \ compile-with "${OFED_C}" dev/mlx5/mlx5_ib/mlx5_ib_mr.c optional mlx5ib pci ofed \ compile-with "${OFED_C}" dev/mlx5/mlx5_ib/mlx5_ib_qp.c optional mlx5ib pci ofed \ compile-with "${OFED_C}" dev/mlx5/mlx5_ib/mlx5_ib_roce.c optional mlx5ib pci ofed \ compile-with "${OFED_C}" dev/mlx5/mlx5_ib/mlx5_ib_srq.c optional mlx5ib pci ofed \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_alloc.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_cmd.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_cq.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_diagnostics.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_eq.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_flow_table.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_fw.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_health.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_mad.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_main.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_mcg.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_mr.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_pagealloc.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_pd.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_port.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_qp.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_srq.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_transobj.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_uar.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_vport.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_wq.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_en/mlx5_en_ethtool.c optional mlx5en pci inet inet6 \ compile-with "${OFED_C}" dev/mlx5/mlx5_en/mlx5_en_main.c optional mlx5en pci inet inet6 \ compile-with "${OFED_C}" dev/mlx5/mlx5_en/mlx5_en_tx.c optional mlx5en pci inet inet6 \ compile-with "${OFED_C}" dev/mlx5/mlx5_en/mlx5_en_flow_table.c optional mlx5en pci inet inet6 \ compile-with "${OFED_C}" dev/mlx5/mlx5_en/mlx5_en_rx.c optional mlx5en pci inet inet6 \ compile-with "${OFED_C}" dev/mlx5/mlx5_en/mlx5_en_txrx.c optional mlx5en pci inet inet6 \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_allocator.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_av.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_catas.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_cmd.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_cq.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_eq.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_mad.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_main.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_mcg.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_memfree.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_mr.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_pd.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_profile.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_provider.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_qp.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_reset.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_srq.c optional mthca \ compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_uar.c optional mthca \ compile-with "${OFED_C}" # crypto support opencrypto/cast.c optional crypto | ipsec | ipsec_support opencrypto/criov.c optional crypto | ipsec | ipsec_support opencrypto/crypto.c optional crypto | ipsec | ipsec_support opencrypto/cryptodev.c optional cryptodev opencrypto/cryptodev_if.m optional crypto | ipsec | ipsec_support opencrypto/cryptosoft.c optional crypto | ipsec | ipsec_support opencrypto/cryptodeflate.c optional crypto | ipsec | ipsec_support opencrypto/gmac.c optional crypto | ipsec | ipsec_support opencrypto/gfmult.c optional crypto | ipsec | ipsec_support opencrypto/rmd160.c optional crypto | ipsec | ipsec_support opencrypto/skipjack.c optional crypto | ipsec | ipsec_support opencrypto/xform.c optional crypto | ipsec | ipsec_support rpc/auth_none.c optional krpc | nfslockd | nfscl | nfsd rpc/auth_unix.c optional krpc | nfslockd | nfscl | nfsd rpc/authunix_prot.c optional krpc | nfslockd | nfscl | nfsd rpc/clnt_bck.c optional krpc | nfslockd | nfscl | nfsd rpc/clnt_dg.c optional krpc | nfslockd | nfscl | nfsd rpc/clnt_rc.c optional krpc | nfslockd | nfscl | nfsd rpc/clnt_vc.c optional krpc | nfslockd | nfscl | nfsd rpc/getnetconfig.c optional krpc | nfslockd | nfscl | nfsd rpc/replay.c optional krpc | nfslockd | nfscl | nfsd rpc/rpc_callmsg.c optional krpc | nfslockd | nfscl | nfsd rpc/rpc_generic.c optional krpc | nfslockd | nfscl | nfsd rpc/rpc_prot.c optional krpc | nfslockd | nfscl | nfsd rpc/rpcb_clnt.c optional krpc | nfslockd | nfscl | nfsd rpc/rpcb_prot.c optional krpc | nfslockd | nfscl | nfsd rpc/svc.c optional krpc | nfslockd | nfscl | nfsd rpc/svc_auth.c optional krpc | nfslockd | nfscl | nfsd rpc/svc_auth_unix.c optional krpc | nfslockd | nfscl | nfsd rpc/svc_dg.c optional krpc | nfslockd | nfscl | nfsd rpc/svc_generic.c optional krpc | nfslockd | nfscl | nfsd rpc/svc_vc.c optional krpc | nfslockd | nfscl | nfsd rpc/rpcsec_gss/rpcsec_gss.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi rpc/rpcsec_gss/rpcsec_gss_conf.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi rpc/rpcsec_gss/rpcsec_gss_misc.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi rpc/rpcsec_gss/rpcsec_gss_prot.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi rpc/rpcsec_gss/svc_rpcsec_gss.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi security/audit/audit.c optional audit security/audit/audit_arg.c optional audit security/audit/audit_bsm.c optional audit security/audit/audit_bsm_klib.c optional audit security/audit/audit_pipe.c optional audit security/audit/audit_syscalls.c standard security/audit/audit_trigger.c optional audit security/audit/audit_worker.c optional audit security/audit/bsm_domain.c optional audit security/audit/bsm_errno.c optional audit security/audit/bsm_fcntl.c optional audit security/audit/bsm_socket_type.c optional audit security/audit/bsm_token.c optional audit security/mac/mac_audit.c optional mac audit security/mac/mac_cred.c optional mac security/mac/mac_framework.c optional mac security/mac/mac_inet.c optional mac inet | mac inet6 security/mac/mac_inet6.c optional mac inet6 security/mac/mac_label.c optional mac security/mac/mac_net.c optional mac security/mac/mac_pipe.c optional mac security/mac/mac_posix_sem.c optional mac security/mac/mac_posix_shm.c optional mac security/mac/mac_priv.c optional mac security/mac/mac_process.c optional mac security/mac/mac_socket.c optional mac security/mac/mac_syscalls.c standard security/mac/mac_system.c optional mac security/mac/mac_sysv_msg.c optional mac security/mac/mac_sysv_sem.c optional mac security/mac/mac_sysv_shm.c optional mac security/mac/mac_vfs.c optional mac security/mac_biba/mac_biba.c optional mac_biba security/mac_bsdextended/mac_bsdextended.c optional mac_bsdextended security/mac_bsdextended/ugidfw_system.c optional mac_bsdextended security/mac_bsdextended/ugidfw_vnode.c optional mac_bsdextended security/mac_ifoff/mac_ifoff.c optional mac_ifoff security/mac_lomac/mac_lomac.c optional mac_lomac security/mac_mls/mac_mls.c optional mac_mls security/mac_none/mac_none.c optional mac_none security/mac_partition/mac_partition.c optional mac_partition security/mac_portacl/mac_portacl.c optional mac_portacl security/mac_seeotheruids/mac_seeotheruids.c optional mac_seeotheruids security/mac_stub/mac_stub.c optional mac_stub security/mac_test/mac_test.c optional mac_test teken/teken.c optional sc | vt ufs/ffs/ffs_alloc.c optional ffs ufs/ffs/ffs_balloc.c optional ffs ufs/ffs/ffs_inode.c optional ffs ufs/ffs/ffs_snapshot.c optional ffs ufs/ffs/ffs_softdep.c optional ffs ufs/ffs/ffs_subr.c optional ffs ufs/ffs/ffs_tables.c optional ffs ufs/ffs/ffs_vfsops.c optional ffs ufs/ffs/ffs_vnops.c optional ffs ufs/ffs/ffs_rawread.c optional ffs directio ufs/ffs/ffs_suspend.c optional ffs ufs/ufs/ufs_acl.c optional ffs ufs/ufs/ufs_bmap.c optional ffs ufs/ufs/ufs_dirhash.c optional ffs ufs/ufs/ufs_extattr.c optional ffs ufs/ufs/ufs_gjournal.c optional ffs UFS_GJOURNAL ufs/ufs/ufs_inode.c optional ffs ufs/ufs/ufs_lookup.c optional ffs ufs/ufs/ufs_quota.c optional ffs ufs/ufs/ufs_vfsops.c optional ffs ufs/ufs/ufs_vnops.c optional ffs vm/default_pager.c standard vm/device_pager.c standard vm/phys_pager.c standard vm/redzone.c optional DEBUG_REDZONE vm/sg_pager.c standard vm/swap_pager.c standard vm/uma_core.c standard vm/uma_dbg.c standard vm/memguard.c optional DEBUG_MEMGUARD vm/vm_domain.c standard vm/vm_fault.c standard vm/vm_glue.c standard vm/vm_init.c standard vm/vm_kern.c standard vm/vm_map.c standard vm/vm_meter.c standard vm/vm_mmap.c standard vm/vm_object.c standard vm/vm_page.c standard vm/vm_pageout.c standard vm/vm_pager.c standard vm/vm_phys.c standard vm/vm_radix.c standard vm/vm_reserv.c standard vm/vm_swapout.c optional !NO_SWAPPING vm/vm_swapout_dummy.c optional NO_SWAPPING vm/vm_unix.c standard vm/vm_zeroidle.c standard vm/vnode_pager.c standard xen/features.c optional xenhvm xen/xenbus/xenbus_if.m optional xenhvm xen/xenbus/xenbus.c optional xenhvm xen/xenbus/xenbusb_if.m optional xenhvm xen/xenbus/xenbusb.c optional xenhvm xen/xenbus/xenbusb_front.c optional xenhvm xen/xenbus/xenbusb_back.c optional xenhvm xen/xenmem/xenmem_if.m optional xenhvm xdr/xdr.c optional krpc | nfslockd | nfscl | nfsd xdr/xdr_array.c optional krpc | nfslockd | nfscl | nfsd xdr/xdr_mbuf.c optional krpc | nfslockd | nfscl | nfsd xdr/xdr_mem.c optional krpc | nfslockd | nfscl | nfsd xdr/xdr_reference.c optional krpc | nfslockd | nfscl | nfsd xdr/xdr_sizeof.c optional krpc | nfslockd | nfscl | nfsd Index: stable/11/sys/dev/mlx4/cmd.h =================================================================== --- stable/11/sys/dev/mlx4/cmd.h (nonexistent) +++ stable/11/sys/dev/mlx4/cmd.h (revision 329159) @@ -0,0 +1,330 @@ +/* + * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MLX4_CMD_H +#define MLX4_CMD_H + +#include +#include + +struct mlx4_counter; + +enum { + /* initialization and general commands */ + MLX4_CMD_SYS_EN = 0x1, + MLX4_CMD_SYS_DIS = 0x2, + MLX4_CMD_MAP_FA = 0xfff, + MLX4_CMD_UNMAP_FA = 0xffe, + MLX4_CMD_RUN_FW = 0xff6, + MLX4_CMD_MOD_STAT_CFG = 0x34, + MLX4_CMD_QUERY_DEV_CAP = 0x3, + MLX4_CMD_QUERY_FW = 0x4, + MLX4_CMD_ENABLE_LAM = 0xff8, + MLX4_CMD_DISABLE_LAM = 0xff7, + MLX4_CMD_QUERY_DDR = 0x5, + MLX4_CMD_QUERY_ADAPTER = 0x6, + MLX4_CMD_INIT_HCA = 0x7, + MLX4_CMD_CLOSE_HCA = 0x8, + MLX4_CMD_INIT_PORT = 0x9, + MLX4_CMD_CLOSE_PORT = 0xa, + MLX4_CMD_QUERY_HCA = 0xb, + MLX4_CMD_QUERY_PORT = 0x43, + MLX4_CMD_SENSE_PORT = 0x4d, + MLX4_CMD_HW_HEALTH_CHECK = 0x50, + MLX4_CMD_SET_PORT = 0xc, + MLX4_CMD_SET_NODE = 0x5a, + MLX4_CMD_QUERY_FUNC = 0x56, + MLX4_CMD_ACCESS_DDR = 0x2e, + MLX4_CMD_MAP_ICM = 0xffa, + MLX4_CMD_UNMAP_ICM = 0xff9, + MLX4_CMD_MAP_ICM_AUX = 0xffc, + MLX4_CMD_UNMAP_ICM_AUX = 0xffb, + MLX4_CMD_SET_ICM_SIZE = 0xffd, + MLX4_CMD_ACCESS_REG = 0x3b, + MLX4_CMD_ALLOCATE_VPP = 0x80, + MLX4_CMD_SET_VPORT_QOS = 0x81, + + /*master notify fw on finish for slave's flr*/ + MLX4_CMD_INFORM_FLR_DONE = 0x5b, + MLX4_CMD_VIRT_PORT_MAP = 0x5c, + MLX4_CMD_GET_OP_REQ = 0x59, + + /* TPT commands */ + MLX4_CMD_SW2HW_MPT = 0xd, + MLX4_CMD_QUERY_MPT = 0xe, + MLX4_CMD_HW2SW_MPT = 0xf, + MLX4_CMD_READ_MTT = 0x10, + MLX4_CMD_WRITE_MTT = 0x11, + MLX4_CMD_SYNC_TPT = 0x2f, + + /* EQ commands */ + MLX4_CMD_MAP_EQ = 0x12, + MLX4_CMD_SW2HW_EQ = 0x13, + MLX4_CMD_HW2SW_EQ = 0x14, + MLX4_CMD_QUERY_EQ = 0x15, + + /* CQ commands */ + MLX4_CMD_SW2HW_CQ = 0x16, + MLX4_CMD_HW2SW_CQ = 0x17, + MLX4_CMD_QUERY_CQ = 0x18, + MLX4_CMD_MODIFY_CQ = 0x2c, + + /* SRQ commands */ + MLX4_CMD_SW2HW_SRQ = 0x35, + MLX4_CMD_HW2SW_SRQ = 0x36, + MLX4_CMD_QUERY_SRQ = 0x37, + MLX4_CMD_ARM_SRQ = 0x40, + + /* QP/EE commands */ + MLX4_CMD_RST2INIT_QP = 0x19, + MLX4_CMD_INIT2RTR_QP = 0x1a, + MLX4_CMD_RTR2RTS_QP = 0x1b, + MLX4_CMD_RTS2RTS_QP = 0x1c, + MLX4_CMD_SQERR2RTS_QP = 0x1d, + MLX4_CMD_2ERR_QP = 0x1e, + MLX4_CMD_RTS2SQD_QP = 0x1f, + MLX4_CMD_SQD2SQD_QP = 0x38, + MLX4_CMD_SQD2RTS_QP = 0x20, + MLX4_CMD_2RST_QP = 0x21, + MLX4_CMD_QUERY_QP = 0x22, + MLX4_CMD_INIT2INIT_QP = 0x2d, + MLX4_CMD_SUSPEND_QP = 0x32, + MLX4_CMD_UNSUSPEND_QP = 0x33, + MLX4_CMD_UPDATE_QP = 0x61, + /* special QP and management commands */ + MLX4_CMD_CONF_SPECIAL_QP = 0x23, + MLX4_CMD_MAD_IFC = 0x24, + MLX4_CMD_MAD_DEMUX = 0x203, + + /* multicast commands */ + MLX4_CMD_READ_MCG = 0x25, + MLX4_CMD_WRITE_MCG = 0x26, + MLX4_CMD_MGID_HASH = 0x27, + + /* miscellaneous commands */ + MLX4_CMD_DIAG_RPRT = 0x30, + MLX4_CMD_NOP = 0x31, + MLX4_CMD_CONFIG_DEV = 0x3a, + MLX4_CMD_ACCESS_MEM = 0x2e, + MLX4_CMD_SET_VEP = 0x52, + + /* Ethernet specific commands */ + MLX4_CMD_SET_VLAN_FLTR = 0x47, + MLX4_CMD_SET_MCAST_FLTR = 0x48, + MLX4_CMD_DUMP_ETH_STATS = 0x49, + + /* Communication channel commands */ + MLX4_CMD_ARM_COMM_CHANNEL = 0x57, + MLX4_CMD_GEN_EQE = 0x58, + + /* virtual commands */ + MLX4_CMD_ALLOC_RES = 0xf00, + MLX4_CMD_FREE_RES = 0xf01, + MLX4_CMD_MCAST_ATTACH = 0xf05, + MLX4_CMD_UCAST_ATTACH = 0xf06, + MLX4_CMD_PROMISC = 0xf08, + MLX4_CMD_QUERY_FUNC_CAP = 0xf0a, + MLX4_CMD_QP_ATTACH = 0xf0b, + + /* debug commands */ + MLX4_CMD_QUERY_DEBUG_MSG = 0x2a, + MLX4_CMD_SET_DEBUG_MSG = 0x2b, + + /* statistics commands */ + MLX4_CMD_QUERY_IF_STAT = 0X54, + MLX4_CMD_SET_IF_STAT = 0X55, + + /* register/delete flow steering network rules */ + MLX4_QP_FLOW_STEERING_ATTACH = 0x65, + MLX4_QP_FLOW_STEERING_DETACH = 0x66, + MLX4_FLOW_STEERING_IB_UC_QP_RANGE = 0x64, + + /* Update and read QCN parameters */ + MLX4_CMD_CONGESTION_CTRL_OPCODE = 0x68, +}; + +enum { + MLX4_CMD_TIME_CLASS_A = 60000, + MLX4_CMD_TIME_CLASS_B = 60000, + MLX4_CMD_TIME_CLASS_C = 60000, +}; + +enum { + /* virtual to physical port mapping opcode modifiers */ + MLX4_GET_PORT_VIRT2PHY = 0x0, + MLX4_SET_PORT_VIRT2PHY = 0x1, +}; + +enum { + MLX4_MAILBOX_SIZE = 4096, + MLX4_ACCESS_MEM_ALIGN = 256, +}; + +enum { + /* Set port opcode modifiers */ + MLX4_SET_PORT_IB_OPCODE = 0x0, + MLX4_SET_PORT_ETH_OPCODE = 0x1, + MLX4_SET_PORT_BEACON_OPCODE = 0x4, +}; + +enum { + /* Set port Ethernet input modifiers */ + MLX4_SET_PORT_GENERAL = 0x0, + MLX4_SET_PORT_RQP_CALC = 0x1, + MLX4_SET_PORT_MAC_TABLE = 0x2, + MLX4_SET_PORT_VLAN_TABLE = 0x3, + MLX4_SET_PORT_PRIO_MAP = 0x4, + MLX4_SET_PORT_GID_TABLE = 0x5, + MLX4_SET_PORT_PRIO2TC = 0x8, + MLX4_SET_PORT_SCHEDULER = 0x9, + MLX4_SET_PORT_VXLAN = 0xB, + MLX4_SET_PORT_ROCE_ADDR = 0xD +}; + +enum { + MLX4_CMD_MAD_DEMUX_CONFIG = 0, + MLX4_CMD_MAD_DEMUX_QUERY_STATE = 1, + MLX4_CMD_MAD_DEMUX_QUERY_RESTR = 2, /* Query mad demux restrictions */ +}; + +enum { + MLX4_CMD_WRAPPED, + MLX4_CMD_NATIVE +}; + +/* + * MLX4_RX_CSUM_MODE_VAL_NON_TCP_UDP - + * Receive checksum value is reported in CQE also for non TCP/UDP packets. + * + * MLX4_RX_CSUM_MODE_L4 - + * L4_CSUM bit in CQE, which indicates whether or not L4 checksum + * was validated correctly, is supported. + * + * MLX4_RX_CSUM_MODE_IP_OK_IP_NON_TCP_UDP - + * IP_OK CQE's field is supported also for non TCP/UDP IP packets. + * + * MLX4_RX_CSUM_MODE_MULTI_VLAN - + * Receive Checksum offload is supported for packets with more than 2 vlan headers. + */ +enum mlx4_rx_csum_mode { + MLX4_RX_CSUM_MODE_VAL_NON_TCP_UDP = 1UL << 0, + MLX4_RX_CSUM_MODE_L4 = 1UL << 1, + MLX4_RX_CSUM_MODE_IP_OK_IP_NON_TCP_UDP = 1UL << 2, + MLX4_RX_CSUM_MODE_MULTI_VLAN = 1UL << 3 +}; + +struct mlx4_config_dev_params { + u16 vxlan_udp_dport; + u8 rx_csum_flags_port_1; + u8 rx_csum_flags_port_2; +}; + +enum mlx4_en_congestion_control_algorithm { + MLX4_CTRL_ALGO_802_1_QAU_REACTION_POINT = 0, +}; + +enum mlx4_en_congestion_control_opmod { + MLX4_CONGESTION_CONTROL_GET_PARAMS, + MLX4_CONGESTION_CONTROL_GET_STATISTICS, + MLX4_CONGESTION_CONTROL_SET_PARAMS = 4, +}; + +struct mlx4_dev; + +struct mlx4_cmd_mailbox { + void *buf; + dma_addr_t dma; +}; + +int __mlx4_cmd(struct mlx4_dev *dev, u64 in_param, u64 *out_param, + int out_is_imm, u32 in_modifier, u8 op_modifier, + u16 op, unsigned long timeout, int native); + +/* Invoke a command with no output parameter */ +static inline int mlx4_cmd(struct mlx4_dev *dev, u64 in_param, u32 in_modifier, + u8 op_modifier, u16 op, unsigned long timeout, + int native) +{ + return __mlx4_cmd(dev, in_param, NULL, 0, in_modifier, + op_modifier, op, timeout, native); +} + +/* Invoke a command with an output mailbox */ +static inline int mlx4_cmd_box(struct mlx4_dev *dev, u64 in_param, u64 out_param, + u32 in_modifier, u8 op_modifier, u16 op, + unsigned long timeout, int native) +{ + return __mlx4_cmd(dev, in_param, &out_param, 0, in_modifier, + op_modifier, op, timeout, native); +} + +/* + * Invoke a command with an immediate output parameter (and copy the + * output into the caller's out_param pointer after the command + * executes). + */ +static inline int mlx4_cmd_imm(struct mlx4_dev *dev, u64 in_param, u64 *out_param, + u32 in_modifier, u8 op_modifier, u16 op, + unsigned long timeout, int native) +{ + return __mlx4_cmd(dev, in_param, out_param, 1, in_modifier, + op_modifier, op, timeout, native); +} + +struct mlx4_cmd_mailbox *mlx4_alloc_cmd_mailbox(struct mlx4_dev *dev); +void mlx4_free_cmd_mailbox(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox); + +int mlx4_get_counter_stats(struct mlx4_dev *dev, int counter_index, + struct mlx4_counter *counter_stats, int reset); +u32 mlx4_comm_get_version(void); +int mlx4_set_vf_mac(struct mlx4_dev *dev, int port, int vf, u64 mac); +int mlx4_set_vf_vlan(struct mlx4_dev *dev, int port, int vf, u16 vlan, + u8 qos, __be16 proto); +int mlx4_set_vf_rate(struct mlx4_dev *dev, int port, int vf, int min_tx_rate, + int max_tx_rate); +int mlx4_set_vf_spoofchk(struct mlx4_dev *dev, int port, int vf, bool setting); +int mlx4_config_dev_retrieval(struct mlx4_dev *dev, + struct mlx4_config_dev_params *params); +void mlx4_cmd_wake_completions(struct mlx4_dev *dev); +void mlx4_report_internal_err_comm_event(struct mlx4_dev *dev); +/* + * mlx4_get_slave_default_vlan - + * return true if VST ( default vlan) + * if VST, will return vlan & qos (if not NULL) + */ +bool mlx4_get_slave_default_vlan(struct mlx4_dev *dev, int port, int slave, + u16 *vlan, u8 *qos); + +#define MLX4_COMM_GET_IF_REV(cmd_chan_ver) (u8)((cmd_chan_ver) >> 8) +#define COMM_CHAN_EVENT_INTERNAL_ERR (1 << 17) + +#endif /* MLX4_CMD_H */ Property changes on: stable/11/sys/dev/mlx4/cmd.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/cq.h =================================================================== --- stable/11/sys/dev/mlx4/cq.h (nonexistent) +++ stable/11/sys/dev/mlx4/cq.h (revision 329159) @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MLX4_CQ_H +#define MLX4_CQ_H + +#include + +#include +#include + +struct mlx4_cqe { + __be32 vlan_my_qpn; + __be32 immed_rss_invalid; + __be32 g_mlpath_rqpn; + __be16 sl_vid; + union { + struct { + __be16 rlid; + __be16 status; + u8 ipv6_ext_mask; + u8 badfcs_enc; + }; + u8 smac[ETH_ALEN]; + }; + __be32 byte_cnt; + __be16 wqe_index; + __be16 checksum; + u8 reserved[3]; + u8 owner_sr_opcode; +}; + +struct mlx4_err_cqe { + __be32 my_qpn; + u32 reserved1[5]; + __be16 wqe_index; + u8 vendor_err_syndrome; + u8 syndrome; + u8 reserved2[3]; + u8 owner_sr_opcode; +}; + +struct mlx4_ts_cqe { + __be32 vlan_my_qpn; + __be32 immed_rss_invalid; + __be32 g_mlpath_rqpn; + __be32 timestamp_hi; + __be16 status; + u8 ipv6_ext_mask; + u8 badfcs_enc; + __be32 byte_cnt; + __be16 wqe_index; + __be16 checksum; + u8 reserved; + __be16 timestamp_lo; + u8 owner_sr_opcode; +} __packed; + +enum { + MLX4_CQE_L2_TUNNEL_IPOK = 1 << 31, + MLX4_CQE_CVLAN_PRESENT_MASK = 1 << 29, + MLX4_CQE_SVLAN_PRESENT_MASK = 1 << 30, + MLX4_CQE_L2_TUNNEL = 1 << 27, + MLX4_CQE_L2_TUNNEL_CSUM = 1 << 26, + MLX4_CQE_L2_TUNNEL_IPV4 = 1 << 25, + + MLX4_CQE_QPN_MASK = 0xffffff, + MLX4_CQE_VID_MASK = 0xfff, +}; + +enum { + MLX4_CQE_OWNER_MASK = 0x80, + MLX4_CQE_IS_SEND_MASK = 0x40, + MLX4_CQE_OPCODE_MASK = 0x1f +}; + +enum { + MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR = 0x01, + MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR = 0x02, + MLX4_CQE_SYNDROME_LOCAL_PROT_ERR = 0x04, + MLX4_CQE_SYNDROME_WR_FLUSH_ERR = 0x05, + MLX4_CQE_SYNDROME_MW_BIND_ERR = 0x06, + MLX4_CQE_SYNDROME_BAD_RESP_ERR = 0x10, + MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR = 0x11, + MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR = 0x12, + MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR = 0x13, + MLX4_CQE_SYNDROME_REMOTE_OP_ERR = 0x14, + MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR = 0x15, + MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR = 0x16, + MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR = 0x22, +}; + +enum { + MLX4_CQE_STATUS_IPV4 = 1 << 6, + MLX4_CQE_STATUS_IPV4F = 1 << 7, + MLX4_CQE_STATUS_IPV6 = 1 << 8, + MLX4_CQE_STATUS_IPV4OPT = 1 << 9, + MLX4_CQE_STATUS_TCP = 1 << 10, + MLX4_CQE_STATUS_UDP = 1 << 11, + MLX4_CQE_STATUS_IPOK = 1 << 12, +}; + +enum { + MLX4_CQE_LLC = 1, + MLX4_CQE_SNAP = 1 << 1, + MLX4_CQE_BAD_FCS = 1 << 4, +}; + +static inline void mlx4_cq_arm(struct mlx4_cq *cq, u32 cmd, + u8 __iomem *uar_page, + spinlock_t *doorbell_lock) +{ + __be32 doorbell[2]; + u32 sn; + u32 ci; + + sn = cq->arm_sn & 3; + ci = cq->cons_index & 0xffffff; + + *cq->arm_db = cpu_to_be32(sn << 28 | cmd | ci); + + /* + * Make sure that the doorbell record in host memory is + * written before ringing the doorbell via PCI MMIO. + */ + wmb(); + + doorbell[0] = cpu_to_be32(sn << 28 | cmd | cq->cqn); + doorbell[1] = cpu_to_be32(ci); + + mlx4_write64(doorbell, uar_page + MLX4_CQ_DOORBELL, doorbell_lock); +} + +static inline void mlx4_cq_set_ci(struct mlx4_cq *cq) +{ + *cq->set_ci_db = cpu_to_be32(cq->cons_index & 0xffffff); +} + +enum { + MLX4_CQ_DB_REQ_NOT_SOL = 1 << 24, + MLX4_CQ_DB_REQ_NOT = 2 << 24 +}; + +int mlx4_cq_modify(struct mlx4_dev *dev, struct mlx4_cq *cq, + u16 count, u16 period); +int mlx4_cq_resize(struct mlx4_dev *dev, struct mlx4_cq *cq, + int entries, struct mlx4_mtt *mtt); + +#endif /* MLX4_CQ_H */ Property changes on: stable/11/sys/dev/mlx4/cq.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/device.h =================================================================== --- stable/11/sys/dev/mlx4/device.h (nonexistent) +++ stable/11/sys/dev/mlx4/device.h (revision 329159) @@ -0,0 +1,1570 @@ +/* + * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MLX4_DEVICE_H +#define MLX4_DEVICE_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#define DEFAULT_UAR_PAGE_SHIFT 12 + +#define MAX_MSIX_P_PORT 17 +#define MAX_MSIX 64 +#define MIN_MSIX_P_PORT 5 +#define MLX4_IS_LEGACY_EQ_MODE(dev_cap) ((dev_cap).num_comp_vectors < \ + (dev_cap).num_ports * MIN_MSIX_P_PORT) + +#define MLX4_MAX_100M_UNITS_VAL 255 /* + * work around: can't set values + * greater then this value when + * using 100 Mbps units. + */ +#define MLX4_RATELIMIT_100M_UNITS 3 /* 100 Mbps */ +#define MLX4_RATELIMIT_1G_UNITS 4 /* 1 Gbps */ +#define MLX4_RATELIMIT_DEFAULT 0x00ff + +#define MLX4_ROCE_MAX_GIDS 128 +#define MLX4_ROCE_PF_GIDS 16 + +#define CORE_CLOCK_MASK 0xffffffffffffULL + +enum { + MLX4_FLAG_MSI_X = 1 << 0, + MLX4_FLAG_OLD_PORT_CMDS = 1 << 1, + MLX4_FLAG_MASTER = 1 << 2, + MLX4_FLAG_SLAVE = 1 << 3, + MLX4_FLAG_SRIOV = 1 << 4, + MLX4_FLAG_OLD_REG_MAC = 1 << 6, + MLX4_FLAG_BONDED = 1 << 7, + MLX4_FLAG_SECURE_HOST = 1 << 8, +}; + +enum { + MLX4_PORT_CAP_IS_SM = 1 << 1, + MLX4_PORT_CAP_DEV_MGMT_SUP = 1 << 19, +}; + +enum { + MLX4_MAX_PORTS = 2, + MLX4_MAX_PORT_PKEYS = 128, + MLX4_MAX_PORT_GIDS = 128 +}; + +/* base qkey for use in sriov tunnel-qp/proxy-qp communication. + * These qkeys must not be allowed for general use. This is a 64k range, + * and to test for violation, we use the mask (protect against future chg). + */ +#define MLX4_RESERVED_QKEY_BASE (0xFFFF0000) +#define MLX4_RESERVED_QKEY_MASK (0xFFFF0000) + +enum { + MLX4_BOARD_ID_LEN = 64 +}; + +enum { + MLX4_MAX_NUM_PF = 16, + MLX4_MAX_NUM_VF = 126, + MLX4_MAX_NUM_VF_P_PORT = 64, + MLX4_MFUNC_MAX = 128, + MLX4_MAX_EQ_NUM = 1024, + MLX4_MFUNC_EQ_NUM = 4, + MLX4_MFUNC_MAX_EQES = 8, + MLX4_MFUNC_EQE_MASK = (MLX4_MFUNC_MAX_EQES - 1) +}; + +/* Driver supports 3 different device methods to manage traffic steering: + * -device managed - High level API for ib and eth flow steering. FW is + * managing flow steering tables. + * - B0 steering mode - Common low level API for ib and (if supported) eth. + * - A0 steering mode - Limited low level API for eth. In case of IB, + * B0 mode is in use. + */ +enum { + MLX4_STEERING_MODE_A0, + MLX4_STEERING_MODE_B0, + MLX4_STEERING_MODE_DEVICE_MANAGED +}; + +enum { + MLX4_STEERING_DMFS_A0_DEFAULT, + MLX4_STEERING_DMFS_A0_DYNAMIC, + MLX4_STEERING_DMFS_A0_STATIC, + MLX4_STEERING_DMFS_A0_DISABLE, + MLX4_STEERING_DMFS_A0_NOT_SUPPORTED +}; + +static inline const char *mlx4_steering_mode_str(int steering_mode) +{ + switch (steering_mode) { + case MLX4_STEERING_MODE_A0: + return "A0 steering"; + + case MLX4_STEERING_MODE_B0: + return "B0 steering"; + + case MLX4_STEERING_MODE_DEVICE_MANAGED: + return "Device managed flow steering"; + + default: + return "Unrecognize steering mode"; + } +} + +enum { + MLX4_TUNNEL_OFFLOAD_MODE_NONE, + MLX4_TUNNEL_OFFLOAD_MODE_VXLAN +}; + +enum { + MLX4_DEV_CAP_FLAG_RC = 1LL << 0, + MLX4_DEV_CAP_FLAG_UC = 1LL << 1, + MLX4_DEV_CAP_FLAG_UD = 1LL << 2, + MLX4_DEV_CAP_FLAG_XRC = 1LL << 3, + MLX4_DEV_CAP_FLAG_SRQ = 1LL << 6, + MLX4_DEV_CAP_FLAG_IPOIB_CSUM = 1LL << 7, + MLX4_DEV_CAP_FLAG_BAD_PKEY_CNTR = 1LL << 8, + MLX4_DEV_CAP_FLAG_BAD_QKEY_CNTR = 1LL << 9, + MLX4_DEV_CAP_FLAG_DPDP = 1LL << 12, + MLX4_DEV_CAP_FLAG_BLH = 1LL << 15, + MLX4_DEV_CAP_FLAG_MEM_WINDOW = 1LL << 16, + MLX4_DEV_CAP_FLAG_APM = 1LL << 17, + MLX4_DEV_CAP_FLAG_ATOMIC = 1LL << 18, + MLX4_DEV_CAP_FLAG_RAW_MCAST = 1LL << 19, + MLX4_DEV_CAP_FLAG_UD_AV_PORT = 1LL << 20, + MLX4_DEV_CAP_FLAG_UD_MCAST = 1LL << 21, + MLX4_DEV_CAP_FLAG_IBOE = 1LL << 30, + MLX4_DEV_CAP_FLAG_UC_LOOPBACK = 1LL << 32, + MLX4_DEV_CAP_FLAG_FCS_KEEP = 1LL << 34, + MLX4_DEV_CAP_FLAG_WOL_PORT1 = 1LL << 37, + MLX4_DEV_CAP_FLAG_WOL_PORT2 = 1LL << 38, + MLX4_DEV_CAP_FLAG_UDP_RSS = 1LL << 40, + MLX4_DEV_CAP_FLAG_VEP_UC_STEER = 1LL << 41, + MLX4_DEV_CAP_FLAG_VEP_MC_STEER = 1LL << 42, + MLX4_DEV_CAP_FLAG_COUNTERS = 1LL << 48, + MLX4_DEV_CAP_FLAG_RSS_IP_FRAG = 1LL << 52, + MLX4_DEV_CAP_FLAG_SET_ETH_SCHED = 1LL << 53, + MLX4_DEV_CAP_FLAG_SENSE_SUPPORT = 1LL << 55, + MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV = 1LL << 59, + MLX4_DEV_CAP_FLAG_64B_EQE = 1LL << 61, + MLX4_DEV_CAP_FLAG_64B_CQE = 1LL << 62 +}; + +enum { + MLX4_DEV_CAP_FLAG2_RSS = 1LL << 0, + MLX4_DEV_CAP_FLAG2_RSS_TOP = 1LL << 1, + MLX4_DEV_CAP_FLAG2_RSS_XOR = 1LL << 2, + MLX4_DEV_CAP_FLAG2_FS_EN = 1LL << 3, + MLX4_DEV_CAP_FLAG2_REASSIGN_MAC_EN = 1LL << 4, + MLX4_DEV_CAP_FLAG2_TS = 1LL << 5, + MLX4_DEV_CAP_FLAG2_VLAN_CONTROL = 1LL << 6, + MLX4_DEV_CAP_FLAG2_FSM = 1LL << 7, + MLX4_DEV_CAP_FLAG2_UPDATE_QP = 1LL << 8, + MLX4_DEV_CAP_FLAG2_DMFS_IPOIB = 1LL << 9, + MLX4_DEV_CAP_FLAG2_VXLAN_OFFLOADS = 1LL << 10, + MLX4_DEV_CAP_FLAG2_MAD_DEMUX = 1LL << 11, + MLX4_DEV_CAP_FLAG2_CQE_STRIDE = 1LL << 12, + MLX4_DEV_CAP_FLAG2_EQE_STRIDE = 1LL << 13, + MLX4_DEV_CAP_FLAG2_ETH_PROT_CTRL = 1LL << 14, + MLX4_DEV_CAP_FLAG2_ETH_BACKPL_AN_REP = 1LL << 15, + MLX4_DEV_CAP_FLAG2_CONFIG_DEV = 1LL << 16, + MLX4_DEV_CAP_FLAG2_SYS_EQS = 1LL << 17, + MLX4_DEV_CAP_FLAG2_80_VFS = 1LL << 18, + MLX4_DEV_CAP_FLAG2_FS_A0 = 1LL << 19, + MLX4_DEV_CAP_FLAG2_RECOVERABLE_ERROR_EVENT = 1LL << 20, + MLX4_DEV_CAP_FLAG2_PORT_REMAP = 1LL << 21, + MLX4_DEV_CAP_FLAG2_QCN = 1LL << 22, + MLX4_DEV_CAP_FLAG2_QP_RATE_LIMIT = 1LL << 23, + MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN = 1LL << 24, + MLX4_DEV_CAP_FLAG2_QOS_VPP = 1LL << 25, + MLX4_DEV_CAP_FLAG2_ETS_CFG = 1LL << 26, + MLX4_DEV_CAP_FLAG2_PORT_BEACON = 1LL << 27, + MLX4_DEV_CAP_FLAG2_IGNORE_FCS = 1LL << 28, + MLX4_DEV_CAP_FLAG2_PHV_EN = 1LL << 29, + MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN = 1LL << 30, + MLX4_DEV_CAP_FLAG2_UPDATE_QP_SRC_CHECK_LB = 1ULL << 31, + MLX4_DEV_CAP_FLAG2_LB_SRC_CHK = 1ULL << 32, + MLX4_DEV_CAP_FLAG2_ROCE_V1_V2 = 1ULL << 33, + MLX4_DEV_CAP_FLAG2_DMFS_UC_MC_SNIFFER = 1ULL << 34, + MLX4_DEV_CAP_FLAG2_DIAG_PER_PORT = 1ULL << 35, + MLX4_DEV_CAP_FLAG2_SVLAN_BY_QP = 1ULL << 36, + MLX4_DEV_CAP_FLAG2_SL_TO_VL_CHANGE_EVENT = 1ULL << 37, +}; + +enum { + MLX4_QUERY_FUNC_FLAGS_BF_RES_QP = 1LL << 0, + MLX4_QUERY_FUNC_FLAGS_A0_RES_QP = 1LL << 1 +}; + +enum { + MLX4_VF_CAP_FLAG_RESET = 1 << 0 +}; + +/* bit enums for an 8-bit flags field indicating special use + * QPs which require special handling in qp_reserve_range. + * Currently, this only includes QPs used by the ETH interface, + * where we expect to use blueflame. These QPs must not have + * bits 6 and 7 set in their qp number. + * + * This enum may use only bits 0..7. + */ +enum { + MLX4_RESERVE_A0_QP = 1 << 6, + MLX4_RESERVE_ETH_BF_QP = 1 << 7, +}; + +enum { + MLX4_DEV_CAP_64B_EQE_ENABLED = 1LL << 0, + MLX4_DEV_CAP_64B_CQE_ENABLED = 1LL << 1, + MLX4_DEV_CAP_CQE_STRIDE_ENABLED = 1LL << 2, + MLX4_DEV_CAP_EQE_STRIDE_ENABLED = 1LL << 3 +}; + +enum { + MLX4_USER_DEV_CAP_LARGE_CQE = 1L << 0 +}; + +enum { + MLX4_FUNC_CAP_64B_EQE_CQE = 1L << 0, + MLX4_FUNC_CAP_EQE_CQE_STRIDE = 1L << 1, + MLX4_FUNC_CAP_DMFS_A0_STATIC = 1L << 2 +}; + + +#define MLX4_ATTR_EXTENDED_PORT_INFO cpu_to_be16(0xff90) + +enum { + MLX4_BMME_FLAG_WIN_TYPE_2B = 1 << 1, + MLX4_BMME_FLAG_LOCAL_INV = 1 << 6, + MLX4_BMME_FLAG_REMOTE_INV = 1 << 7, + MLX4_BMME_FLAG_TYPE_2_WIN = 1 << 9, + MLX4_BMME_FLAG_RESERVED_LKEY = 1 << 10, + MLX4_BMME_FLAG_FAST_REG_WR = 1 << 11, + MLX4_BMME_FLAG_ROCE_V1_V2 = 1 << 19, + MLX4_BMME_FLAG_PORT_REMAP = 1 << 24, + MLX4_BMME_FLAG_VSD_INIT2RTR = 1 << 28, +}; + +enum { + MLX4_FLAG_PORT_REMAP = MLX4_BMME_FLAG_PORT_REMAP, + MLX4_FLAG_ROCE_V1_V2 = MLX4_BMME_FLAG_ROCE_V1_V2 +}; + +enum mlx4_event { + MLX4_EVENT_TYPE_COMP = 0x00, + MLX4_EVENT_TYPE_PATH_MIG = 0x01, + MLX4_EVENT_TYPE_COMM_EST = 0x02, + MLX4_EVENT_TYPE_SQ_DRAINED = 0x03, + MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE = 0x13, + MLX4_EVENT_TYPE_SRQ_LIMIT = 0x14, + MLX4_EVENT_TYPE_CQ_ERROR = 0x04, + MLX4_EVENT_TYPE_WQ_CATAS_ERROR = 0x05, + MLX4_EVENT_TYPE_EEC_CATAS_ERROR = 0x06, + MLX4_EVENT_TYPE_PATH_MIG_FAILED = 0x07, + MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10, + MLX4_EVENT_TYPE_WQ_ACCESS_ERROR = 0x11, + MLX4_EVENT_TYPE_SRQ_CATAS_ERROR = 0x12, + MLX4_EVENT_TYPE_LOCAL_CATAS_ERROR = 0x08, + MLX4_EVENT_TYPE_PORT_CHANGE = 0x09, + MLX4_EVENT_TYPE_EQ_OVERFLOW = 0x0f, + MLX4_EVENT_TYPE_ECC_DETECT = 0x0e, + MLX4_EVENT_TYPE_CMD = 0x0a, + MLX4_EVENT_TYPE_VEP_UPDATE = 0x19, + MLX4_EVENT_TYPE_COMM_CHANNEL = 0x18, + MLX4_EVENT_TYPE_OP_REQUIRED = 0x1a, + MLX4_EVENT_TYPE_FATAL_WARNING = 0x1b, + MLX4_EVENT_TYPE_FLR_EVENT = 0x1c, + MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT = 0x1d, + MLX4_EVENT_TYPE_RECOVERABLE_ERROR_EVENT = 0x3e, + MLX4_EVENT_TYPE_NONE = 0xff, +}; + +enum { + MLX4_PORT_CHANGE_SUBTYPE_DOWN = 1, + MLX4_PORT_CHANGE_SUBTYPE_ACTIVE = 4 +}; + +enum { + MLX4_RECOVERABLE_ERROR_EVENT_SUBTYPE_BAD_CABLE = 1, + MLX4_RECOVERABLE_ERROR_EVENT_SUBTYPE_UNSUPPORTED_CABLE = 2, +}; + +enum { + MLX4_FATAL_WARNING_SUBTYPE_WARMING = 0, +}; + +enum slave_port_state { + SLAVE_PORT_DOWN = 0, + SLAVE_PENDING_UP, + SLAVE_PORT_UP, +}; + +enum slave_port_gen_event { + SLAVE_PORT_GEN_EVENT_DOWN = 0, + SLAVE_PORT_GEN_EVENT_UP, + SLAVE_PORT_GEN_EVENT_NONE, +}; + +enum slave_port_state_event { + MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN, + MLX4_PORT_STATE_DEV_EVENT_PORT_UP, + MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID, + MLX4_PORT_STATE_IB_EVENT_GID_INVALID, +}; + +enum { + MLX4_PERM_LOCAL_READ = 1 << 10, + MLX4_PERM_LOCAL_WRITE = 1 << 11, + MLX4_PERM_REMOTE_READ = 1 << 12, + MLX4_PERM_REMOTE_WRITE = 1 << 13, + MLX4_PERM_ATOMIC = 1 << 14, + MLX4_PERM_BIND_MW = 1 << 15, + MLX4_PERM_MASK = 0xFC00 +}; + +enum { + MLX4_OPCODE_NOP = 0x00, + MLX4_OPCODE_SEND_INVAL = 0x01, + MLX4_OPCODE_RDMA_WRITE = 0x08, + MLX4_OPCODE_RDMA_WRITE_IMM = 0x09, + MLX4_OPCODE_SEND = 0x0a, + MLX4_OPCODE_SEND_IMM = 0x0b, + MLX4_OPCODE_LSO = 0x0e, + MLX4_OPCODE_RDMA_READ = 0x10, + MLX4_OPCODE_ATOMIC_CS = 0x11, + MLX4_OPCODE_ATOMIC_FA = 0x12, + MLX4_OPCODE_MASKED_ATOMIC_CS = 0x14, + MLX4_OPCODE_MASKED_ATOMIC_FA = 0x15, + MLX4_OPCODE_BIND_MW = 0x18, + MLX4_OPCODE_FMR = 0x19, + MLX4_OPCODE_LOCAL_INVAL = 0x1b, + MLX4_OPCODE_CONFIG_CMD = 0x1f, + + MLX4_RECV_OPCODE_RDMA_WRITE_IMM = 0x00, + MLX4_RECV_OPCODE_SEND = 0x01, + MLX4_RECV_OPCODE_SEND_IMM = 0x02, + MLX4_RECV_OPCODE_SEND_INVAL = 0x03, + + MLX4_CQE_OPCODE_ERROR = 0x1e, + MLX4_CQE_OPCODE_RESIZE = 0x16, +}; + +enum { + MLX4_STAT_RATE_OFFSET = 5 +}; + +enum mlx4_protocol { + MLX4_PROT_IB_IPV6 = 0, + MLX4_PROT_ETH, + MLX4_PROT_IB_IPV4, + MLX4_PROT_FCOE +}; + +enum { + MLX4_MTT_FLAG_PRESENT = 1 +}; + +enum mlx4_qp_region { + MLX4_QP_REGION_FW = 0, + MLX4_QP_REGION_RSS_RAW_ETH, + MLX4_QP_REGION_BOTTOM = MLX4_QP_REGION_RSS_RAW_ETH, + MLX4_QP_REGION_ETH_ADDR, + MLX4_QP_REGION_FC_ADDR, + MLX4_QP_REGION_FC_EXCH, + MLX4_NUM_QP_REGION +}; + +enum mlx4_port_type { + MLX4_PORT_TYPE_NONE = 0, + MLX4_PORT_TYPE_IB = 1, + MLX4_PORT_TYPE_ETH = 2, + MLX4_PORT_TYPE_AUTO = 3 +}; + +enum mlx4_special_vlan_idx { + MLX4_NO_VLAN_IDX = 0, + MLX4_VLAN_MISS_IDX, + MLX4_VLAN_REGULAR +}; + +enum mlx4_steer_type { + MLX4_MC_STEER = 0, + MLX4_UC_STEER, + MLX4_NUM_STEERS +}; + +enum { + MLX4_NUM_FEXCH = 64 * 1024, +}; + +enum { + MLX4_MAX_FAST_REG_PAGES = 511, +}; + +enum { + /* + * Max wqe size for rdma read is 512 bytes, so this + * limits our max_sge_rd as the wqe needs to fit: + * - ctrl segment (16 bytes) + * - rdma segment (16 bytes) + * - scatter elements (16 bytes each) + */ + MLX4_MAX_SGE_RD = (512 - 16 - 16) / 16 +}; + +enum { + MLX4_DEV_PMC_SUBTYPE_GUID_INFO = 0x14, + MLX4_DEV_PMC_SUBTYPE_PORT_INFO = 0x15, + MLX4_DEV_PMC_SUBTYPE_PKEY_TABLE = 0x16, + MLX4_DEV_PMC_SUBTYPE_SL_TO_VL_MAP = 0x17, +}; + +/* Port mgmt change event handling */ +enum { + MLX4_EQ_PORT_INFO_MSTR_SM_LID_CHANGE_MASK = 1 << 0, + MLX4_EQ_PORT_INFO_GID_PFX_CHANGE_MASK = 1 << 1, + MLX4_EQ_PORT_INFO_LID_CHANGE_MASK = 1 << 2, + MLX4_EQ_PORT_INFO_CLIENT_REREG_MASK = 1 << 3, + MLX4_EQ_PORT_INFO_MSTR_SM_SL_CHANGE_MASK = 1 << 4, +}; + +union sl2vl_tbl_to_u64 { + u8 sl8[8]; + u64 sl64; +}; + +enum { + MLX4_DEVICE_STATE_UP = 1 << 0, + MLX4_DEVICE_STATE_INTERNAL_ERROR = 1 << 1, +}; + +enum { + MLX4_INTERFACE_STATE_UP = 1 << 0, + MLX4_INTERFACE_STATE_DELETION = 1 << 1, +}; + +#define MSTR_SM_CHANGE_MASK (MLX4_EQ_PORT_INFO_MSTR_SM_SL_CHANGE_MASK | \ + MLX4_EQ_PORT_INFO_MSTR_SM_LID_CHANGE_MASK) + +enum mlx4_module_id { + MLX4_MODULE_ID_SFP = 0x3, + MLX4_MODULE_ID_QSFP = 0xC, + MLX4_MODULE_ID_QSFP_PLUS = 0xD, + MLX4_MODULE_ID_QSFP28 = 0x11, +}; + +enum { /* rl */ + MLX4_QP_RATE_LIMIT_NONE = 0, + MLX4_QP_RATE_LIMIT_KBS = 1, + MLX4_QP_RATE_LIMIT_MBS = 2, + MLX4_QP_RATE_LIMIT_GBS = 3 +}; + +struct mlx4_rate_limit_caps { + u16 num_rates; /* Number of different rates */ + u8 min_unit; + u16 min_val; + u8 max_unit; + u16 max_val; +}; + +static inline u64 mlx4_fw_ver(u64 major, u64 minor, u64 subminor) +{ + return (major << 32) | (minor << 16) | subminor; +} + +struct mlx4_phys_caps { + u32 gid_phys_table_len[MLX4_MAX_PORTS + 1]; + u32 pkey_phys_table_len[MLX4_MAX_PORTS + 1]; + u32 num_phys_eqs; + u32 base_sqpn; + u32 base_proxy_sqpn; + u32 base_tunnel_sqpn; +}; + +struct mlx4_caps { + u64 fw_ver; + u32 function; + int num_ports; + int vl_cap[MLX4_MAX_PORTS + 1]; + int ib_mtu_cap[MLX4_MAX_PORTS + 1]; + __be32 ib_port_def_cap[MLX4_MAX_PORTS + 1]; + u64 def_mac[MLX4_MAX_PORTS + 1]; + int eth_mtu_cap[MLX4_MAX_PORTS + 1]; + int gid_table_len[MLX4_MAX_PORTS + 1]; + int pkey_table_len[MLX4_MAX_PORTS + 1]; + int trans_type[MLX4_MAX_PORTS + 1]; + int vendor_oui[MLX4_MAX_PORTS + 1]; + int wavelength[MLX4_MAX_PORTS + 1]; + u64 trans_code[MLX4_MAX_PORTS + 1]; + int local_ca_ack_delay; + int num_uars; + u32 uar_page_size; + int bf_reg_size; + int bf_regs_per_page; + int max_sq_sg; + int max_rq_sg; + int num_qps; + int max_wqes; + int max_sq_desc_sz; + int max_rq_desc_sz; + int max_qp_init_rdma; + int max_qp_dest_rdma; + int max_tc_eth; + u32 *qp0_qkey; + u32 *qp0_proxy; + u32 *qp1_proxy; + u32 *qp0_tunnel; + u32 *qp1_tunnel; + int num_srqs; + int max_srq_wqes; + int max_srq_sge; + int reserved_srqs; + int num_cqs; + int max_cqes; + int reserved_cqs; + int num_sys_eqs; + int num_eqs; + int reserved_eqs; + int num_comp_vectors; + int num_mpts; + int max_fmr_maps; + int num_mtts; + int fmr_reserved_mtts; + int reserved_mtts; + int reserved_mrws; + int reserved_uars; + int num_mgms; + int num_amgms; + int reserved_mcgs; + int num_qp_per_mgm; + int steering_mode; + int dmfs_high_steer_mode; + int fs_log_max_ucast_qp_range_size; + int num_pds; + int reserved_pds; + int max_xrcds; + int reserved_xrcds; + int mtt_entry_sz; + u32 max_msg_sz; + u32 page_size_cap; + u64 flags; + u64 flags2; + u32 bmme_flags; + u32 reserved_lkey; + u16 stat_rate_support; + u8 port_width_cap[MLX4_MAX_PORTS + 1]; + int max_gso_sz; + int max_rss_tbl_sz; + int reserved_qps_cnt[MLX4_NUM_QP_REGION]; + int reserved_qps; + int reserved_qps_base[MLX4_NUM_QP_REGION]; + int log_num_macs; + int log_num_vlans; + enum mlx4_port_type port_type[MLX4_MAX_PORTS + 1]; + u8 supported_type[MLX4_MAX_PORTS + 1]; + u8 suggested_type[MLX4_MAX_PORTS + 1]; + u8 default_sense[MLX4_MAX_PORTS + 1]; + u32 port_mask[MLX4_MAX_PORTS + 1]; + enum mlx4_port_type possible_type[MLX4_MAX_PORTS + 1]; + u32 max_counters; + u8 port_ib_mtu[MLX4_MAX_PORTS + 1]; + u16 sqp_demux; + u32 eqe_size; + u32 cqe_size; + u8 eqe_factor; + u32 userspace_caps; /* userspace must be aware of these */ + u32 function_caps; /* VFs must be aware of these */ + u16 hca_core_clock; + u64 phys_port_id[MLX4_MAX_PORTS + 1]; + int tunnel_offload_mode; + u8 rx_checksum_flags_port[MLX4_MAX_PORTS + 1]; + u8 phv_bit[MLX4_MAX_PORTS + 1]; + u8 alloc_res_qp_mask; + u32 dmfs_high_rate_qpn_base; + u32 dmfs_high_rate_qpn_range; + u32 vf_caps; + struct mlx4_rate_limit_caps rl_caps; +}; + +struct mlx4_buf_list { + void *buf; + dma_addr_t map; +}; + +struct mlx4_buf { + struct mlx4_buf_list direct; + struct mlx4_buf_list *page_list; + int nbufs; + int npages; + int page_shift; +}; + +struct mlx4_mtt { + u32 offset; + int order; + int page_shift; +}; + +enum { + MLX4_DB_PER_PAGE = PAGE_SIZE / 4 +}; + +struct mlx4_db_pgdir { + struct list_head list; + DECLARE_BITMAP(order0, MLX4_DB_PER_PAGE); + DECLARE_BITMAP(order1, MLX4_DB_PER_PAGE / 2); + unsigned long *bits[2]; + __be32 *db_page; + dma_addr_t db_dma; +}; + +struct mlx4_ib_user_db_page; + +struct mlx4_db { + __be32 *db; + union { + struct mlx4_db_pgdir *pgdir; + struct mlx4_ib_user_db_page *user_page; + } u; + dma_addr_t dma; + int index; + int order; +}; + +struct mlx4_hwq_resources { + struct mlx4_db db; + struct mlx4_mtt mtt; + struct mlx4_buf buf; +}; + +struct mlx4_mr { + struct mlx4_mtt mtt; + u64 iova; + u64 size; + u32 key; + u32 pd; + u32 access; + int enabled; +}; + +enum mlx4_mw_type { + MLX4_MW_TYPE_1 = 1, + MLX4_MW_TYPE_2 = 2, +}; + +struct mlx4_mw { + u32 key; + u32 pd; + enum mlx4_mw_type type; + int enabled; +}; + +struct mlx4_fmr { + struct mlx4_mr mr; + struct mlx4_mpt_entry *mpt; + __be64 *mtts; + dma_addr_t dma_handle; + int max_pages; + int max_maps; + int maps; + u8 page_shift; +}; + +struct mlx4_uar { + unsigned long pfn; + int index; + struct list_head bf_list; + unsigned free_bf_bmap; + void __iomem *map; + void __iomem *bf_map; +}; + +struct mlx4_bf { + unsigned int offset; + int buf_size; + struct mlx4_uar *uar; + void __iomem *reg; +}; + +struct mlx4_cq { + void (*comp) (struct mlx4_cq *); + void (*event) (struct mlx4_cq *, enum mlx4_event); + + struct mlx4_uar *uar; + + u32 cons_index; + + u16 irq; + __be32 *set_ci_db; + __be32 *arm_db; + int arm_sn; + + int cqn; + unsigned vector; + + atomic_t refcount; + struct completion free; + int reset_notify_added; + struct list_head reset_notify; +}; + +struct mlx4_qp { + void (*event) (struct mlx4_qp *, enum mlx4_event); + + int qpn; + + atomic_t refcount; + struct completion free; +}; + +struct mlx4_srq { + void (*event) (struct mlx4_srq *, enum mlx4_event); + + int srqn; + int max; + int max_gs; + int wqe_shift; + + atomic_t refcount; + struct completion free; +}; + +struct mlx4_av { + __be32 port_pd; + u8 reserved1; + u8 g_slid; + __be16 dlid; + u8 reserved2; + u8 gid_index; + u8 stat_rate; + u8 hop_limit; + __be32 sl_tclass_flowlabel; + u8 dgid[16]; +}; + +struct mlx4_eth_av { + __be32 port_pd; + u8 reserved1; + u8 smac_idx; + u16 reserved2; + u8 reserved3; + u8 gid_index; + u8 stat_rate; + u8 hop_limit; + __be32 sl_tclass_flowlabel; + u8 dgid[16]; + u8 s_mac[6]; + u8 reserved4[2]; + __be16 vlan; + u8 mac[ETH_ALEN]; +}; + +union mlx4_ext_av { + struct mlx4_av ib; + struct mlx4_eth_av eth; +}; + +struct mlx4_counter { + u8 reserved1[3]; + u8 counter_mode; + __be32 num_ifc; + u32 reserved2[2]; + __be64 rx_frames; + __be64 rx_bytes; + __be64 tx_frames; + __be64 tx_bytes; +}; + +struct mlx4_quotas { + int qp; + int cq; + int srq; + int mpt; + int mtt; + int counter; + int xrcd; +}; + +struct mlx4_vf_dev { + u8 min_port; + u8 n_ports; +}; + +enum mlx4_pci_status { + MLX4_PCI_STATUS_DISABLED, + MLX4_PCI_STATUS_ENABLED, +}; + +struct mlx4_dev_persistent { + struct pci_dev *pdev; + struct mlx4_dev *dev; + int nvfs[MLX4_MAX_PORTS + 1]; + int num_vfs; + enum mlx4_port_type curr_port_type[MLX4_MAX_PORTS + 1]; + enum mlx4_port_type curr_port_poss_type[MLX4_MAX_PORTS + 1]; + struct work_struct catas_work; + struct workqueue_struct *catas_wq; + struct mutex device_state_mutex; /* protect HW state */ + u8 state; + struct mutex interface_state_mutex; /* protect SW state */ + u8 interface_state; + struct mutex pci_status_mutex; /* sync pci state */ + enum mlx4_pci_status pci_status; +}; + +struct mlx4_dev { + struct mlx4_dev_persistent *persist; + unsigned long flags; + unsigned long num_slaves; + struct mlx4_caps caps; + struct mlx4_phys_caps phys_caps; + struct mlx4_quotas quotas; + struct radix_tree_root qp_table_tree; + u8 rev_id; + u8 port_random_macs; + char board_id[MLX4_BOARD_ID_LEN]; + int numa_node; + int oper_log_mgm_entry_size; + u64 regid_promisc_array[MLX4_MAX_PORTS + 1]; + u64 regid_allmulti_array[MLX4_MAX_PORTS + 1]; + struct mlx4_vf_dev *dev_vfs; + u8 uar_page_shift; +}; + +struct mlx4_clock_params { + u64 offset; + u8 bar; + u8 size; +}; + +struct mlx4_eqe { + u8 reserved1; + u8 type; + u8 reserved2; + u8 subtype; + union { + u32 raw[6]; + struct { + __be32 cqn; + } __packed comp; + struct { + u16 reserved1; + __be16 token; + u32 reserved2; + u8 reserved3[3]; + u8 status; + __be64 out_param; + } __packed cmd; + struct { + __be32 qpn; + } __packed qp; + struct { + __be32 srqn; + } __packed srq; + struct { + __be32 cqn; + u32 reserved1; + u8 reserved2[3]; + u8 syndrome; + } __packed cq_err; + struct { + u32 reserved1[2]; + __be32 port; + } __packed port_change; + struct { + #define COMM_CHANNEL_BIT_ARRAY_SIZE 4 + u32 reserved; + u32 bit_vec[COMM_CHANNEL_BIT_ARRAY_SIZE]; + } __packed comm_channel_arm; + struct { + u8 port; + u8 reserved[3]; + __be64 mac; + } __packed mac_update; + struct { + __be32 slave_id; + } __packed flr_event; + struct { + __be16 current_temperature; + __be16 warning_threshold; + } __packed warming; + struct { + u8 reserved[3]; + u8 port; + union { + struct { + __be16 mstr_sm_lid; + __be16 port_lid; + __be32 changed_attr; + u8 reserved[3]; + u8 mstr_sm_sl; + __be64 gid_prefix; + } __packed port_info; + struct { + __be32 block_ptr; + __be32 tbl_entries_mask; + } __packed tbl_change_info; + struct { + u8 sl2vl_table[8]; + } __packed sl2vl_tbl_change_info; + } params; + } __packed port_mgmt_change; + struct { + u8 reserved[3]; + u8 port; + u32 reserved1[5]; + } __packed bad_cable; + } event; + u8 slave_id; + u8 reserved3[2]; + u8 owner; +} __packed; + +struct mlx4_init_port_param { + int set_guid0; + int set_node_guid; + int set_si_guid; + u16 mtu; + int port_width_cap; + u16 vl_cap; + u16 max_gid; + u16 max_pkey; + u64 guid0; + u64 node_guid; + u64 si_guid; +}; + +#define MAD_IFC_DATA_SZ 192 +/* MAD IFC Mailbox */ +struct mlx4_mad_ifc { + u8 base_version; + u8 mgmt_class; + u8 class_version; + u8 method; + __be16 status; + __be16 class_specific; + __be64 tid; + __be16 attr_id; + __be16 resv; + __be32 attr_mod; + __be64 mkey; + __be16 dr_slid; + __be16 dr_dlid; + u8 reserved[28]; + u8 data[MAD_IFC_DATA_SZ]; +} __packed; + +#define mlx4_foreach_port(port, dev, type) \ + for ((port) = 1; (port) <= (dev)->caps.num_ports; (port)++) \ + if ((type) == (dev)->caps.port_mask[(port)]) + +#define mlx4_foreach_ib_transport_port(port, dev) \ + for ((port) = 1; (port) <= (dev)->caps.num_ports; (port)++) \ + if (((dev)->caps.port_mask[port] == MLX4_PORT_TYPE_IB) || \ + ((dev)->caps.flags & MLX4_DEV_CAP_FLAG_IBOE) || \ + ((dev)->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2)) + +#define MLX4_INVALID_SLAVE_ID 0xFF +#define MLX4_SINK_COUNTER_INDEX(dev) (dev->caps.max_counters - 1) + +void handle_port_mgmt_change_event(struct work_struct *work); + +static inline int mlx4_master_func_num(struct mlx4_dev *dev) +{ + return dev->caps.function; +} + +static inline int mlx4_is_master(struct mlx4_dev *dev) +{ + return dev->flags & MLX4_FLAG_MASTER; +} + +static inline int mlx4_num_reserved_sqps(struct mlx4_dev *dev) +{ + return dev->phys_caps.base_sqpn + 8 + + 16 * MLX4_MFUNC_MAX * !!mlx4_is_master(dev); +} + +static inline int mlx4_is_qp_reserved(struct mlx4_dev *dev, u32 qpn) +{ + return (qpn < dev->phys_caps.base_sqpn + 8 + + 16 * MLX4_MFUNC_MAX * !!mlx4_is_master(dev) && + qpn >= dev->phys_caps.base_sqpn) || + (qpn < dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW]); +} + +static inline int mlx4_is_guest_proxy(struct mlx4_dev *dev, int slave, u32 qpn) +{ + int guest_proxy_base = dev->phys_caps.base_proxy_sqpn + slave * 8; + + if (qpn >= guest_proxy_base && qpn < guest_proxy_base + 8) + return 1; + + return 0; +} + +static inline int mlx4_is_mfunc(struct mlx4_dev *dev) +{ + return dev->flags & (MLX4_FLAG_SLAVE | MLX4_FLAG_MASTER); +} + +static inline int mlx4_is_slave(struct mlx4_dev *dev) +{ + return dev->flags & MLX4_FLAG_SLAVE; +} + +static inline int mlx4_is_eth(struct mlx4_dev *dev, int port) +{ + return dev->caps.port_type[port] == MLX4_PORT_TYPE_IB ? 0 : 1; +} + +int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, + struct mlx4_buf *buf, gfp_t gfp); +void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf); +static inline void *mlx4_buf_offset(struct mlx4_buf *buf, int offset) +{ + if (BITS_PER_LONG == 64 || buf->nbufs == 1) + return (u8 *)buf->direct.buf + offset; + else + return (u8 *)buf->page_list[offset >> PAGE_SHIFT].buf + + (offset & (PAGE_SIZE - 1)); +} + +int mlx4_pd_alloc(struct mlx4_dev *dev, u32 *pdn); +void mlx4_pd_free(struct mlx4_dev *dev, u32 pdn); +int mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn); +void mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn); + +int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar); +void mlx4_uar_free(struct mlx4_dev *dev, struct mlx4_uar *uar); +int mlx4_bf_alloc(struct mlx4_dev *dev, struct mlx4_bf *bf, int node); +void mlx4_bf_free(struct mlx4_dev *dev, struct mlx4_bf *bf); + +int mlx4_mtt_init(struct mlx4_dev *dev, int npages, int page_shift, + struct mlx4_mtt *mtt); +void mlx4_mtt_cleanup(struct mlx4_dev *dev, struct mlx4_mtt *mtt); +u64 mlx4_mtt_addr(struct mlx4_dev *dev, struct mlx4_mtt *mtt); + +int mlx4_mr_alloc(struct mlx4_dev *dev, u32 pd, u64 iova, u64 size, u32 access, + int npages, int page_shift, struct mlx4_mr *mr); +int mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr); +int mlx4_mr_enable(struct mlx4_dev *dev, struct mlx4_mr *mr); +int mlx4_mw_alloc(struct mlx4_dev *dev, u32 pd, enum mlx4_mw_type type, + struct mlx4_mw *mw); +void mlx4_mw_free(struct mlx4_dev *dev, struct mlx4_mw *mw); +int mlx4_mw_enable(struct mlx4_dev *dev, struct mlx4_mw *mw); +int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + int start_index, int npages, u64 *page_list); +int mlx4_buf_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + struct mlx4_buf *buf, gfp_t gfp); + +int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order, + gfp_t gfp); +void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db); + +int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, + int size, int max_direct); +void mlx4_free_hwq_res(struct mlx4_dev *mdev, struct mlx4_hwq_resources *wqres, + int size); + +int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt, + struct mlx4_uar *uar, u64 db_rec, struct mlx4_cq *cq, + unsigned vector, int collapsed, int timestamp_en); +void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq); +int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, + int *base, u8 flags); +void mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt); + +int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp, + gfp_t gfp); +void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp); + +int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, u32 cqn, u16 xrcdn, + struct mlx4_mtt *mtt, u64 db_rec, struct mlx4_srq *srq); +void mlx4_srq_free(struct mlx4_dev *dev, struct mlx4_srq *srq); +int mlx4_srq_arm(struct mlx4_dev *dev, struct mlx4_srq *srq, int limit_watermark); +int mlx4_srq_query(struct mlx4_dev *dev, struct mlx4_srq *srq, int *limit_watermark); + +int mlx4_INIT_PORT(struct mlx4_dev *dev, int port); +int mlx4_CLOSE_PORT(struct mlx4_dev *dev, int port); + +int mlx4_unicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + int block_mcast_loopback, enum mlx4_protocol prot); +int mlx4_unicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + enum mlx4_protocol prot); +int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + u8 port, int block_mcast_loopback, + enum mlx4_protocol protocol, u64 *reg_id); +int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + enum mlx4_protocol protocol, u64 reg_id); + +enum { + MLX4_DOMAIN_UVERBS = 0x1000, + MLX4_DOMAIN_ETHTOOL = 0x2000, + MLX4_DOMAIN_RFS = 0x3000, + MLX4_DOMAIN_NIC = 0x5000, +}; + +enum mlx4_net_trans_rule_id { + MLX4_NET_TRANS_RULE_ID_ETH = 0, + MLX4_NET_TRANS_RULE_ID_IB, + MLX4_NET_TRANS_RULE_ID_IPV6, + MLX4_NET_TRANS_RULE_ID_IPV4, + MLX4_NET_TRANS_RULE_ID_TCP, + MLX4_NET_TRANS_RULE_ID_UDP, + MLX4_NET_TRANS_RULE_ID_VXLAN, + MLX4_NET_TRANS_RULE_NUM, /* should be last */ + MLX4_NET_TRANS_RULE_DUMMY = -1, /* force enum to be signed */ +}; + +extern const u16 __sw_id_hw[]; + +static inline int map_hw_to_sw_id(u16 header_id) +{ + + int i; + for (i = 0; i < MLX4_NET_TRANS_RULE_NUM; i++) { + if (header_id == __sw_id_hw[i]) + return i; + } + return -EINVAL; +} + +enum mlx4_net_trans_promisc_mode { + MLX4_FS_REGULAR = 1, + MLX4_FS_ALL_DEFAULT, + MLX4_FS_MC_DEFAULT, + MLX4_FS_MIRROR_RX_PORT, + MLX4_FS_MIRROR_SX_PORT, + MLX4_FS_UC_SNIFFER, + MLX4_FS_MC_SNIFFER, + MLX4_FS_MODE_NUM, /* should be last */ + MLX4_FS_MODE_DUMMY = -1, /* force enum to be signed */ +}; + +struct mlx4_spec_eth { + u8 dst_mac[ETH_ALEN]; + u8 dst_mac_msk[ETH_ALEN]; + u8 src_mac[ETH_ALEN]; + u8 src_mac_msk[ETH_ALEN]; + u8 ether_type_enable; + __be16 ether_type; + __be16 vlan_id_msk; + __be16 vlan_id; +}; + +struct mlx4_spec_tcp_udp { + __be16 dst_port; + __be16 dst_port_msk; + __be16 src_port; + __be16 src_port_msk; +}; + +struct mlx4_spec_ipv4 { + __be32 dst_ip; + __be32 dst_ip_msk; + __be32 src_ip; + __be32 src_ip_msk; +}; + +struct mlx4_spec_ib { + __be32 l3_qpn; + __be32 qpn_msk; + u8 dst_gid[16]; + u8 dst_gid_msk[16]; +}; + +struct mlx4_spec_vxlan { + __be32 vni; + __be32 vni_mask; + +}; + +struct mlx4_spec_list { + struct list_head list; + enum mlx4_net_trans_rule_id id; + union { + struct mlx4_spec_eth eth; + struct mlx4_spec_ib ib; + struct mlx4_spec_ipv4 ipv4; + struct mlx4_spec_tcp_udp tcp_udp; + struct mlx4_spec_vxlan vxlan; + }; +}; + +enum mlx4_net_trans_hw_rule_queue { + MLX4_NET_TRANS_Q_FIFO, + MLX4_NET_TRANS_Q_LIFO, +}; + +struct mlx4_net_trans_rule { + struct list_head list; + enum mlx4_net_trans_hw_rule_queue queue_mode; + bool exclusive; + bool allow_loopback; + enum mlx4_net_trans_promisc_mode promisc_mode; + u8 port; + u16 priority; + u32 qpn; +}; + +struct mlx4_net_trans_rule_hw_ctrl { + __be16 prio; + u8 type; + u8 flags; + u8 rsvd1; + u8 funcid; + u8 vep; + u8 port; + __be32 qpn; + __be32 rsvd2; +}; + +struct mlx4_net_trans_rule_hw_ib { + u8 size; + u8 rsvd1; + __be16 id; + u32 rsvd2; + __be32 l3_qpn; + __be32 qpn_mask; + u8 dst_gid[16]; + u8 dst_gid_msk[16]; +} __packed; + +struct mlx4_net_trans_rule_hw_eth { + u8 size; + u8 rsvd; + __be16 id; + u8 rsvd1[6]; + u8 dst_mac[6]; + u16 rsvd2; + u8 dst_mac_msk[6]; + u16 rsvd3; + u8 src_mac[6]; + u16 rsvd4; + u8 src_mac_msk[6]; + u8 rsvd5; + u8 ether_type_enable; + __be16 ether_type; + __be16 vlan_tag_msk; + __be16 vlan_tag; +} __packed; + +struct mlx4_net_trans_rule_hw_tcp_udp { + u8 size; + u8 rsvd; + __be16 id; + __be16 rsvd1[3]; + __be16 dst_port; + __be16 rsvd2; + __be16 dst_port_msk; + __be16 rsvd3; + __be16 src_port; + __be16 rsvd4; + __be16 src_port_msk; +} __packed; + +struct mlx4_net_trans_rule_hw_ipv4 { + u8 size; + u8 rsvd; + __be16 id; + __be32 rsvd1; + __be32 dst_ip; + __be32 dst_ip_msk; + __be32 src_ip; + __be32 src_ip_msk; +} __packed; + +struct mlx4_net_trans_rule_hw_vxlan { + u8 size; + u8 rsvd; + __be16 id; + __be32 rsvd1; + __be32 vni; + __be32 vni_mask; +} __packed; + +struct _rule_hw { + union { + struct { + u8 size; + u8 rsvd; + __be16 id; + }; + struct mlx4_net_trans_rule_hw_eth eth; + struct mlx4_net_trans_rule_hw_ib ib; + struct mlx4_net_trans_rule_hw_ipv4 ipv4; + struct mlx4_net_trans_rule_hw_tcp_udp tcp_udp; + struct mlx4_net_trans_rule_hw_vxlan vxlan; + }; +}; + +enum { + VXLAN_STEER_BY_OUTER_MAC = 1 << 0, + VXLAN_STEER_BY_OUTER_VLAN = 1 << 1, + VXLAN_STEER_BY_VSID_VNI = 1 << 2, + VXLAN_STEER_BY_INNER_MAC = 1 << 3, + VXLAN_STEER_BY_INNER_VLAN = 1 << 4, +}; + +enum { + MLX4_OP_MOD_QUERY_TRANSPORT_CI_ERRORS = 0x2, +}; + +int mlx4_flow_steer_promisc_add(struct mlx4_dev *dev, u8 port, u32 qpn, + enum mlx4_net_trans_promisc_mode mode); +int mlx4_flow_steer_promisc_remove(struct mlx4_dev *dev, u8 port, + enum mlx4_net_trans_promisc_mode mode); +int mlx4_multicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port); +int mlx4_multicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port); +int mlx4_unicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port); +int mlx4_unicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port); +int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, u64 mac, u64 clear, u8 mode); + +int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac); +void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, u64 mac); +int mlx4_get_base_qpn(struct mlx4_dev *dev, u8 port); +int __mlx4_replace_mac(struct mlx4_dev *dev, u8 port, int qpn, u64 new_mac); +int mlx4_SET_PORT_general(struct mlx4_dev *dev, u8 port, int mtu, + u8 pptx, u8 pfctx, u8 pprx, u8 pfcrx); +int mlx4_SET_PORT_qpn_calc(struct mlx4_dev *dev, u8 port, u32 base_qpn, + u8 promisc); +int mlx4_SET_PORT_BEACON(struct mlx4_dev *dev, u8 port, u16 time); +int mlx4_SET_PORT_fcs_check(struct mlx4_dev *dev, u8 port, + u8 ignore_fcs_value); +int mlx4_SET_PORT_VXLAN(struct mlx4_dev *dev, u8 port, u8 steering, int enable); +int set_phv_bit(struct mlx4_dev *dev, u8 port, int new_val); +int get_phv_bit(struct mlx4_dev *dev, u8 port, int *phv); +int mlx4_get_is_vlan_offload_disabled(struct mlx4_dev *dev, u8 port, + bool *vlan_offload_disabled); +int mlx4_find_cached_mac(struct mlx4_dev *dev, u8 port, u64 mac, int *idx); +int mlx4_find_cached_vlan(struct mlx4_dev *dev, u8 port, u16 vid, int *idx); +int mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, int *index); +void mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, u16 vlan); + +int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list, + int npages, u64 iova, u32 *lkey, u32 *rkey); +int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages, + int max_maps, u8 page_shift, struct mlx4_fmr *fmr); +int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr); +void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr, + u32 *lkey, u32 *rkey); +int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr); +int mlx4_SYNC_TPT(struct mlx4_dev *dev); +int mlx4_test_interrupt(struct mlx4_dev *dev, int vector); +int mlx4_test_async(struct mlx4_dev *dev); +int mlx4_query_diag_counters(struct mlx4_dev *dev, u8 op_modifier, + const u32 offset[], u32 value[], + size_t array_len, u8 port); +u32 mlx4_get_eqs_per_port(struct mlx4_dev *dev, u8 port); +bool mlx4_is_eq_vector_valid(struct mlx4_dev *dev, u8 port, int vector); +int mlx4_assign_eq(struct mlx4_dev *dev, u8 port, int *vector); +void mlx4_release_eq(struct mlx4_dev *dev, int vec); + +int mlx4_is_eq_shared(struct mlx4_dev *dev, int vector); +int mlx4_eq_get_irq(struct mlx4_dev *dev, int vec); + +int mlx4_get_phys_port_id(struct mlx4_dev *dev); +int mlx4_wol_read(struct mlx4_dev *dev, u64 *config, int port); +int mlx4_wol_write(struct mlx4_dev *dev, u64 config, int port); + +int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx); +void mlx4_counter_free(struct mlx4_dev *dev, u32 idx); +int mlx4_get_default_counter_index(struct mlx4_dev *dev, int port); + +void mlx4_set_admin_guid(struct mlx4_dev *dev, __be64 guid, int entry, + int port); +__be64 mlx4_get_admin_guid(struct mlx4_dev *dev, int entry, int port); +void mlx4_set_random_admin_guid(struct mlx4_dev *dev, int entry, int port); +int mlx4_flow_attach(struct mlx4_dev *dev, + struct mlx4_net_trans_rule *rule, u64 *reg_id); +int mlx4_flow_detach(struct mlx4_dev *dev, u64 reg_id); +int mlx4_map_sw_to_hw_steering_mode(struct mlx4_dev *dev, + enum mlx4_net_trans_promisc_mode flow_type); +int mlx4_map_sw_to_hw_steering_id(struct mlx4_dev *dev, + enum mlx4_net_trans_rule_id id); +int mlx4_hw_rule_sz(struct mlx4_dev *dev, enum mlx4_net_trans_rule_id id); + +int mlx4_tunnel_steer_add(struct mlx4_dev *dev, unsigned char *addr, + int port, int qpn, u16 prio, u64 *reg_id); + +void mlx4_sync_pkey_table(struct mlx4_dev *dev, int slave, int port, + int i, int val); + +int mlx4_get_parav_qkey(struct mlx4_dev *dev, u32 qpn, u32 *qkey); + +int mlx4_is_slave_active(struct mlx4_dev *dev, int slave); +int mlx4_gen_pkey_eqe(struct mlx4_dev *dev, int slave, u8 port); +int mlx4_gen_guid_change_eqe(struct mlx4_dev *dev, int slave, u8 port); +int mlx4_gen_slaves_port_mgt_ev(struct mlx4_dev *dev, u8 port, int attr); +int mlx4_gen_port_state_change_eqe(struct mlx4_dev *dev, int slave, u8 port, u8 port_subtype_change); +enum slave_port_state mlx4_get_slave_port_state(struct mlx4_dev *dev, int slave, u8 port); +int set_and_calc_slave_port_state(struct mlx4_dev *dev, int slave, u8 port, int event, enum slave_port_gen_event *gen_event); + +void mlx4_put_slave_node_guid(struct mlx4_dev *dev, int slave, __be64 guid); +__be64 mlx4_get_slave_node_guid(struct mlx4_dev *dev, int slave); + +int mlx4_get_slave_from_roce_gid(struct mlx4_dev *dev, int port, u8 *gid, + int *slave_id); +int mlx4_get_roce_gid_from_slave(struct mlx4_dev *dev, int port, int slave_id, + u8 *gid); + +int mlx4_FLOW_STEERING_IB_UC_QP_RANGE(struct mlx4_dev *dev, u32 min_range_qpn, + u32 max_range_qpn); + +s64 mlx4_read_clock(struct mlx4_dev *dev); + +struct mlx4_active_ports { + DECLARE_BITMAP(ports, MLX4_MAX_PORTS); +}; +/* Returns a bitmap of the physical ports which are assigned to slave */ +struct mlx4_active_ports mlx4_get_active_ports(struct mlx4_dev *dev, int slave); + +/* Returns the physical port that represents the virtual port of the slave, */ +/* or a value < 0 in case of an error. If a slave has 2 ports, the identity */ +/* mapping is returned. */ +int mlx4_slave_convert_port(struct mlx4_dev *dev, int slave, int port); + +struct mlx4_slaves_pport { + DECLARE_BITMAP(slaves, MLX4_MFUNC_MAX); +}; +/* Returns a bitmap of all slaves that are assigned to port. */ +struct mlx4_slaves_pport mlx4_phys_to_slaves_pport(struct mlx4_dev *dev, + int port); + +/* Returns a bitmap of all slaves that are assigned exactly to all the */ +/* the ports that are set in crit_ports. */ +struct mlx4_slaves_pport mlx4_phys_to_slaves_pport_actv( + struct mlx4_dev *dev, + const struct mlx4_active_ports *crit_ports); + +/* Returns the slave's virtual port that represents the physical port. */ +int mlx4_phys_to_slave_port(struct mlx4_dev *dev, int slave, int port); + +int mlx4_get_base_gid_ix(struct mlx4_dev *dev, int slave, int port); + +int mlx4_config_vxlan_port(struct mlx4_dev *dev, __be16 udp_port); +int mlx4_disable_rx_port_check(struct mlx4_dev *dev, bool dis); +int mlx4_config_roce_v2_port(struct mlx4_dev *dev, u16 udp_port); +int mlx4_virt2phy_port_map(struct mlx4_dev *dev, u32 port1, u32 port2); +int mlx4_vf_smi_enabled(struct mlx4_dev *dev, int slave, int port); +int mlx4_vf_get_enable_smi_admin(struct mlx4_dev *dev, int slave, int port); +int mlx4_vf_set_enable_smi_admin(struct mlx4_dev *dev, int slave, int port, + int enable); +int mlx4_mr_hw_get_mpt(struct mlx4_dev *dev, struct mlx4_mr *mmr, + struct mlx4_mpt_entry ***mpt_entry); +int mlx4_mr_hw_write_mpt(struct mlx4_dev *dev, struct mlx4_mr *mmr, + struct mlx4_mpt_entry **mpt_entry); +int mlx4_mr_hw_change_pd(struct mlx4_dev *dev, struct mlx4_mpt_entry *mpt_entry, + u32 pdn); +int mlx4_mr_hw_change_access(struct mlx4_dev *dev, + struct mlx4_mpt_entry *mpt_entry, + u32 access); +void mlx4_mr_hw_put_mpt(struct mlx4_dev *dev, + struct mlx4_mpt_entry **mpt_entry); +void mlx4_mr_rereg_mem_cleanup(struct mlx4_dev *dev, struct mlx4_mr *mr); +int mlx4_mr_rereg_mem_write(struct mlx4_dev *dev, struct mlx4_mr *mr, + u64 iova, u64 size, int npages, + int page_shift, struct mlx4_mpt_entry *mpt_entry); + +int mlx4_get_module_info(struct mlx4_dev *dev, u8 port, + u16 offset, u16 size, u8 *data); +int mlx4_max_tc(struct mlx4_dev *dev); + +/* Returns true if running in low memory profile (kdump kernel) */ +static inline bool mlx4_low_memory_profile(void) +{ + return false; +} + +/* ACCESS REG commands */ +enum mlx4_access_reg_method { + MLX4_ACCESS_REG_QUERY = 0x1, + MLX4_ACCESS_REG_WRITE = 0x2, +}; + +/* ACCESS PTYS Reg command */ +enum mlx4_ptys_proto { + MLX4_PTYS_IB = 1<<0, + MLX4_PTYS_EN = 1<<2, +}; + +struct mlx4_ptys_reg { + u8 resrvd1; + u8 local_port; + u8 resrvd2; + u8 proto_mask; + __be32 resrvd3[2]; + __be32 eth_proto_cap; + __be16 ib_width_cap; + __be16 ib_speed_cap; + __be32 resrvd4; + __be32 eth_proto_admin; + __be16 ib_width_admin; + __be16 ib_speed_admin; + __be32 resrvd5; + __be32 eth_proto_oper; + __be16 ib_width_oper; + __be16 ib_speed_oper; + __be32 resrvd6; + __be32 eth_proto_lp_adv; +} __packed; + +int mlx4_ACCESS_PTYS_REG(struct mlx4_dev *dev, + enum mlx4_access_reg_method method, + struct mlx4_ptys_reg *ptys_reg); + +int mlx4_get_internal_clock_params(struct mlx4_dev *dev, + struct mlx4_clock_params *params); + +static inline int mlx4_to_hw_uar_index(struct mlx4_dev *dev, int index) +{ + return (index << (PAGE_SHIFT - dev->uar_page_shift)); +} + +static inline int mlx4_get_num_reserved_uar(struct mlx4_dev *dev) +{ + /* The first 128 UARs are used for EQ doorbells */ + return (128 >> (PAGE_SHIFT - dev->uar_page_shift)); +} +#endif /* MLX4_DEVICE_H */ Property changes on: stable/11/sys/dev/mlx4/device.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/driver.h =================================================================== --- stable/11/sys/dev/mlx4/driver.h (nonexistent) +++ stable/11/sys/dev/mlx4/driver.h (revision 329159) @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MLX4_DRIVER_H +#define MLX4_DRIVER_H + +#include + +struct mlx4_dev; + +#define MLX4_MAC_MASK 0xffffffffffffULL + +enum mlx4_dev_event { + MLX4_DEV_EVENT_CATASTROPHIC_ERROR, + MLX4_DEV_EVENT_PORT_UP, + MLX4_DEV_EVENT_PORT_DOWN, + MLX4_DEV_EVENT_PORT_REINIT, + MLX4_DEV_EVENT_PORT_MGMT_CHANGE, + MLX4_DEV_EVENT_SLAVE_INIT, + MLX4_DEV_EVENT_SLAVE_SHUTDOWN, +}; + +enum { + MLX4_INTFF_BONDING = 1 << 0 +}; + +struct mlx4_interface { + void * (*add) (struct mlx4_dev *dev); + void (*remove)(struct mlx4_dev *dev, void *context); + void (*event) (struct mlx4_dev *dev, void *context, + enum mlx4_dev_event event, unsigned long param); + void * (*get_dev)(struct mlx4_dev *dev, void *context, u8 port); + void (*activate)(struct mlx4_dev *dev, void *context); + struct list_head list; + enum mlx4_protocol protocol; + int flags; +}; + +int mlx4_register_interface(struct mlx4_interface *intf); +void mlx4_unregister_interface(struct mlx4_interface *intf); + +int mlx4_bond(struct mlx4_dev *dev); +int mlx4_unbond(struct mlx4_dev *dev); +static inline int mlx4_is_bonded(struct mlx4_dev *dev) +{ + return !!(dev->flags & MLX4_FLAG_BONDED); +} + +static inline int mlx4_is_mf_bonded(struct mlx4_dev *dev) +{ + return (mlx4_is_bonded(dev) && mlx4_is_mfunc(dev)); +} + +struct mlx4_port_map { + u8 port1; + u8 port2; +}; + +int mlx4_port_map_set(struct mlx4_dev *dev, struct mlx4_port_map *v2p); + +void *mlx4_get_protocol_dev(struct mlx4_dev *dev, enum mlx4_protocol proto, int port); + +static inline u64 mlx4_mac_to_u64(const u8 *addr) +{ + u64 mac = 0; + int i; + + for (i = 0; i < ETH_ALEN; i++) { + mac <<= 8; + mac |= addr[i]; + } + return mac; +} + +#endif /* MLX4_DRIVER_H */ Property changes on: stable/11/sys/dev/mlx4/driver.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/fw.h =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/fw.h (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/fw.h (revision 329159) @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * Copyright (c) 2006, 2007 Cisco Systems. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MLX4_FW_H +#define MLX4_FW_H + +#include "mlx4.h" +#include "icm.h" + +struct mlx4_mod_stat_cfg { + u8 log_pg_sz; + u8 log_pg_sz_m; +}; + +struct mlx4_port_cap { + u8 link_state; + u8 supported_port_types; + u8 suggested_type; + u8 default_sense; + u8 log_max_macs; + u8 log_max_vlans; + int ib_mtu; + int max_port_width; + int max_vl; + int max_tc_eth; + int max_gids; + int max_pkeys; + u64 def_mac; + u16 eth_mtu; + int trans_type; + int vendor_oui; + u16 wavelength; + u64 trans_code; + u8 dmfs_optimized_state; +}; + +struct mlx4_dev_cap { + int max_srq_sz; + int max_qp_sz; + int reserved_qps; + int max_qps; + int reserved_srqs; + int max_srqs; + int max_cq_sz; + int reserved_cqs; + int max_cqs; + int max_mpts; + int reserved_eqs; + int max_eqs; + int num_sys_eqs; + int reserved_mtts; + int reserved_mrws; + int max_requester_per_qp; + int max_responder_per_qp; + int max_rdma_global; + int local_ca_ack_delay; + int num_ports; + u32 max_msg_sz; + u16 stat_rate_support; + int fs_log_max_ucast_qp_range_size; + int fs_max_num_qp_per_entry; + u64 flags; + u64 flags2; + int reserved_uars; + int uar_size; + int min_page_sz; + int bf_reg_size; + int bf_regs_per_page; + int max_sq_sg; + int max_sq_desc_sz; + int max_rq_sg; + int max_rq_desc_sz; + int max_qp_per_mcg; + int reserved_mgms; + int max_mcgs; + int reserved_pds; + int max_pds; + int reserved_xrcds; + int max_xrcds; + int qpc_entry_sz; + int rdmarc_entry_sz; + int altc_entry_sz; + int aux_entry_sz; + int srq_entry_sz; + int cqc_entry_sz; + int eqc_entry_sz; + int dmpt_entry_sz; + int cmpt_entry_sz; + int mtt_entry_sz; + int resize_srq; + u32 bmme_flags; + u32 reserved_lkey; + u64 max_icm_sz; + int max_gso_sz; + int max_rss_tbl_sz; + u32 max_counters; + u32 dmfs_high_rate_qpn_base; + u32 dmfs_high_rate_qpn_range; + struct mlx4_rate_limit_caps rl_caps; + struct mlx4_port_cap port_cap[MLX4_MAX_PORTS + 1]; +}; + +struct mlx4_func_cap { + u8 num_ports; + u8 flags; + u32 pf_context_behaviour; + int qp_quota; + int cq_quota; + int srq_quota; + int mpt_quota; + int mtt_quota; + int max_eq; + int reserved_eq; + int mcg_quota; + u32 qp0_qkey; + u32 qp0_tunnel_qpn; + u32 qp0_proxy_qpn; + u32 qp1_tunnel_qpn; + u32 qp1_proxy_qpn; + u32 reserved_lkey; + u8 physical_port; + u8 flags0; + u8 flags1; + u64 phys_port_id; + u32 extra_flags; +}; + +struct mlx4_func { + int bus; + int device; + int function; + int physical_function; + int rsvd_eqs; + int max_eq; + int rsvd_uars; +}; + +struct mlx4_adapter { + char board_id[MLX4_BOARD_ID_LEN]; + u8 inta_pin; +}; + +struct mlx4_init_hca_param { + u64 qpc_base; + u64 rdmarc_base; + u64 auxc_base; + u64 altc_base; + u64 srqc_base; + u64 cqc_base; + u64 eqc_base; + u64 mc_base; + u64 dmpt_base; + u64 cmpt_base; + u64 mtt_base; + u64 global_caps; + u16 log_mc_entry_sz; + u16 log_mc_hash_sz; + u16 hca_core_clock; /* Internal Clock Frequency (in MHz) */ + u8 log_num_qps; + u8 log_num_srqs; + u8 log_num_cqs; + u8 log_num_eqs; + u16 num_sys_eqs; + u8 log_rd_per_qp; + u8 log_mc_table_sz; + u8 log_mpt_sz; + u8 log_uar_sz; + u8 mw_enabled; /* Enable memory windows */ + u8 uar_page_sz; /* log pg sz in 4k chunks */ + u8 steering_mode; /* for QUERY_HCA */ + u8 dmfs_high_steer_mode; /* for QUERY_HCA */ + u64 dev_cap_enabled; + u16 cqe_size; /* For use only when CQE stride feature enabled */ + u16 eqe_size; /* For use only when EQE stride feature enabled */ + u8 rss_ip_frags; + u8 phv_check_en; /* for QUERY_HCA */ +}; + +struct mlx4_init_ib_param { + int port_width; + int vl_cap; + int mtu_cap; + u16 gid_cap; + u16 pkey_cap; + int set_guid0; + u64 guid0; + int set_node_guid; + u64 node_guid; + int set_si_guid; + u64 si_guid; +}; + +struct mlx4_set_ib_param { + int set_si_guid; + int reset_qkey_viol; + u64 si_guid; + u32 cap_mask; +}; + +void mlx4_dev_cap_dump(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap); +int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap); +int mlx4_QUERY_PORT(struct mlx4_dev *dev, int port, struct mlx4_port_cap *port_cap); +int mlx4_QUERY_FUNC_CAP(struct mlx4_dev *dev, u8 gen_or_port, + struct mlx4_func_cap *func_cap); +int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_FUNC(struct mlx4_dev *dev, struct mlx4_func *func, int slave); +int mlx4_MAP_FA(struct mlx4_dev *dev, struct mlx4_icm *icm); +int mlx4_UNMAP_FA(struct mlx4_dev *dev); +int mlx4_RUN_FW(struct mlx4_dev *dev); +int mlx4_QUERY_FW(struct mlx4_dev *dev); +int mlx4_QUERY_ADAPTER(struct mlx4_dev *dev, struct mlx4_adapter *adapter); +int mlx4_INIT_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param); +int mlx4_QUERY_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param); +int mlx4_CLOSE_HCA(struct mlx4_dev *dev, int panic); +int mlx4_map_cmd(struct mlx4_dev *dev, u16 op, struct mlx4_icm *icm, u64 virt); +int mlx4_SET_ICM_SIZE(struct mlx4_dev *dev, u64 icm_size, u64 *aux_pages); +int mlx4_MAP_ICM_AUX(struct mlx4_dev *dev, struct mlx4_icm *icm); +int mlx4_UNMAP_ICM_AUX(struct mlx4_dev *dev); +int mlx4_NOP(struct mlx4_dev *dev); +int mlx4_MOD_STAT_CFG(struct mlx4_dev *dev, struct mlx4_mod_stat_cfg *cfg); +void mlx4_opreq_action(struct work_struct *work); + +#endif /* MLX4_FW_H */ Property changes on: stable/11/sys/dev/mlx4/mlx4_core/fw.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/fw_qos.h =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/fw_qos.h (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/fw_qos.h (revision 329159) @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. + * All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MLX4_FW_QOS_H +#define MLX4_FW_QOS_H + +#include +#include + +#define MLX4_NUM_UP 8 +#define MLX4_NUM_TC 8 + +/* Default supported priorities for VPP allocation */ +#define MLX4_DEFAULT_QOS_PRIO (0) + +/* Derived from FW feature definition, 0 is the default vport fo all QPs */ +#define MLX4_VPP_DEFAULT_VPORT (0) + +struct mlx4_vport_qos_param { + u32 bw_share; + u32 max_avg_bw; + u8 enable; +}; + +/** + * mlx4_SET_PORT_PRIO2TC - This routine maps user priorities to traffic + * classes of a given port and device. + * + * @dev: mlx4_dev. + * @port: Physical port number. + * @prio2tc: Array of TC associated with each priorities. + * + * Returns 0 on success or a negative mlx4_core errno code. + **/ +int mlx4_SET_PORT_PRIO2TC(struct mlx4_dev *dev, u8 port, u8 *prio2tc); + +/** + * mlx4_SET_PORT_SCHEDULER - This routine configures the arbitration between + * traffic classes (ETS) and configured rate limit for traffic classes. + * tc_tx_bw, pg and ratelimit are arrays where each index represents a TC. + * The description for those parameters below refers to a single TC. + * + * @dev: mlx4_dev. + * @port: Physical port number. + * @tc_tx_bw: The percentage of the bandwidth allocated for traffic class + * within a TC group. The sum of the bw_percentage of all the traffic + * classes within a TC group must equal 100% for correct operation. + * @pg: The TC group the traffic class is associated with. + * @ratelimit: The maximal bandwidth allowed for the use by this traffic class. + * + * Returns 0 on success or a negative mlx4_core errno code. + **/ +int mlx4_SET_PORT_SCHEDULER(struct mlx4_dev *dev, u8 port, u8 *tc_tx_bw, + u8 *pg, u16 *ratelimit); +/** + * mlx4_ALLOCATE_VPP_get - Query port VPP availible resources and allocation. + * Before distribution of VPPs to priorities, only availible_vpp is returned. + * After initialization it returns the distribution of VPPs among priorities. + * + * @dev: mlx4_dev. + * @port: Physical port number. + * @availible_vpp: Pointer to variable where number of availible VPPs is stored + * @vpp_p_up: Distribution of VPPs to priorities is stored in this array + * + * Returns 0 on success or a negative mlx4_core errno code. + **/ +int mlx4_ALLOCATE_VPP_get(struct mlx4_dev *dev, u8 port, + u16 *availible_vpp, u8 *vpp_p_up); +/** + * mlx4_ALLOCATE_VPP_set - Distribution of VPPs among differnt priorities. + * The total number of VPPs assigned to all for a port must not exceed + * the value reported by availible_vpp in mlx4_ALLOCATE_VPP_get. + * VPP allocation is allowed only after the port type has been set, + * and while no QPs are open for this port. + * + * @dev: mlx4_dev. + * @port: Physical port number. + * @vpp_p_up: Allocation of VPPs to different priorities. + * + * Returns 0 on success or a negative mlx4_core errno code. + **/ +int mlx4_ALLOCATE_VPP_set(struct mlx4_dev *dev, u8 port, u8 *vpp_p_up); + +/** + * mlx4_SET_VPORT_QOS_get - Query QoS proporties of a Vport. + * Each priority allowed for the Vport is assigned with a share of the BW, + * and a BW limitation. This commands query the current QoS values. + * + * @dev: mlx4_dev. + * @port: Physical port number. + * @vport: Vport id. + * @out_param: Array of mlx4_vport_qos_param that will contain the values. + * + * Returns 0 on success or a negative mlx4_core errno code. + **/ +int mlx4_SET_VPORT_QOS_get(struct mlx4_dev *dev, u8 port, u8 vport, + struct mlx4_vport_qos_param *out_param); + +/** + * mlx4_SET_VPORT_QOS_set - Set QoS proporties of a Vport. + * QoS parameters can be modified at any time, but must be initialized + * before any QP is associated with the VPort. + * + * @dev: mlx4_dev. + * @port: Physical port number. + * @vport: Vport id. + * @out_param: Array of mlx4_vport_qos_param which holds the requested values. + * + * Returns 0 on success or a negative mlx4_core errno code. + **/ +int mlx4_SET_VPORT_QOS_set(struct mlx4_dev *dev, u8 port, u8 vport, + struct mlx4_vport_qos_param *in_param); + +#endif /* MLX4_FW_QOS_H */ Property changes on: stable/11/sys/dev/mlx4/mlx4_core/fw_qos.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/icm.h =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/icm.h (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/icm.h (revision 329159) @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MLX4_ICM_H +#define MLX4_ICM_H + +#include +#include +#include +#include + +#define MLX4_ICM_CHUNK_LEN \ + ((256 - sizeof (struct list_head) - 2 * sizeof (int)) / \ + (sizeof (struct scatterlist))) + +enum { + MLX4_ICM_PAGE_SHIFT = 12, + MLX4_ICM_PAGE_SIZE = 1 << MLX4_ICM_PAGE_SHIFT, +}; + +struct mlx4_icm_chunk { + struct list_head list; + int npages; + int nsg; + struct scatterlist mem[MLX4_ICM_CHUNK_LEN]; +}; + +struct mlx4_icm { + struct list_head chunk_list; + int refcount; +}; + +struct mlx4_icm_iter { + struct mlx4_icm *icm; + struct mlx4_icm_chunk *chunk; + int page_idx; +}; + +struct mlx4_dev; + +struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages, + gfp_t gfp_mask, int coherent); +void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm, int coherent); + +int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj, + gfp_t gfp); +void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj); +int mlx4_table_get_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, + u32 start, u32 end); +void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, + u32 start, u32 end); +int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table, + u64 virt, int obj_size, u32 nobj, int reserved, + int use_lowmem, int use_coherent); +void mlx4_cleanup_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table); +void *mlx4_table_find(struct mlx4_icm_table *table, u32 obj, dma_addr_t *dma_handle); + +static inline void mlx4_icm_first(struct mlx4_icm *icm, + struct mlx4_icm_iter *iter) +{ + iter->icm = icm; + iter->chunk = list_empty(&icm->chunk_list) ? + NULL : list_entry(icm->chunk_list.next, + struct mlx4_icm_chunk, list); + iter->page_idx = 0; +} + +static inline int mlx4_icm_last(struct mlx4_icm_iter *iter) +{ + return !iter->chunk; +} + +static inline void mlx4_icm_next(struct mlx4_icm_iter *iter) +{ + if (++iter->page_idx >= iter->chunk->nsg) { + if (iter->chunk->list.next == &iter->icm->chunk_list) { + iter->chunk = NULL; + return; + } + + iter->chunk = list_entry(iter->chunk->list.next, + struct mlx4_icm_chunk, list); + iter->page_idx = 0; + } +} + +static inline dma_addr_t mlx4_icm_addr(struct mlx4_icm_iter *iter) +{ + return sg_dma_address(&iter->chunk->mem[iter->page_idx]); +} + +static inline unsigned long mlx4_icm_size(struct mlx4_icm_iter *iter) +{ + return sg_dma_len(&iter->chunk->mem[iter->page_idx]); +} + +int mlx4_MAP_ICM_AUX(struct mlx4_dev *dev, struct mlx4_icm *icm); +int mlx4_UNMAP_ICM_AUX(struct mlx4_dev *dev); + +#endif /* MLX4_ICM_H */ Property changes on: stable/11/sys/dev/mlx4/mlx4_core/icm.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4.h =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4.h (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4.h (revision 329159) @@ -0,0 +1,1464 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. + * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * Copyright (c) 2004 Voltaire, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MLX4_H +#define MLX4_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRV_NAME "mlx4_core" +#define PFX DRV_NAME ": " +#define DRV_VERSION "3.4.1" +#define DRV_RELDATE "October 2017" + +#define MLX4_FS_UDP_UC_EN (1 << 1) +#define MLX4_FS_TCP_UC_EN (1 << 2) +#define MLX4_FS_NUM_OF_L2_ADDR 8 +#define MLX4_FS_MGM_LOG_ENTRY_SIZE 7 +#define MLX4_FS_NUM_MCG (1 << 17) + +#define INIT_HCA_TPT_MW_ENABLE (1 << 7) + +#define MLX4_QUERY_IF_STAT_RESET BIT(31) + +#define ETH_P_8021AD 0x88A8 + +enum { + MLX4_HCR_BASE = 0x80680, + MLX4_HCR_SIZE = 0x0001c, + MLX4_CLR_INT_SIZE = 0x00008, + MLX4_SLAVE_COMM_BASE = 0x0, + MLX4_COMM_PAGESIZE = 0x1000, + MLX4_CLOCK_SIZE = 0x00008, + MLX4_COMM_CHAN_CAPS = 0x8, + MLX4_COMM_CHAN_FLAGS = 0xc +}; + +enum { + MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE = 10, + MLX4_MIN_MGM_LOG_ENTRY_SIZE = 7, + MLX4_MAX_MGM_LOG_ENTRY_SIZE = 12, + MLX4_MAX_QP_PER_MGM = 4 * ((1 << MLX4_MAX_MGM_LOG_ENTRY_SIZE) / 16 - 2), + MLX4_MTT_ENTRY_PER_SEG = 8, +}; + +enum { + MLX4_NUM_PDS = 1 << 15 +}; + +enum { + MLX4_CMPT_TYPE_QP = 0, + MLX4_CMPT_TYPE_SRQ = 1, + MLX4_CMPT_TYPE_CQ = 2, + MLX4_CMPT_TYPE_EQ = 3, + MLX4_CMPT_NUM_TYPE +}; + +enum { + MLX4_CMPT_SHIFT = 24, + MLX4_NUM_CMPTS = MLX4_CMPT_NUM_TYPE << MLX4_CMPT_SHIFT +}; + +enum mlx4_mpt_state { + MLX4_MPT_DISABLED = 0, + MLX4_MPT_EN_HW, + MLX4_MPT_EN_SW +}; + +#define MLX4_COMM_TIME 10000 +#define MLX4_COMM_OFFLINE_TIME_OUT 30000 +#define MLX4_COMM_CMD_NA_OP 0x0 + + +enum { + MLX4_COMM_CMD_RESET, + MLX4_COMM_CMD_VHCR0, + MLX4_COMM_CMD_VHCR1, + MLX4_COMM_CMD_VHCR2, + MLX4_COMM_CMD_VHCR_EN, + MLX4_COMM_CMD_VHCR_POST, + MLX4_COMM_CMD_FLR = 254 +}; + +enum { + MLX4_VF_SMI_DISABLED, + MLX4_VF_SMI_ENABLED +}; + +/*The flag indicates that the slave should delay the RESET cmd*/ +#define MLX4_DELAY_RESET_SLAVE 0xbbbbbbb +/*indicates how many retries will be done if we are in the middle of FLR*/ +#define NUM_OF_RESET_RETRIES 10 +#define SLEEP_TIME_IN_RESET (2 * 1000) +enum mlx4_resource { + RES_QP, + RES_CQ, + RES_SRQ, + RES_XRCD, + RES_MPT, + RES_MTT, + RES_MAC, + RES_VLAN, + RES_NPORT_ID, + RES_COUNTER, + RES_FS_RULE, + RES_EQ, + MLX4_NUM_OF_RESOURCE_TYPE +}; + +enum mlx4_alloc_mode { + RES_OP_RESERVE, + RES_OP_RESERVE_AND_MAP, + RES_OP_MAP_ICM, +}; + +enum mlx4_res_tracker_free_type { + RES_TR_FREE_ALL, + RES_TR_FREE_SLAVES_ONLY, + RES_TR_FREE_STRUCTS_ONLY, +}; + +/* + *Virtual HCR structures. + * mlx4_vhcr is the sw representation, in machine endianness + * + * mlx4_vhcr_cmd is the formalized structure, the one that is passed + * to FW to go through communication channel. + * It is big endian, and has the same structure as the physical HCR + * used by command interface + */ +struct mlx4_vhcr { + u64 in_param; + u64 out_param; + u32 in_modifier; + u32 errno; + u16 op; + u16 token; + u8 op_modifier; + u8 e_bit; +}; + +struct mlx4_vhcr_cmd { + __be64 in_param; + __be32 in_modifier; + u32 reserved1; + __be64 out_param; + __be16 token; + u16 reserved; + u8 status; + u8 flags; + __be16 opcode; +}; + +struct mlx4_cmd_info { + u16 opcode; + bool has_inbox; + bool has_outbox; + bool out_is_imm; + bool encode_slave_id; + int (*verify)(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox); + int (*wrapper)(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +}; + +#ifdef CONFIG_MLX4_DEBUG +extern int mlx4_debug_level; +#else /* CONFIG_MLX4_DEBUG */ +#define mlx4_debug_level (0) +#endif /* CONFIG_MLX4_DEBUG */ + +#define mlx4_dbg(mdev, format, ...) \ +do { \ + if (mlx4_debug_level) \ + dev_printk(KERN_DEBUG, \ + &(mdev)->persist->pdev->dev, format, \ + ##__VA_ARGS__); \ +} while (0) + +#define mlx4_err(mdev, format, ...) \ + dev_err(&(mdev)->persist->pdev->dev, format, ##__VA_ARGS__) +#define mlx4_info(mdev, format, ...) \ + dev_info(&(mdev)->persist->pdev->dev, format, ##__VA_ARGS__) +#define mlx4_warn(mdev, format, ...) \ + dev_warn(&(mdev)->persist->pdev->dev, format, ##__VA_ARGS__) + +extern int mlx4_log_num_mgm_entry_size; +extern int log_mtts_per_seg; +extern int mlx4_internal_err_reset; + +#define MLX4_MAX_NUM_SLAVES (min(MLX4_MAX_NUM_PF + MLX4_MAX_NUM_VF, \ + MLX4_MFUNC_MAX)) +#define ALL_SLAVES 0xff + +struct mlx4_bitmap { + u32 last; + u32 top; + u32 max; + u32 reserved_top; + u32 mask; + u32 avail; + u32 effective_len; + spinlock_t lock; + unsigned long *table; +}; + +struct mlx4_buddy { + unsigned long **bits; + unsigned int *num_free; + u32 max_order; + spinlock_t lock; +}; + +struct mlx4_icm; + +struct mlx4_icm_table { + u64 virt; + int num_icm; + u32 num_obj; + int obj_size; + int lowmem; + int coherent; + struct mutex mutex; + struct mlx4_icm **icm; +}; + +#define MLX4_MPT_FLAG_SW_OWNS (0xfUL << 28) +#define MLX4_MPT_FLAG_FREE (0x3UL << 28) +#define MLX4_MPT_FLAG_MIO (1 << 17) +#define MLX4_MPT_FLAG_BIND_ENABLE (1 << 15) +#define MLX4_MPT_FLAG_PHYSICAL (1 << 9) +#define MLX4_MPT_FLAG_REGION (1 << 8) + +#define MLX4_MPT_PD_MASK (0x1FFFFUL) +#define MLX4_MPT_PD_VF_MASK (0xFE0000UL) +#define MLX4_MPT_PD_FLAG_FAST_REG (1 << 27) +#define MLX4_MPT_PD_FLAG_RAE (1 << 28) +#define MLX4_MPT_PD_FLAG_EN_INV (3 << 24) + +#define MLX4_MPT_QP_FLAG_BOUND_QP (1 << 7) + +#define MLX4_MPT_STATUS_SW 0xF0 +#define MLX4_MPT_STATUS_HW 0x00 + +#define MLX4_CQE_SIZE_MASK_STRIDE 0x3 +#define MLX4_EQE_SIZE_MASK_STRIDE 0x30 + +#define MLX4_EQ_ASYNC 0 +#define MLX4_EQ_TO_CQ_VECTOR(vector) ((vector) - \ + !!((int)(vector) >= MLX4_EQ_ASYNC)) +#define MLX4_CQ_TO_EQ_VECTOR(vector) ((vector) + \ + !!((int)(vector) >= MLX4_EQ_ASYNC)) + +/* + * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits. + */ +struct mlx4_mpt_entry { + __be32 flags; + __be32 qpn; + __be32 key; + __be32 pd_flags; + __be64 start; + __be64 length; + __be32 lkey; + __be32 win_cnt; + u8 reserved1[3]; + u8 mtt_rep; + __be64 mtt_addr; + __be32 mtt_sz; + __be32 entity_size; + __be32 first_byte_offset; +} __packed; + +/* + * Must be packed because start is 64 bits but only aligned to 32 bits. + */ +struct mlx4_eq_context { + __be32 flags; + u16 reserved1[3]; + __be16 page_offset; + u8 log_eq_size; + u8 reserved2[4]; + u8 eq_period; + u8 reserved3; + u8 eq_max_count; + u8 reserved4[3]; + u8 intr; + u8 log_page_size; + u8 reserved5[2]; + u8 mtt_base_addr_h; + __be32 mtt_base_addr_l; + u32 reserved6[2]; + __be32 consumer_index; + __be32 producer_index; + u32 reserved7[4]; +}; + +struct mlx4_cq_context { + __be32 flags; + u16 reserved1[3]; + __be16 page_offset; + __be32 logsize_usrpage; + __be16 cq_period; + __be16 cq_max_count; + u8 reserved2[3]; + u8 comp_eqn; + u8 log_page_size; + u8 reserved3[2]; + u8 mtt_base_addr_h; + __be32 mtt_base_addr_l; + __be32 last_notified_index; + __be32 solicit_producer_index; + __be32 consumer_index; + __be32 producer_index; + u32 reserved4[2]; + __be64 db_rec_addr; +}; + +struct mlx4_srq_context { + __be32 state_logsize_srqn; + u8 logstride; + u8 reserved1; + __be16 xrcd; + __be32 pg_offset_cqn; + u32 reserved2; + u8 log_page_size; + u8 reserved3[2]; + u8 mtt_base_addr_h; + __be32 mtt_base_addr_l; + __be32 pd; + __be16 limit_watermark; + __be16 wqe_cnt; + u16 reserved4; + __be16 wqe_counter; + u32 reserved5; + __be64 db_rec_addr; +}; + +struct mlx4_eq { + struct mlx4_dev *dev; + void __iomem *doorbell; + int eqn; + u32 cons_index; + u16 irq; + u16 have_irq; + int nent; + struct mlx4_buf_list *page_list; + struct mlx4_mtt mtt; + u32 ncqs; + struct mlx4_active_ports actv_ports; + u32 ref_count; + int affinity_cpu_id; +}; + +struct mlx4_slave_eqe { + u8 type; + u8 port; + u32 param; +}; + +struct mlx4_slave_event_eq_info { + int eqn; + u16 token; +}; + +struct mlx4_profile { + int num_qp; + int rdmarc_per_qp; + int num_srq; + int num_cq; + int num_mcg; + int num_mpt; + unsigned num_mtt; +}; + +struct mlx4_fw { + u64 clr_int_base; + u64 catas_offset; + u64 comm_base; + u64 clock_offset; + struct mlx4_icm *fw_icm; + struct mlx4_icm *aux_icm; + u32 catas_size; + u16 fw_pages; + u8 clr_int_bar; + u8 catas_bar; + u8 comm_bar; + u8 clock_bar; +}; + +struct mlx4_comm { + u32 slave_write; + u32 slave_read; +}; + +enum { + MLX4_MCAST_CONFIG = 0, + MLX4_MCAST_DISABLE = 1, + MLX4_MCAST_ENABLE = 2, +}; + +#define VLAN_FLTR_SIZE 128 + +struct mlx4_vlan_fltr { + __be32 entry[VLAN_FLTR_SIZE]; +}; + +struct mlx4_mcast_entry { + struct list_head list; + u64 addr; +}; + +struct mlx4_promisc_qp { + struct list_head list; + u32 qpn; +}; + +struct mlx4_steer_index { + struct list_head list; + unsigned int index; + struct list_head duplicates; +}; + +#define MLX4_EVENT_TYPES_NUM 64 + +struct mlx4_slave_state { + u8 comm_toggle; + u8 last_cmd; + u8 init_port_mask; + bool active; + bool old_vlan_api; + bool vst_qinq_supported; + u8 function; + dma_addr_t vhcr_dma; + u16 mtu[MLX4_MAX_PORTS + 1]; + __be32 ib_cap_mask[MLX4_MAX_PORTS + 1]; + struct mlx4_slave_eqe eq[MLX4_MFUNC_MAX_EQES]; + struct list_head mcast_filters[MLX4_MAX_PORTS + 1]; + struct mlx4_vlan_fltr *vlan_filter[MLX4_MAX_PORTS + 1]; + /* event type to eq number lookup */ + struct mlx4_slave_event_eq_info event_eq[MLX4_EVENT_TYPES_NUM]; + u16 eq_pi; + u16 eq_ci; + spinlock_t lock; + /*initialized via the kzalloc*/ + u8 is_slave_going_down; + u32 cookie; + enum slave_port_state port_state[MLX4_MAX_PORTS + 1]; +}; + +#define MLX4_VGT 4095 +#define NO_INDX (-1) + +struct mlx4_vport_state { + u64 mac; + u16 default_vlan; + u8 default_qos; + __be16 vlan_proto; + u32 tx_rate; + bool spoofchk; + u8 qos_vport; + __be64 guid; +}; + +struct mlx4_vf_admin_state { + struct mlx4_vport_state vport[MLX4_MAX_PORTS + 1]; + u8 enable_smi[MLX4_MAX_PORTS + 1]; +}; + +struct mlx4_vport_oper_state { + struct mlx4_vport_state state; + int mac_idx; + int vlan_idx; +}; + +struct mlx4_vf_oper_state { + struct mlx4_vport_oper_state vport[MLX4_MAX_PORTS + 1]; + u8 smi_enabled[MLX4_MAX_PORTS + 1]; +}; + +struct slave_list { + struct mutex mutex; + struct list_head res_list[MLX4_NUM_OF_RESOURCE_TYPE]; +}; + +struct resource_allocator { + spinlock_t alloc_lock; /* protect quotas */ + union { + int res_reserved; + int res_port_rsvd[MLX4_MAX_PORTS]; + }; + union { + int res_free; + int res_port_free[MLX4_MAX_PORTS]; + }; + int *quota; + int *allocated; + int *guaranteed; +}; + +struct mlx4_resource_tracker { + spinlock_t lock; + /* tree for each resources */ + struct rb_root res_tree[MLX4_NUM_OF_RESOURCE_TYPE]; + /* num_of_slave's lists, one per slave */ + struct slave_list *slave_list; + struct resource_allocator res_alloc[MLX4_NUM_OF_RESOURCE_TYPE]; +}; + +#define SLAVE_EVENT_EQ_SIZE 128 +struct mlx4_slave_event_eq { + u32 eqn; + u32 cons; + u32 prod; + spinlock_t event_lock; + struct mlx4_eqe event_eqe[SLAVE_EVENT_EQ_SIZE]; +}; + +struct mlx4_qos_manager { + int num_of_qos_vfs; + DECLARE_BITMAP(priority_bm, MLX4_NUM_UP); +}; + +struct mlx4_master_qp0_state { + int proxy_qp0_active; + int qp0_active; + int port_active; +}; + +struct mlx4_mfunc_master_ctx { + struct mlx4_slave_state *slave_state; + struct mlx4_vf_admin_state *vf_admin; + struct mlx4_vf_oper_state *vf_oper; + struct mlx4_master_qp0_state qp0_state[MLX4_MAX_PORTS + 1]; + int init_port_ref[MLX4_MAX_PORTS + 1]; + u16 max_mtu[MLX4_MAX_PORTS + 1]; + u8 pptx; + u8 pprx; + int disable_mcast_ref[MLX4_MAX_PORTS + 1]; + struct mlx4_resource_tracker res_tracker; + struct workqueue_struct *comm_wq; + struct work_struct comm_work; + struct work_struct slave_event_work; + struct work_struct slave_flr_event_work; + spinlock_t slave_state_lock; + __be32 comm_arm_bit_vector[4]; + struct mlx4_eqe cmd_eqe; + struct mlx4_slave_event_eq slave_eq; + struct mutex gen_eqe_mutex[MLX4_MFUNC_MAX]; + struct mlx4_qos_manager qos_ctl[MLX4_MAX_PORTS + 1]; +}; + +struct mlx4_mfunc { + struct mlx4_comm __iomem *comm; + struct mlx4_vhcr_cmd *vhcr; + dma_addr_t vhcr_dma; + + struct mlx4_mfunc_master_ctx master; +}; + +#define MGM_QPN_MASK 0x00FFFFFF +#define MGM_BLCK_LB_BIT 30 + +struct mlx4_mgm { + __be32 next_gid_index; + __be32 members_count; + u32 reserved[2]; + u8 gid[16]; + __be32 qp[MLX4_MAX_QP_PER_MGM]; +}; + +struct mlx4_cmd { + struct pci_pool *pool; + void __iomem *hcr; + struct mutex slave_cmd_mutex; + struct semaphore poll_sem; + struct semaphore event_sem; + struct rw_semaphore switch_sem; + int max_cmds; + spinlock_t context_lock; + int free_head; + struct mlx4_cmd_context *context; + u16 token_mask; + u8 use_events; + u8 toggle; + u8 comm_toggle; + u8 initialized; +}; + +enum { + MLX4_VF_IMMED_VLAN_FLAG_VLAN = 1 << 0, + MLX4_VF_IMMED_VLAN_FLAG_QOS = 1 << 1, + MLX4_VF_IMMED_VLAN_FLAG_LINK_DISABLE = 1 << 2, +}; +struct mlx4_vf_immed_vlan_work { + struct work_struct work; + struct mlx4_priv *priv; + int flags; + int slave; + int vlan_ix; + int orig_vlan_ix; + u8 port; + u8 qos; + u8 qos_vport; + u16 vlan_id; + u16 orig_vlan_id; + __be16 vlan_proto; +}; + + +struct mlx4_uar_table { + struct mlx4_bitmap bitmap; +}; + +struct mlx4_mr_table { + struct mlx4_bitmap mpt_bitmap; + struct mlx4_buddy mtt_buddy; + u64 mtt_base; + u64 mpt_base; + struct mlx4_icm_table mtt_table; + struct mlx4_icm_table dmpt_table; +}; + +struct mlx4_cq_table { + struct mlx4_bitmap bitmap; + spinlock_t lock; + struct radix_tree_root tree; + struct mlx4_icm_table table; + struct mlx4_icm_table cmpt_table; +}; + +struct mlx4_eq_table { + struct mlx4_bitmap bitmap; + char *irq_names; + void __iomem *clr_int; + void __iomem **uar_map; + u32 clr_mask; + struct mlx4_eq *eq; + struct mlx4_icm_table table; + struct mlx4_icm_table cmpt_table; + int have_irq; + u8 inta_pin; +}; + +struct mlx4_srq_table { + struct mlx4_bitmap bitmap; + spinlock_t lock; + struct radix_tree_root tree; + struct mlx4_icm_table table; + struct mlx4_icm_table cmpt_table; +}; + +enum mlx4_qp_table_zones { + MLX4_QP_TABLE_ZONE_GENERAL, + MLX4_QP_TABLE_ZONE_RSS, + MLX4_QP_TABLE_ZONE_RAW_ETH, + MLX4_QP_TABLE_ZONE_NUM +}; + +struct mlx4_qp_table { + struct mlx4_bitmap *bitmap_gen; + struct mlx4_zone_allocator *zones; + u32 zones_uids[MLX4_QP_TABLE_ZONE_NUM]; + u32 rdmarc_base; + int rdmarc_shift; + spinlock_t lock; + struct mlx4_icm_table qp_table; + struct mlx4_icm_table auxc_table; + struct mlx4_icm_table altc_table; + struct mlx4_icm_table rdmarc_table; + struct mlx4_icm_table cmpt_table; +}; + +struct mlx4_mcg_table { + struct mutex mutex; + struct mlx4_bitmap bitmap; + struct mlx4_icm_table table; +}; + +struct mlx4_catas_err { + u32 __iomem *map; + struct timer_list timer; + struct list_head list; +}; + +#define MLX4_MAX_MAC_NUM 128 +#define MLX4_MAC_TABLE_SIZE (MLX4_MAX_MAC_NUM << 3) + +struct mlx4_mac_table { + __be64 entries[MLX4_MAX_MAC_NUM]; + int refs[MLX4_MAX_MAC_NUM]; + bool is_dup[MLX4_MAX_MAC_NUM]; + struct mutex mutex; + int total; + int max; +}; + +#define MLX4_ROCE_GID_ENTRY_SIZE 16 + +struct mlx4_roce_gid_entry { + u8 raw[MLX4_ROCE_GID_ENTRY_SIZE]; +}; + +struct mlx4_roce_gid_table { + struct mlx4_roce_gid_entry roce_gids[MLX4_ROCE_MAX_GIDS]; + struct mutex mutex; +}; + +#define MLX4_MAX_VLAN_NUM 128 +#define MLX4_VLAN_TABLE_SIZE (MLX4_MAX_VLAN_NUM << 2) + +struct mlx4_vlan_table { + __be32 entries[MLX4_MAX_VLAN_NUM]; + int refs[MLX4_MAX_VLAN_NUM]; + int is_dup[MLX4_MAX_VLAN_NUM]; + struct mutex mutex; + int total; + int max; +}; + +#define SET_PORT_GEN_ALL_VALID 0x7 +#define SET_PORT_PROMISC_SHIFT 31 +#define SET_PORT_MC_PROMISC_SHIFT 30 + +enum { + MCAST_DIRECT_ONLY = 0, + MCAST_DIRECT = 1, + MCAST_DEFAULT = 2 +}; + +struct mlx4_set_port_general_context { + u16 reserved1; + u8 v_ignore_fcs; + u8 flags; + union { + u8 ignore_fcs; + u8 roce_mode; + }; + u8 reserved2; + __be16 mtu; + u8 pptx; + u8 pfctx; + u16 reserved3; + u8 pprx; + u8 pfcrx; + u16 reserved4; + u32 reserved5; + u8 phv_en; + u8 reserved6[3]; +}; + +struct mlx4_set_port_rqp_calc_context { + __be32 base_qpn; + u8 rererved; + u8 n_mac; + u8 n_vlan; + u8 n_prio; + u8 reserved2[3]; + u8 mac_miss; + u8 intra_no_vlan; + u8 no_vlan; + u8 intra_vlan_miss; + u8 vlan_miss; + u8 reserved3[3]; + u8 no_vlan_prio; + __be32 promisc; + __be32 mcast; +}; + +struct mlx4_port_info { + struct mlx4_dev *dev; + int port; + char dev_name[16]; + struct device_attribute port_attr; + enum mlx4_port_type tmp_type; + char dev_mtu_name[16]; + struct device_attribute port_mtu_attr; + struct mlx4_mac_table mac_table; + struct mlx4_vlan_table vlan_table; + struct mlx4_roce_gid_table gid_table; + int base_qpn; +}; + +struct mlx4_sense { + struct mlx4_dev *dev; + u8 do_sense_port[MLX4_MAX_PORTS + 1]; + u8 sense_allowed[MLX4_MAX_PORTS + 1]; + struct delayed_work sense_poll; + int gone; +}; + +struct mlx4_msix_ctl { + DECLARE_BITMAP(pool_bm, MAX_MSIX); + struct mutex pool_lock; +}; + +struct mlx4_steer { + struct list_head promisc_qps[MLX4_NUM_STEERS]; + struct list_head steer_entries[MLX4_NUM_STEERS]; +}; + +enum { + MLX4_PCI_DEV_IS_VF = 1 << 0, + MLX4_PCI_DEV_FORCE_SENSE_PORT = 1 << 1, +}; + +enum { + MLX4_NO_RR = 0, + MLX4_USE_RR = 1, +}; + +struct mlx4_priv { + struct mlx4_dev dev; + + struct list_head dev_list; + struct list_head ctx_list; + spinlock_t ctx_lock; + + int pci_dev_data; + int removed; + + struct list_head pgdir_list; + struct mutex pgdir_mutex; + + struct mlx4_fw fw; + struct mlx4_cmd cmd; + struct mlx4_mfunc mfunc; + + struct mlx4_bitmap pd_bitmap; + struct mlx4_bitmap xrcd_bitmap; + struct mlx4_uar_table uar_table; + struct mlx4_mr_table mr_table; + struct mlx4_cq_table cq_table; + struct mlx4_eq_table eq_table; + struct mlx4_srq_table srq_table; + struct mlx4_qp_table qp_table; + struct mlx4_mcg_table mcg_table; + struct mlx4_bitmap counters_bitmap; + int def_counter[MLX4_MAX_PORTS]; + + struct mlx4_catas_err catas_err; + + void __iomem *clr_base; + + struct mlx4_uar driver_uar; + void __iomem *kar; + struct mlx4_port_info port[MLX4_MAX_PORTS + 1]; + struct mlx4_sense sense; + struct mutex port_mutex; + struct mlx4_msix_ctl msix_ctl; + struct mlx4_steer *steer; + struct list_head bf_list; + struct mutex bf_mutex; + struct io_mapping *bf_mapping; + void __iomem *clock_mapping; + int reserved_mtts; + int fs_hash_mode; + u8 virt2phys_pkey[MLX4_MFUNC_MAX][MLX4_MAX_PORTS][MLX4_MAX_PORT_PKEYS]; + struct mlx4_port_map v2p; /* cached port mapping configuration */ + struct mutex bond_mutex; /* for bond mode */ + __be64 slave_node_guids[MLX4_MFUNC_MAX]; + + atomic_t opreq_count; + struct work_struct opreq_task; +}; + +static inline struct mlx4_priv *mlx4_priv(struct mlx4_dev *dev) +{ + return container_of(dev, struct mlx4_priv, dev); +} + +#define MLX4_SENSE_RANGE (HZ * 3) + +extern struct workqueue_struct *mlx4_wq; + +u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap); +void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr); +u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, + int align, u32 skip_mask); +void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt, + int use_rr); +u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap); +int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, + u32 reserved_bot, u32 resetrved_top); +void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap); + +int mlx4_reset(struct mlx4_dev *dev); + +int mlx4_alloc_eq_table(struct mlx4_dev *dev); +void mlx4_free_eq_table(struct mlx4_dev *dev); + +int mlx4_init_pd_table(struct mlx4_dev *dev); +int mlx4_init_xrcd_table(struct mlx4_dev *dev); +int mlx4_init_uar_table(struct mlx4_dev *dev); +int mlx4_init_mr_table(struct mlx4_dev *dev); +int mlx4_init_eq_table(struct mlx4_dev *dev); +int mlx4_init_cq_table(struct mlx4_dev *dev); +int mlx4_init_qp_table(struct mlx4_dev *dev); +int mlx4_init_srq_table(struct mlx4_dev *dev); +int mlx4_init_mcg_table(struct mlx4_dev *dev); + +void mlx4_cleanup_pd_table(struct mlx4_dev *dev); +void mlx4_cleanup_xrcd_table(struct mlx4_dev *dev); +void mlx4_cleanup_uar_table(struct mlx4_dev *dev); +void mlx4_cleanup_mr_table(struct mlx4_dev *dev); +void mlx4_cleanup_eq_table(struct mlx4_dev *dev); +void mlx4_cleanup_cq_table(struct mlx4_dev *dev); +void mlx4_cleanup_qp_table(struct mlx4_dev *dev); +void mlx4_cleanup_srq_table(struct mlx4_dev *dev); +void mlx4_cleanup_mcg_table(struct mlx4_dev *dev); +int __mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn, gfp_t gfp); +void __mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn); +int __mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn); +void __mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn); +int __mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn); +void __mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn); +int __mlx4_mpt_reserve(struct mlx4_dev *dev); +void __mlx4_mpt_release(struct mlx4_dev *dev, u32 index); +int __mlx4_mpt_alloc_icm(struct mlx4_dev *dev, u32 index, gfp_t gfp); +void __mlx4_mpt_free_icm(struct mlx4_dev *dev, u32 index); +u32 __mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order); +void __mlx4_free_mtt_range(struct mlx4_dev *dev, u32 first_seg, int order); + +int mlx4_WRITE_MTT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SYNC_TPT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SW2HW_MPT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_HW2SW_MPT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_MPT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SW2HW_EQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_CONFIG_DEV_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_DMA_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int __mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, + int *base, u8 flags); +void __mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt); +int __mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac); +void __mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, u64 mac); +int __mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + int start_index, int npages, u64 *page_list); +int __mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx); +void __mlx4_counter_free(struct mlx4_dev *dev, u32 idx); +int mlx4_calc_vf_counters(struct mlx4_dev *dev, int slave, int port, + struct mlx4_counter *data); +int __mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn); +void __mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn); + +void mlx4_start_catas_poll(struct mlx4_dev *dev); +void mlx4_stop_catas_poll(struct mlx4_dev *dev); +int mlx4_catas_init(struct mlx4_dev *dev); +void mlx4_catas_end(struct mlx4_dev *dev); +int mlx4_restart_one(struct pci_dev *pdev); +int mlx4_register_device(struct mlx4_dev *dev); +void mlx4_unregister_device(struct mlx4_dev *dev); +void mlx4_dispatch_event(struct mlx4_dev *dev, enum mlx4_dev_event type, + unsigned long param); + +struct mlx4_dev_cap; +struct mlx4_init_hca_param; + +u64 mlx4_make_profile(struct mlx4_dev *dev, + struct mlx4_profile *request, + struct mlx4_dev_cap *dev_cap, + struct mlx4_init_hca_param *init_hca); +void mlx4_master_comm_channel(struct work_struct *work); +void mlx4_gen_slave_eqe(struct work_struct *work); +void mlx4_master_handle_slave_flr(struct work_struct *work); + +int mlx4_ALLOC_RES_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_FREE_RES_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_MAP_EQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_COMM_INT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_HW2SW_EQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_EQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SW2HW_CQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_HW2SW_CQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_CQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_MODIFY_CQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SW2HW_SRQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_HW2SW_SRQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_SRQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_ARM_SRQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_GEN_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_RST2INIT_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_INIT2INIT_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_INIT2RTR_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_RTR2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_RTS2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SQERR2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_2ERR_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_RTS2SQD_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SQD2SQD_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SQD2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_2RST_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); + +int mlx4_GEN_EQE(struct mlx4_dev *dev, int slave, struct mlx4_eqe *eqe); + +enum { + MLX4_CMD_CLEANUP_STRUCT = 1UL << 0, + MLX4_CMD_CLEANUP_POOL = 1UL << 1, + MLX4_CMD_CLEANUP_HCR = 1UL << 2, + MLX4_CMD_CLEANUP_VHCR = 1UL << 3, + MLX4_CMD_CLEANUP_ALL = (MLX4_CMD_CLEANUP_VHCR << 1) - 1 +}; + +int mlx4_cmd_init(struct mlx4_dev *dev); +void mlx4_cmd_cleanup(struct mlx4_dev *dev, int cleanup_mask); +int mlx4_multi_func_init(struct mlx4_dev *dev); +int mlx4_ARM_COMM_CHANNEL(struct mlx4_dev *dev); +void mlx4_multi_func_cleanup(struct mlx4_dev *dev); +void mlx4_cmd_event(struct mlx4_dev *dev, u16 token, u8 status, u64 out_param); +int mlx4_cmd_use_events(struct mlx4_dev *dev); +void mlx4_cmd_use_polling(struct mlx4_dev *dev); + +int mlx4_comm_cmd(struct mlx4_dev *dev, u8 cmd, u16 param, + u16 op, unsigned long timeout); + +void mlx4_cq_tasklet_cb(unsigned long data); +void mlx4_cq_completion(struct mlx4_dev *dev, u32 cqn); +void mlx4_cq_event(struct mlx4_dev *dev, u32 cqn, int event_type); + +void mlx4_qp_event(struct mlx4_dev *dev, u32 qpn, int event_type); + +void mlx4_srq_event(struct mlx4_dev *dev, u32 srqn, int event_type); + +void mlx4_enter_error_state(struct mlx4_dev_persistent *persist); + +int mlx4_SENSE_PORT(struct mlx4_dev *dev, int port, + enum mlx4_port_type *type); +void mlx4_do_sense_ports(struct mlx4_dev *dev, + enum mlx4_port_type *stype, + enum mlx4_port_type *defaults); +void mlx4_start_sense(struct mlx4_dev *dev); +void mlx4_stop_sense(struct mlx4_dev *dev); +void mlx4_sense_init(struct mlx4_dev *dev); +int mlx4_check_port_params(struct mlx4_dev *dev, + enum mlx4_port_type *port_type); +int mlx4_change_port_types(struct mlx4_dev *dev, + enum mlx4_port_type *port_types); + +void mlx4_init_mac_table(struct mlx4_dev *dev, struct mlx4_mac_table *table); +void mlx4_init_vlan_table(struct mlx4_dev *dev, struct mlx4_vlan_table *table); +void mlx4_init_roce_gid_table(struct mlx4_dev *dev, + struct mlx4_roce_gid_table *table); +void __mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, u16 vlan); +int __mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, int *index); +int mlx4_bond_vlan_table(struct mlx4_dev *dev); +int mlx4_unbond_vlan_table(struct mlx4_dev *dev); +int mlx4_bond_mac_table(struct mlx4_dev *dev); +int mlx4_unbond_mac_table(struct mlx4_dev *dev); + +int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port, int pkey_tbl_sz); +/* resource tracker functions*/ +int mlx4_get_slave_from_resource_id(struct mlx4_dev *dev, + enum mlx4_resource resource_type, + u64 resource_id, int *slave); +void mlx4_delete_all_resources_for_slave(struct mlx4_dev *dev, int slave_id); +void mlx4_reset_roce_gids(struct mlx4_dev *dev, int slave); +int mlx4_init_resource_tracker(struct mlx4_dev *dev); + +void mlx4_free_resource_tracker(struct mlx4_dev *dev, + enum mlx4_res_tracker_free_type type); + +int mlx4_QUERY_FW_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SET_PORT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_INIT_PORT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_CLOSE_PORT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_DEV_CAP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_PORT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_get_port_ib_caps(struct mlx4_dev *dev, u8 port, __be32 *caps); + +int mlx4_get_slave_pkey_gid_tbl_len(struct mlx4_dev *dev, u8 port, + int *gid_tbl_len, int *pkey_tbl_len); + +int mlx4_QP_ATTACH_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); + +int mlx4_UPDATE_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); + +int mlx4_PROMISC_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_qp_detach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + enum mlx4_protocol prot, enum mlx4_steer_type steer); +int mlx4_qp_attach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + int block_mcast_loopback, enum mlx4_protocol prot, + enum mlx4_steer_type steer); +int mlx4_trans_to_dmfs_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, + u8 gid[16], u8 port, + int block_mcast_loopback, + enum mlx4_protocol prot, u64 *reg_id); +int mlx4_SET_MCAST_FLTR_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SET_VLAN_FLTR_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_common_set_vlan_fltr(struct mlx4_dev *dev, int function, + int port, void *buf); +int mlx4_DUMP_ETH_STATS_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_PKEY_TABLE_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_IF_STAT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QP_FLOW_STEERING_ATTACH_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QP_FLOW_STEERING_DETACH_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_ACCESS_REG_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); + +int mlx4_get_mgm_entry_size(struct mlx4_dev *dev); +int mlx4_get_qp_per_mgm(struct mlx4_dev *dev); + +static inline void set_param_l(u64 *arg, u32 val) +{ + *arg = (*arg & 0xffffffff00000000ULL) | (u64) val; +} + +static inline void set_param_h(u64 *arg, u32 val) +{ + *arg = (*arg & 0xffffffff) | ((u64) val << 32); +} + +static inline u32 get_param_l(u64 *arg) +{ + return (u32) (*arg & 0xffffffff); +} + +static inline u32 get_param_h(u64 *arg) +{ + return (u32)(*arg >> 32); +} + +static inline spinlock_t *mlx4_tlock(struct mlx4_dev *dev) +{ + return &mlx4_priv(dev)->mfunc.master.res_tracker.lock; +} + +#define NOT_MASKED_PD_BITS 17 + +void mlx4_vf_immed_vlan_work_handler(struct work_struct *_work); + +void mlx4_init_quotas(struct mlx4_dev *dev); + +/* for VFs, replace zero MACs with randomly-generated MACs at driver start */ +void mlx4_replace_zero_macs(struct mlx4_dev *dev); +int mlx4_get_slave_num_gids(struct mlx4_dev *dev, int slave, int port); +/* Returns the VF index of slave */ +int mlx4_get_vf_indx(struct mlx4_dev *dev, int slave); +int mlx4_config_mad_demux(struct mlx4_dev *dev); +int mlx4_do_bond(struct mlx4_dev *dev, bool enable); +int mlx4_bond_fs_rules(struct mlx4_dev *dev); +int mlx4_unbond_fs_rules(struct mlx4_dev *dev); + +enum mlx4_zone_flags { + MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO = 1UL << 0, + MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO = 1UL << 1, + MLX4_ZONE_FALLBACK_TO_HIGHER_PRIO = 1UL << 2, + MLX4_ZONE_USE_RR = 1UL << 3, +}; + +enum mlx4_zone_alloc_flags { + /* No two objects could overlap between zones. UID + * could be left unused. If this flag is given and + * two overlapped zones are used, an object will be free'd + * from the smallest possible matching zone. + */ + MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP = 1UL << 0, +}; + +struct mlx4_zone_allocator; + +/* Create a new zone allocator */ +struct mlx4_zone_allocator *mlx4_zone_allocator_create(enum mlx4_zone_alloc_flags flags); + +/* Attach a mlx4_bitmap of priority to the zone allocator + * . Allocating an object from this zone adds an offset . + * Similarly, when searching for an object to free, this offset it taken into + * account. The use_rr mlx4_ib parameter for allocating objects from this + * is given through the MLX4_ZONE_USE_RR flag in . + * When an allocation fails, tries to allocate from other zones + * according to the policy set by . is the unique identifier + * received to this zone. + */ +int mlx4_zone_add_one(struct mlx4_zone_allocator *zone_alloc, + struct mlx4_bitmap *bitmap, + u32 flags, + int priority, + int offset, + u32 *puid); + +/* Remove bitmap indicated by from */ +int mlx4_zone_remove_one(struct mlx4_zone_allocator *zone_alloc, u32 uid); + +/* Delete the zone allocator objects with align and skip_mask + * from the mlx4_bitmap whose uid is . The bitmap which we actually + * allocated from is returned in . If the allocation fails, a negative + * number is returned. Otherwise, the offset of the first object is returned. + */ +u32 mlx4_zone_alloc_entries(struct mlx4_zone_allocator *zones, u32 uid, int count, + int align, u32 skip_mask, u32 *puid); + +/* Free objects, start from of the uid from zone_allocator + * . + */ +u32 mlx4_zone_free_entries(struct mlx4_zone_allocator *zones, + u32 uid, u32 obj, u32 count); + +/* If was allocated with MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP, instead of + * specifying the uid when freeing an object, zone allocator could figure it by + * itself. Other parameters are similar to mlx4_zone_free. + */ +u32 mlx4_zone_free_entries_unique(struct mlx4_zone_allocator *zones, u32 obj, u32 count); + +/* Returns a pointer to mlx4_bitmap that was attached to with */ +struct mlx4_bitmap *mlx4_zone_get_bitmap(struct mlx4_zone_allocator *zones, u32 uid); + +#endif /* MLX4_H */ Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_alloc.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_alloc.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_alloc.c (revision 329159) @@ -0,0 +1,834 @@ +/* + * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include + +#include "mlx4.h" + +u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap) +{ + u32 obj; + + spin_lock(&bitmap->lock); + + obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last); + if (obj >= bitmap->max) { + bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) + & bitmap->mask; + obj = find_first_zero_bit(bitmap->table, bitmap->max); + } + + if (obj < bitmap->max) { + set_bit(obj, bitmap->table); + bitmap->last = (obj + 1); + if (bitmap->last == bitmap->max) + bitmap->last = 0; + obj |= bitmap->top; + } else + obj = -1; + + if (obj != -1) + --bitmap->avail; + + spin_unlock(&bitmap->lock); + + return obj; +} + +void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr) +{ + mlx4_bitmap_free_range(bitmap, obj, 1, use_rr); +} + +static unsigned long find_aligned_range(unsigned long *bitmap, + u32 start, u32 nbits, + int len, int align, u32 skip_mask) +{ + unsigned long end, i; + +again: + start = ALIGN(start, align); + + while ((start < nbits) && (test_bit(start, bitmap) || + (start & skip_mask))) + start += align; + + if (start >= nbits) + return -1; + + end = start+len; + if (end > nbits) + return -1; + + for (i = start + 1; i < end; i++) { + if (test_bit(i, bitmap) || ((u32)i & skip_mask)) { + start = i + 1; + goto again; + } + } + + return start; +} + +u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, + int align, u32 skip_mask) +{ + u32 obj; + + if (likely(cnt == 1 && align == 1 && !skip_mask)) + return mlx4_bitmap_alloc(bitmap); + + spin_lock(&bitmap->lock); + + obj = find_aligned_range(bitmap->table, bitmap->last, + bitmap->max, cnt, align, skip_mask); + if (obj >= bitmap->max) { + bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) + & bitmap->mask; + obj = find_aligned_range(bitmap->table, 0, bitmap->max, + cnt, align, skip_mask); + } + + if (obj < bitmap->max) { + bitmap_set(bitmap->table, obj, cnt); + if (obj == bitmap->last) { + bitmap->last = (obj + cnt); + if (bitmap->last >= bitmap->max) + bitmap->last = 0; + } + obj |= bitmap->top; + } else + obj = -1; + + if (obj != -1) + bitmap->avail -= cnt; + + spin_unlock(&bitmap->lock); + + return obj; +} + +u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap) +{ + return bitmap->avail; +} + +static u32 mlx4_bitmap_masked_value(struct mlx4_bitmap *bitmap, u32 obj) +{ + return obj & (bitmap->max + bitmap->reserved_top - 1); +} + +void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt, + int use_rr) +{ + obj &= bitmap->max + bitmap->reserved_top - 1; + + spin_lock(&bitmap->lock); + if (!use_rr) { + bitmap->last = min(bitmap->last, obj); + bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) + & bitmap->mask; + } + bitmap_clear(bitmap->table, obj, cnt); + bitmap->avail += cnt; + spin_unlock(&bitmap->lock); +} + +int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, + u32 reserved_bot, u32 reserved_top) +{ + /* num must be a power of 2 */ + if (num != roundup_pow_of_two(num)) + return -EINVAL; + + bitmap->last = 0; + bitmap->top = 0; + bitmap->max = num - reserved_top; + bitmap->mask = mask; + bitmap->reserved_top = reserved_top; + bitmap->avail = num - reserved_top - reserved_bot; + bitmap->effective_len = bitmap->avail; + spin_lock_init(&bitmap->lock); + bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) * + sizeof (long), GFP_KERNEL); + if (!bitmap->table) + return -ENOMEM; + + bitmap_set(bitmap->table, 0, reserved_bot); + + return 0; +} + +void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap) +{ + kfree(bitmap->table); +} + +struct mlx4_zone_allocator { + struct list_head entries; + struct list_head prios; + u32 last_uid; + u32 mask; + /* protect the zone_allocator from concurrent accesses */ + spinlock_t lock; + enum mlx4_zone_alloc_flags flags; +}; + +struct mlx4_zone_entry { + struct list_head list; + struct list_head prio_list; + u32 uid; + struct mlx4_zone_allocator *allocator; + struct mlx4_bitmap *bitmap; + int use_rr; + int priority; + int offset; + enum mlx4_zone_flags flags; +}; + +struct mlx4_zone_allocator *mlx4_zone_allocator_create(enum mlx4_zone_alloc_flags flags) +{ + struct mlx4_zone_allocator *zones = kmalloc(sizeof(*zones), GFP_KERNEL); + + if (NULL == zones) + return NULL; + + INIT_LIST_HEAD(&zones->entries); + INIT_LIST_HEAD(&zones->prios); + spin_lock_init(&zones->lock); + zones->last_uid = 0; + zones->mask = 0; + zones->flags = flags; + + return zones; +} + +int mlx4_zone_add_one(struct mlx4_zone_allocator *zone_alloc, + struct mlx4_bitmap *bitmap, + u32 flags, + int priority, + int offset, + u32 *puid) +{ + u32 mask = mlx4_bitmap_masked_value(bitmap, (u32)-1); + struct mlx4_zone_entry *it; + struct mlx4_zone_entry *zone = kmalloc(sizeof(*zone), GFP_KERNEL); + + if (NULL == zone) + return -ENOMEM; + + zone->flags = flags; + zone->bitmap = bitmap; + zone->use_rr = (flags & MLX4_ZONE_USE_RR) ? MLX4_USE_RR : 0; + zone->priority = priority; + zone->offset = offset; + + spin_lock(&zone_alloc->lock); + + zone->uid = zone_alloc->last_uid++; + zone->allocator = zone_alloc; + + if (zone_alloc->mask < mask) + zone_alloc->mask = mask; + + list_for_each_entry(it, &zone_alloc->prios, prio_list) + if (it->priority >= priority) + break; + + if (&it->prio_list == &zone_alloc->prios || it->priority > priority) + list_add_tail(&zone->prio_list, &it->prio_list); + list_add_tail(&zone->list, &it->list); + + spin_unlock(&zone_alloc->lock); + + *puid = zone->uid; + + return 0; +} + +/* Should be called under a lock */ +static int __mlx4_zone_remove_one_entry(struct mlx4_zone_entry *entry) +{ + struct mlx4_zone_allocator *zone_alloc = entry->allocator; + + if (!list_empty(&entry->prio_list)) { + /* Check if we need to add an alternative node to the prio list */ + if (!list_is_last(&entry->list, &zone_alloc->entries)) { + struct mlx4_zone_entry *next = list_first_entry(&entry->list, + typeof(*next), + list); + + if (next->priority == entry->priority) + list_add_tail(&next->prio_list, &entry->prio_list); + } + + list_del(&entry->prio_list); + } + + list_del(&entry->list); + + if (zone_alloc->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP) { + u32 mask = 0; + struct mlx4_zone_entry *it; + + list_for_each_entry(it, &zone_alloc->prios, prio_list) { + u32 cur_mask = mlx4_bitmap_masked_value(it->bitmap, (u32)-1); + + if (mask < cur_mask) + mask = cur_mask; + } + zone_alloc->mask = mask; + } + + return 0; +} + +void mlx4_zone_allocator_destroy(struct mlx4_zone_allocator *zone_alloc) +{ + struct mlx4_zone_entry *zone, *tmp; + + spin_lock(&zone_alloc->lock); + + list_for_each_entry_safe(zone, tmp, &zone_alloc->entries, list) { + list_del(&zone->list); + list_del(&zone->prio_list); + kfree(zone); + } + + spin_unlock(&zone_alloc->lock); + kfree(zone_alloc); +} + +/* Should be called under a lock */ +static u32 __mlx4_alloc_from_zone(struct mlx4_zone_entry *zone, int count, + int align, u32 skip_mask, u32 *puid) +{ + u32 uid = 0; + u32 res; + struct mlx4_zone_allocator *zone_alloc = zone->allocator; + struct mlx4_zone_entry *curr_node; + + res = mlx4_bitmap_alloc_range(zone->bitmap, count, + align, skip_mask); + + if (res != (u32)-1) { + res += zone->offset; + uid = zone->uid; + goto out; + } + + list_for_each_entry(curr_node, &zone_alloc->prios, prio_list) { + if (unlikely(curr_node->priority == zone->priority)) + break; + } + + if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO) { + struct mlx4_zone_entry *it = curr_node; + + list_for_each_entry_continue_reverse(it, &zone_alloc->entries, list) { + res = mlx4_bitmap_alloc_range(it->bitmap, count, + align, skip_mask); + if (res != (u32)-1) { + res += it->offset; + uid = it->uid; + goto out; + } + } + } + + if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO) { + struct mlx4_zone_entry *it = curr_node; + + list_for_each_entry_from(it, &zone_alloc->entries, list) { + if (unlikely(it == zone)) + continue; + + if (unlikely(it->priority != curr_node->priority)) + break; + + res = mlx4_bitmap_alloc_range(it->bitmap, count, + align, skip_mask); + if (res != (u32)-1) { + res += it->offset; + uid = it->uid; + goto out; + } + } + } + + if (zone->flags & MLX4_ZONE_FALLBACK_TO_HIGHER_PRIO) { + if (list_is_last(&curr_node->prio_list, &zone_alloc->prios)) + goto out; + + curr_node = list_first_entry(&curr_node->prio_list, + typeof(*curr_node), + prio_list); + + list_for_each_entry_from(curr_node, &zone_alloc->entries, list) { + res = mlx4_bitmap_alloc_range(curr_node->bitmap, count, + align, skip_mask); + if (res != (u32)-1) { + res += curr_node->offset; + uid = curr_node->uid; + goto out; + } + } + } + +out: + if (NULL != puid && res != (u32)-1) + *puid = uid; + return res; +} + +/* Should be called under a lock */ +static void __mlx4_free_from_zone(struct mlx4_zone_entry *zone, u32 obj, + u32 count) +{ + mlx4_bitmap_free_range(zone->bitmap, obj - zone->offset, count, zone->use_rr); +} + +/* Should be called under a lock */ +static struct mlx4_zone_entry *__mlx4_find_zone_by_uid( + struct mlx4_zone_allocator *zones, u32 uid) +{ + struct mlx4_zone_entry *zone; + + list_for_each_entry(zone, &zones->entries, list) { + if (zone->uid == uid) + return zone; + } + + return NULL; +} + +struct mlx4_bitmap *mlx4_zone_get_bitmap(struct mlx4_zone_allocator *zones, u32 uid) +{ + struct mlx4_zone_entry *zone; + struct mlx4_bitmap *bitmap; + + spin_lock(&zones->lock); + + zone = __mlx4_find_zone_by_uid(zones, uid); + + bitmap = zone == NULL ? NULL : zone->bitmap; + + spin_unlock(&zones->lock); + + return bitmap; +} + +int mlx4_zone_remove_one(struct mlx4_zone_allocator *zones, u32 uid) +{ + struct mlx4_zone_entry *zone; + int res; + + spin_lock(&zones->lock); + + zone = __mlx4_find_zone_by_uid(zones, uid); + + if (NULL == zone) { + res = -1; + goto out; + } + + res = __mlx4_zone_remove_one_entry(zone); + +out: + spin_unlock(&zones->lock); + kfree(zone); + + return res; +} + +/* Should be called under a lock */ +static struct mlx4_zone_entry *__mlx4_find_zone_by_uid_unique( + struct mlx4_zone_allocator *zones, u32 obj) +{ + struct mlx4_zone_entry *zone, *zone_candidate = NULL; + u32 dist = (u32)-1; + + /* Search for the smallest zone that this obj could be + * allocated from. This is done in order to handle + * situations when small bitmaps are allocated from bigger + * bitmaps (and the allocated space is marked as reserved in + * the bigger bitmap. + */ + list_for_each_entry(zone, &zones->entries, list) { + if (obj >= zone->offset) { + u32 mobj = (obj - zone->offset) & zones->mask; + + if (mobj < zone->bitmap->max) { + u32 curr_dist = zone->bitmap->effective_len; + + if (curr_dist < dist) { + dist = curr_dist; + zone_candidate = zone; + } + } + } + } + + return zone_candidate; +} + +u32 mlx4_zone_alloc_entries(struct mlx4_zone_allocator *zones, u32 uid, int count, + int align, u32 skip_mask, u32 *puid) +{ + struct mlx4_zone_entry *zone; + int res = -1; + + spin_lock(&zones->lock); + + zone = __mlx4_find_zone_by_uid(zones, uid); + + if (NULL == zone) + goto out; + + res = __mlx4_alloc_from_zone(zone, count, align, skip_mask, puid); + +out: + spin_unlock(&zones->lock); + + return res; +} + +u32 mlx4_zone_free_entries(struct mlx4_zone_allocator *zones, u32 uid, u32 obj, u32 count) +{ + struct mlx4_zone_entry *zone; + int res = 0; + + spin_lock(&zones->lock); + + zone = __mlx4_find_zone_by_uid(zones, uid); + + if (NULL == zone) { + res = -1; + goto out; + } + + __mlx4_free_from_zone(zone, obj, count); + +out: + spin_unlock(&zones->lock); + + return res; +} + +u32 mlx4_zone_free_entries_unique(struct mlx4_zone_allocator *zones, u32 obj, u32 count) +{ + struct mlx4_zone_entry *zone; + int res; + + if (!(zones->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP)) + return -EFAULT; + + spin_lock(&zones->lock); + + zone = __mlx4_find_zone_by_uid_unique(zones, obj); + + if (NULL == zone) { + res = -1; + goto out; + } + + __mlx4_free_from_zone(zone, obj, count); + res = 0; + +out: + spin_unlock(&zones->lock); + + return res; +} +/* + * Handling for queue buffers -- we allocate a bunch of memory and + * register it in a memory region at HCA virtual address 0. If the + * requested size is > max_direct, we split the allocation into + * multiple pages, so we don't require too much contiguous memory. + */ + +int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, + struct mlx4_buf *buf, gfp_t gfp) +{ + dma_addr_t t; + + if (size <= max_direct) { + buf->nbufs = 1; + buf->npages = 1; + buf->page_shift = get_order(size) + PAGE_SHIFT; + buf->direct.buf = dma_alloc_coherent(&dev->persist->pdev->dev, + size, &t, gfp); + if (!buf->direct.buf) + return -ENOMEM; + + buf->direct.map = t; + + while (t & ((1 << buf->page_shift) - 1)) { + --buf->page_shift; + buf->npages *= 2; + } + + memset(buf->direct.buf, 0, size); + } else { + int i; + + buf->direct.buf = NULL; + buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE; + buf->npages = buf->nbufs; + buf->page_shift = PAGE_SHIFT; + buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), + gfp); + if (!buf->page_list) + return -ENOMEM; + + for (i = 0; i < buf->nbufs; ++i) { + buf->page_list[i].buf = + dma_alloc_coherent(&dev->persist->pdev->dev, + PAGE_SIZE, + &t, gfp); + if (!buf->page_list[i].buf) + goto err_free; + + buf->page_list[i].map = t; + + memset(buf->page_list[i].buf, 0, PAGE_SIZE); + } + + if (BITS_PER_LONG == 64) { + struct page **pages; + pages = kmalloc(sizeof *pages * buf->nbufs, gfp); + if (!pages) + goto err_free; + for (i = 0; i < buf->nbufs; ++i) + pages[i] = virt_to_page(buf->page_list[i].buf); + buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL); + kfree(pages); + if (!buf->direct.buf) + goto err_free; + } + } + + return 0; + +err_free: + mlx4_buf_free(dev, size, buf); + + return -ENOMEM; +} +EXPORT_SYMBOL_GPL(mlx4_buf_alloc); + +void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf) +{ + int i; + + if (buf->nbufs == 1) + dma_free_coherent(&dev->persist->pdev->dev, size, + buf->direct.buf, + buf->direct.map); + else { + if (BITS_PER_LONG == 64) + vunmap(buf->direct.buf); + + for (i = 0; i < buf->nbufs; ++i) + if (buf->page_list[i].buf) + dma_free_coherent(&dev->persist->pdev->dev, + PAGE_SIZE, + buf->page_list[i].buf, + buf->page_list[i].map); + kfree(buf->page_list); + } +} +EXPORT_SYMBOL_GPL(mlx4_buf_free); + +static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device, + gfp_t gfp) +{ + struct mlx4_db_pgdir *pgdir; + + pgdir = kzalloc(sizeof *pgdir, gfp); + if (!pgdir) + return NULL; + + bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2); + pgdir->bits[0] = pgdir->order0; + pgdir->bits[1] = pgdir->order1; + pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE, + &pgdir->db_dma, gfp); + if (!pgdir->db_page) { + kfree(pgdir); + return NULL; + } + + return pgdir; +} + +static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, + struct mlx4_db *db, int order) +{ + int o; + int i; + + for (o = order; o <= 1; ++o) { + i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o); + if (i < MLX4_DB_PER_PAGE >> o) + goto found; + } + + return -ENOMEM; + +found: + clear_bit(i, pgdir->bits[o]); + + i <<= o; + + if (o > order) + set_bit(i ^ 1, pgdir->bits[order]); + + db->u.pgdir = pgdir; + db->index = i; + db->db = pgdir->db_page + db->index; + db->dma = pgdir->db_dma + db->index * 4; + db->order = order; + + return 0; +} + +int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order, gfp_t gfp) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_db_pgdir *pgdir; + int ret = 0; + + mutex_lock(&priv->pgdir_mutex); + + list_for_each_entry(pgdir, &priv->pgdir_list, list) + if (!mlx4_alloc_db_from_pgdir(pgdir, db, order)) + goto out; + + pgdir = mlx4_alloc_db_pgdir(&dev->persist->pdev->dev, gfp); + if (!pgdir) { + ret = -ENOMEM; + goto out; + } + + list_add(&pgdir->list, &priv->pgdir_list); + + /* This should never fail -- we just allocated an empty page: */ + WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order)); + +out: + mutex_unlock(&priv->pgdir_mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(mlx4_db_alloc); + +void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int o; + int i; + + mutex_lock(&priv->pgdir_mutex); + + o = db->order; + i = db->index; + + if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { + clear_bit(i ^ 1, db->u.pgdir->order0); + ++o; + } + i >>= o; + set_bit(i, db->u.pgdir->bits[o]); + + if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) { + dma_free_coherent(&dev->persist->pdev->dev, PAGE_SIZE, + db->u.pgdir->db_page, db->u.pgdir->db_dma); + list_del(&db->u.pgdir->list); + kfree(db->u.pgdir); + } + + mutex_unlock(&priv->pgdir_mutex); +} +EXPORT_SYMBOL_GPL(mlx4_db_free); + +int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, + int size, int max_direct) +{ + int err; + + err = mlx4_db_alloc(dev, &wqres->db, 1, GFP_KERNEL); + if (err) + return err; + + *wqres->db.db = 0; + + err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf, GFP_KERNEL); + if (err) + goto err_db; + + err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift, + &wqres->mtt); + if (err) + goto err_buf; + + err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf, GFP_KERNEL); + if (err) + goto err_mtt; + + return 0; + +err_mtt: + mlx4_mtt_cleanup(dev, &wqres->mtt); +err_buf: + mlx4_buf_free(dev, size, &wqres->buf); +err_db: + mlx4_db_free(dev, &wqres->db); + + return err; +} +EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res); + +void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, + int size) +{ + mlx4_mtt_cleanup(dev, &wqres->mtt); + mlx4_buf_free(dev, size, &wqres->buf); + mlx4_db_free(dev, &wqres->db); +} +EXPORT_SYMBOL_GPL(mlx4_free_hwq_res); Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_alloc.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_catas.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_catas.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_catas.c (revision 329159) @@ -0,0 +1,329 @@ +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#define LINUXKPI_PARAM_PREFIX mlx4_ + +#include +#include + +#include + +#include "mlx4.h" + +#define MLX4_CATAS_POLL_INTERVAL (5 * HZ) + + + +int mlx4_internal_err_reset = 1; +module_param_named(internal_err_reset, mlx4_internal_err_reset, int, 0644); +MODULE_PARM_DESC(internal_err_reset, + "Reset device on internal errors if non-zero (default 1)"); + +static int read_vendor_id(struct mlx4_dev *dev) +{ + u16 vendor_id = 0; + int ret; + + ret = pci_read_config_word(dev->persist->pdev, 0, &vendor_id); + if (ret) { + mlx4_err(dev, "Failed to read vendor ID, ret=%d\n", ret); + return ret; + } + + if (vendor_id == 0xffff) { + mlx4_err(dev, "PCI can't be accessed to read vendor id\n"); + return -EINVAL; + } + + return 0; +} + +static int mlx4_reset_master(struct mlx4_dev *dev) +{ + int err = 0; + + if (mlx4_is_master(dev)) + mlx4_report_internal_err_comm_event(dev); + + if (!pci_channel_offline(dev->persist->pdev)) { + err = read_vendor_id(dev); + /* If PCI can't be accessed to read vendor ID we assume that its + * link was disabled and chip was already reset. + */ + if (err) + return 0; + + err = mlx4_reset(dev); + if (err) + mlx4_err(dev, "Fail to reset HCA\n"); + } + + return err; +} + +static int mlx4_reset_slave(struct mlx4_dev *dev) +{ +#define COM_CHAN_RST_REQ_OFFSET 0x10 +#define COM_CHAN_RST_ACK_OFFSET 0x08 + + u32 comm_flags; + u32 rst_req; + u32 rst_ack; + unsigned long end; + struct mlx4_priv *priv = mlx4_priv(dev); + + if (pci_channel_offline(dev->persist->pdev)) + return 0; + + comm_flags = swab32(readl((__iomem char *)priv->mfunc.comm + + MLX4_COMM_CHAN_FLAGS)); + if (comm_flags == 0xffffffff) { + mlx4_err(dev, "VF reset is not needed\n"); + return 0; + } + + if (!(dev->caps.vf_caps & MLX4_VF_CAP_FLAG_RESET)) { + mlx4_err(dev, "VF reset is not supported\n"); + return -EOPNOTSUPP; + } + + rst_req = (comm_flags & (u32)(1 << COM_CHAN_RST_REQ_OFFSET)) >> + COM_CHAN_RST_REQ_OFFSET; + rst_ack = (comm_flags & (u32)(1 << COM_CHAN_RST_ACK_OFFSET)) >> + COM_CHAN_RST_ACK_OFFSET; + if (rst_req != rst_ack) { + mlx4_err(dev, "Communication channel isn't sync, fail to send reset\n"); + return -EIO; + } + + rst_req ^= 1; + mlx4_warn(dev, "VF is sending reset request to Firmware\n"); + comm_flags = rst_req << COM_CHAN_RST_REQ_OFFSET; + __raw_writel((__force u32)cpu_to_be32(comm_flags), + (__iomem char *)priv->mfunc.comm + MLX4_COMM_CHAN_FLAGS); + /* Make sure that our comm channel write doesn't + * get mixed in with writes from another CPU. + */ + mmiowb(); + + end = msecs_to_jiffies(MLX4_COMM_TIME) + jiffies; + while (time_before(jiffies, end)) { + comm_flags = swab32(readl((__iomem char *)priv->mfunc.comm + + MLX4_COMM_CHAN_FLAGS)); + rst_ack = (comm_flags & (u32)(1 << COM_CHAN_RST_ACK_OFFSET)) >> + COM_CHAN_RST_ACK_OFFSET; + + /* Reading rst_req again since the communication channel can + * be reset at any time by the PF and all its bits will be + * set to zero. + */ + rst_req = (comm_flags & (u32)(1 << COM_CHAN_RST_REQ_OFFSET)) >> + COM_CHAN_RST_REQ_OFFSET; + + if (rst_ack == rst_req) { + mlx4_warn(dev, "VF Reset succeed\n"); + return 0; + } + cond_resched(); + } + mlx4_err(dev, "Fail to send reset over the communication channel\n"); + return -ETIMEDOUT; +} + +static int mlx4_comm_internal_err(u32 slave_read) +{ + return (u32)COMM_CHAN_EVENT_INTERNAL_ERR == + (slave_read & (u32)COMM_CHAN_EVENT_INTERNAL_ERR) ? 1 : 0; +} + +void mlx4_enter_error_state(struct mlx4_dev_persistent *persist) +{ + int err; + struct mlx4_dev *dev; + + if (!mlx4_internal_err_reset) + return; + + mutex_lock(&persist->device_state_mutex); + if (persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) + goto out; + + dev = persist->dev; + mlx4_err(dev, "device is going to be reset\n"); + if (mlx4_is_slave(dev)) + err = mlx4_reset_slave(dev); + else + err = mlx4_reset_master(dev); + BUG_ON(err != 0); + + dev->persist->state |= MLX4_DEVICE_STATE_INTERNAL_ERROR; + mlx4_err(dev, "device was reset successfully\n"); + mutex_unlock(&persist->device_state_mutex); + + /* At that step HW was already reset, now notify clients */ + mlx4_dispatch_event(dev, MLX4_DEV_EVENT_CATASTROPHIC_ERROR, 0); + mlx4_cmd_wake_completions(dev); + return; + +out: + mutex_unlock(&persist->device_state_mutex); +} + +static void mlx4_handle_error_state(struct mlx4_dev_persistent *persist) +{ + int err = 0; + + mlx4_enter_error_state(persist); + mutex_lock(&persist->interface_state_mutex); + if (persist->interface_state & MLX4_INTERFACE_STATE_UP && + !(persist->interface_state & MLX4_INTERFACE_STATE_DELETION)) { + err = mlx4_restart_one(persist->pdev); + mlx4_info(persist->dev, "mlx4_restart_one was ended, ret=%d\n", + err); + } + mutex_unlock(&persist->interface_state_mutex); +} + +static void dump_err_buf(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + int i; + + mlx4_err(dev, "Internal error detected:\n"); + for (i = 0; i < priv->fw.catas_size; ++i) + mlx4_err(dev, " buf[%02x]: %08x\n", + i, swab32(readl(priv->catas_err.map + i))); +} + +static void poll_catas(unsigned long dev_ptr) +{ + struct mlx4_dev *dev = (struct mlx4_dev *) dev_ptr; + struct mlx4_priv *priv = mlx4_priv(dev); + u32 slave_read; + + if (mlx4_is_slave(dev)) { + slave_read = swab32(readl(&priv->mfunc.comm->slave_read)); + if (mlx4_comm_internal_err(slave_read)) { + mlx4_warn(dev, "Internal error detected on the communication channel\n"); + goto internal_err; + } + } else if (readl(priv->catas_err.map)) { + dump_err_buf(dev); + goto internal_err; + } + + if (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) { + mlx4_warn(dev, "Internal error mark was detected on device\n"); + goto internal_err; + } + + mod_timer(&priv->catas_err.timer, + round_jiffies(jiffies + MLX4_CATAS_POLL_INTERVAL)); + return; + +internal_err: + if (mlx4_internal_err_reset) + queue_work(dev->persist->catas_wq, &dev->persist->catas_work); +} + +static void catas_reset(struct work_struct *work) +{ + struct mlx4_dev_persistent *persist = + container_of(work, struct mlx4_dev_persistent, + catas_work); + + mlx4_handle_error_state(persist); +} + +void mlx4_start_catas_poll(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + phys_addr_t addr; + + INIT_LIST_HEAD(&priv->catas_err.list); + init_timer(&priv->catas_err.timer); + priv->catas_err.map = NULL; + + if (!mlx4_is_slave(dev)) { + addr = pci_resource_start(dev->persist->pdev, + priv->fw.catas_bar) + + priv->fw.catas_offset; + + priv->catas_err.map = ioremap(addr, priv->fw.catas_size * 4); + if (!priv->catas_err.map) { + mlx4_warn(dev, "Failed to map internal error buffer at 0x%llx\n", + (unsigned long long)addr); + return; + } + } + + priv->catas_err.timer.data = (unsigned long) dev; + priv->catas_err.timer.function = poll_catas; + priv->catas_err.timer.expires = + round_jiffies(jiffies + MLX4_CATAS_POLL_INTERVAL); + add_timer(&priv->catas_err.timer); +} + +void mlx4_stop_catas_poll(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + del_timer_sync(&priv->catas_err.timer); + + if (priv->catas_err.map) { + iounmap(priv->catas_err.map); + priv->catas_err.map = NULL; + } + + if (dev->persist->interface_state & MLX4_INTERFACE_STATE_DELETION) + flush_workqueue(dev->persist->catas_wq); +} + +int mlx4_catas_init(struct mlx4_dev *dev) +{ + INIT_WORK(&dev->persist->catas_work, catas_reset); + dev->persist->catas_wq = create_singlethread_workqueue("mlx4_health"); + if (!dev->persist->catas_wq) + return -ENOMEM; + + return 0; +} + +void mlx4_catas_end(struct mlx4_dev *dev) +{ + if (dev->persist->catas_wq) { + destroy_workqueue(dev->persist->catas_wq); + dev->persist->catas_wq = NULL; + } +} Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_catas.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_cmd.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_cmd.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_cmd.c (revision 329159) @@ -0,0 +1,3275 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include "mlx4.h" +#include "fw.h" +#include "fw_qos.h" + +#define CMD_POLL_TOKEN 0xffff +#define INBOX_MASK 0xffffffffffffff00ULL + +#define CMD_CHAN_VER 1 +#define CMD_CHAN_IF_REV 1 + +enum { + /* command completed successfully: */ + CMD_STAT_OK = 0x00, + /* Internal error (such as a bus error) occurred while processing command: */ + CMD_STAT_INTERNAL_ERR = 0x01, + /* Operation/command not supported or opcode modifier not supported: */ + CMD_STAT_BAD_OP = 0x02, + /* Parameter not supported or parameter out of range: */ + CMD_STAT_BAD_PARAM = 0x03, + /* System not enabled or bad system state: */ + CMD_STAT_BAD_SYS_STATE = 0x04, + /* Attempt to access reserved or unallocaterd resource: */ + CMD_STAT_BAD_RESOURCE = 0x05, + /* Requested resource is currently executing a command, or is otherwise busy: */ + CMD_STAT_RESOURCE_BUSY = 0x06, + /* Required capability exceeds device limits: */ + CMD_STAT_EXCEED_LIM = 0x08, + /* Resource is not in the appropriate state or ownership: */ + CMD_STAT_BAD_RES_STATE = 0x09, + /* Index out of range: */ + CMD_STAT_BAD_INDEX = 0x0a, + /* FW image corrupted: */ + CMD_STAT_BAD_NVMEM = 0x0b, + /* Error in ICM mapping (e.g. not enough auxiliary ICM pages to execute command): */ + CMD_STAT_ICM_ERROR = 0x0c, + /* Attempt to modify a QP/EE which is not in the presumed state: */ + CMD_STAT_BAD_QP_STATE = 0x10, + /* Bad segment parameters (Address/Size): */ + CMD_STAT_BAD_SEG_PARAM = 0x20, + /* Memory Region has Memory Windows bound to: */ + CMD_STAT_REG_BOUND = 0x21, + /* HCA local attached memory not present: */ + CMD_STAT_LAM_NOT_PRE = 0x22, + /* Bad management packet (silently discarded): */ + CMD_STAT_BAD_PKT = 0x30, + /* More outstanding CQEs in CQ than new CQ size: */ + CMD_STAT_BAD_SIZE = 0x40, + /* Multi Function device support required: */ + CMD_STAT_MULTI_FUNC_REQ = 0x50, +}; + +enum { + HCR_IN_PARAM_OFFSET = 0x00, + HCR_IN_MODIFIER_OFFSET = 0x08, + HCR_OUT_PARAM_OFFSET = 0x0c, + HCR_TOKEN_OFFSET = 0x14, + HCR_STATUS_OFFSET = 0x18, + + HCR_OPMOD_SHIFT = 12, + HCR_T_BIT = 21, + HCR_E_BIT = 22, + HCR_GO_BIT = 23 +}; + +enum { + GO_BIT_TIMEOUT_MSECS = 10000 +}; + +enum mlx4_vlan_transition { + MLX4_VLAN_TRANSITION_VST_VST = 0, + MLX4_VLAN_TRANSITION_VST_VGT = 1, + MLX4_VLAN_TRANSITION_VGT_VST = 2, + MLX4_VLAN_TRANSITION_VGT_VGT = 3, +}; + + +struct mlx4_cmd_context { + struct completion done; + int result; + int next; + u64 out_param; + u16 token; + u8 fw_status; +}; + +static int mlx4_master_process_vhcr(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr_cmd *in_vhcr); + +static int mlx4_status_to_errno(u8 status) +{ + static const int trans_table[] = { + [CMD_STAT_INTERNAL_ERR] = -EIO, + [CMD_STAT_BAD_OP] = -EPERM, + [CMD_STAT_BAD_PARAM] = -EINVAL, + [CMD_STAT_BAD_SYS_STATE] = -ENXIO, + [CMD_STAT_BAD_RESOURCE] = -EBADF, + [CMD_STAT_RESOURCE_BUSY] = -EBUSY, + [CMD_STAT_EXCEED_LIM] = -ENOMEM, + [CMD_STAT_BAD_RES_STATE] = -EBADF, + [CMD_STAT_BAD_INDEX] = -EBADF, + [CMD_STAT_BAD_NVMEM] = -EFAULT, + [CMD_STAT_ICM_ERROR] = -ENFILE, + [CMD_STAT_BAD_QP_STATE] = -EINVAL, + [CMD_STAT_BAD_SEG_PARAM] = -EFAULT, + [CMD_STAT_REG_BOUND] = -EBUSY, + [CMD_STAT_LAM_NOT_PRE] = -EAGAIN, + [CMD_STAT_BAD_PKT] = -EINVAL, + [CMD_STAT_BAD_SIZE] = -ENOMEM, + [CMD_STAT_MULTI_FUNC_REQ] = -EACCES, + }; + + if (status >= ARRAY_SIZE(trans_table) || + (status != CMD_STAT_OK && trans_table[status] == 0)) + return -EIO; + + return trans_table[status]; +} + +static u8 mlx4_errno_to_status(int errno) +{ + switch (errno) { + case -EPERM: + return CMD_STAT_BAD_OP; + case -EINVAL: + return CMD_STAT_BAD_PARAM; + case -ENXIO: + return CMD_STAT_BAD_SYS_STATE; + case -EBUSY: + return CMD_STAT_RESOURCE_BUSY; + case -ENOMEM: + return CMD_STAT_EXCEED_LIM; + case -ENFILE: + return CMD_STAT_ICM_ERROR; + default: + return CMD_STAT_INTERNAL_ERR; + } +} + +static int mlx4_internal_err_ret_value(struct mlx4_dev *dev, u16 op, + u8 op_modifier) +{ + switch (op) { + case MLX4_CMD_UNMAP_ICM: + case MLX4_CMD_UNMAP_ICM_AUX: + case MLX4_CMD_UNMAP_FA: + case MLX4_CMD_2RST_QP: + case MLX4_CMD_HW2SW_EQ: + case MLX4_CMD_HW2SW_CQ: + case MLX4_CMD_HW2SW_SRQ: + case MLX4_CMD_HW2SW_MPT: + case MLX4_CMD_CLOSE_HCA: + case MLX4_QP_FLOW_STEERING_DETACH: + case MLX4_CMD_FREE_RES: + case MLX4_CMD_CLOSE_PORT: + return CMD_STAT_OK; + + case MLX4_CMD_QP_ATTACH: + /* On Detach case return success */ + if (op_modifier == 0) + return CMD_STAT_OK; + return mlx4_status_to_errno(CMD_STAT_INTERNAL_ERR); + + default: + return mlx4_status_to_errno(CMD_STAT_INTERNAL_ERR); + } +} + +static int mlx4_closing_cmd_fatal_error(u16 op, u8 fw_status) +{ + /* Any error during the closing commands below is considered fatal */ + if (op == MLX4_CMD_CLOSE_HCA || + op == MLX4_CMD_HW2SW_EQ || + op == MLX4_CMD_HW2SW_CQ || + op == MLX4_CMD_2RST_QP || + op == MLX4_CMD_HW2SW_SRQ || + op == MLX4_CMD_SYNC_TPT || + op == MLX4_CMD_UNMAP_ICM || + op == MLX4_CMD_UNMAP_ICM_AUX || + op == MLX4_CMD_UNMAP_FA) + return 1; + /* Error on MLX4_CMD_HW2SW_MPT is fatal except when fw status equals + * CMD_STAT_REG_BOUND. + * This status indicates that memory region has memory windows bound to it + * which may result from invalid user space usage and is not fatal. + */ + if (op == MLX4_CMD_HW2SW_MPT && fw_status != CMD_STAT_REG_BOUND) + return 1; + return 0; +} + +static int mlx4_cmd_reset_flow(struct mlx4_dev *dev, u16 op, u8 op_modifier, + int err) +{ + /* Only if reset flow is really active return code is based on + * command, otherwise current error code is returned. + */ + if (mlx4_internal_err_reset) { + mlx4_enter_error_state(dev->persist); + err = mlx4_internal_err_ret_value(dev, op, op_modifier); + } + + return err; +} + +static int comm_pending(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + u32 status = readl(&priv->mfunc.comm->slave_read); + + return (swab32(status) >> 31) != priv->cmd.comm_toggle; +} + +static int mlx4_comm_cmd_post(struct mlx4_dev *dev, u8 cmd, u16 param) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + u32 val; + + /* To avoid writing to unknown addresses after the device state was + * changed to internal error and the function was rest, + * check the INTERNAL_ERROR flag which is updated under + * device_state_mutex lock. + */ + mutex_lock(&dev->persist->device_state_mutex); + + if (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) { + mutex_unlock(&dev->persist->device_state_mutex); + return -EIO; + } + + priv->cmd.comm_toggle ^= 1; + val = param | (cmd << 16) | (priv->cmd.comm_toggle << 31); + __raw_writel((__force u32) cpu_to_be32(val), + &priv->mfunc.comm->slave_write); + mmiowb(); + mutex_unlock(&dev->persist->device_state_mutex); + return 0; +} + +static int mlx4_comm_cmd_poll(struct mlx4_dev *dev, u8 cmd, u16 param, + unsigned long timeout) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + unsigned long end; + int err = 0; + int ret_from_pending = 0; + + /* First, verify that the master reports correct status */ + if (comm_pending(dev)) { + mlx4_warn(dev, "Communication channel is not idle - my toggle is %d (cmd:0x%x)\n", + priv->cmd.comm_toggle, cmd); + return -EAGAIN; + } + + /* Write command */ + down(&priv->cmd.poll_sem); + if (mlx4_comm_cmd_post(dev, cmd, param)) { + /* Only in case the device state is INTERNAL_ERROR, + * mlx4_comm_cmd_post returns with an error + */ + err = mlx4_status_to_errno(CMD_STAT_INTERNAL_ERR); + goto out; + } + + end = msecs_to_jiffies(timeout) + jiffies; + while (comm_pending(dev) && time_before(jiffies, end)) + cond_resched(); + ret_from_pending = comm_pending(dev); + if (ret_from_pending) { + /* check if the slave is trying to boot in the middle of + * FLR process. The only non-zero result in the RESET command + * is MLX4_DELAY_RESET_SLAVE*/ + if ((MLX4_COMM_CMD_RESET == cmd)) { + err = MLX4_DELAY_RESET_SLAVE; + goto out; + } else { + mlx4_warn(dev, "Communication channel command 0x%x timed out\n", + cmd); + err = mlx4_status_to_errno(CMD_STAT_INTERNAL_ERR); + } + } + + if (err) + mlx4_enter_error_state(dev->persist); +out: + up(&priv->cmd.poll_sem); + return err; +} + +static int mlx4_comm_cmd_wait(struct mlx4_dev *dev, u8 vhcr_cmd, + u16 param, u16 op, unsigned long timeout) +{ + struct mlx4_cmd *cmd = &mlx4_priv(dev)->cmd; + struct mlx4_cmd_context *context; + unsigned long end; + int err = 0; + + down(&cmd->event_sem); + + spin_lock(&cmd->context_lock); + BUG_ON(cmd->free_head < 0); + context = &cmd->context[cmd->free_head]; + context->token += cmd->token_mask + 1; + cmd->free_head = context->next; + spin_unlock(&cmd->context_lock); + + reinit_completion(&context->done); + + if (mlx4_comm_cmd_post(dev, vhcr_cmd, param)) { + /* Only in case the device state is INTERNAL_ERROR, + * mlx4_comm_cmd_post returns with an error + */ + err = mlx4_status_to_errno(CMD_STAT_INTERNAL_ERR); + goto out; + } + + if (!wait_for_completion_timeout(&context->done, + msecs_to_jiffies(timeout))) { + mlx4_warn(dev, "communication channel command 0x%x (op=0x%x) timed out\n", + vhcr_cmd, op); + goto out_reset; + } + + err = context->result; + if (err && context->fw_status != CMD_STAT_MULTI_FUNC_REQ) { + mlx4_err(dev, "command 0x%x failed: fw status = 0x%x\n", + vhcr_cmd, context->fw_status); + if (mlx4_closing_cmd_fatal_error(op, context->fw_status)) + goto out_reset; + } + + /* wait for comm channel ready + * this is necessary for prevention the race + * when switching between event to polling mode + * Skipping this section in case the device is in FATAL_ERROR state, + * In this state, no commands are sent via the comm channel until + * the device has returned from reset. + */ + if (!(dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR)) { + end = msecs_to_jiffies(timeout) + jiffies; + while (comm_pending(dev) && time_before(jiffies, end)) + cond_resched(); + } + goto out; + +out_reset: + err = mlx4_status_to_errno(CMD_STAT_INTERNAL_ERR); + mlx4_enter_error_state(dev->persist); +out: + spin_lock(&cmd->context_lock); + context->next = cmd->free_head; + cmd->free_head = context - cmd->context; + spin_unlock(&cmd->context_lock); + + up(&cmd->event_sem); + return err; +} + +int mlx4_comm_cmd(struct mlx4_dev *dev, u8 cmd, u16 param, + u16 op, unsigned long timeout) +{ + if (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) + return mlx4_status_to_errno(CMD_STAT_INTERNAL_ERR); + + if (mlx4_priv(dev)->cmd.use_events) + return mlx4_comm_cmd_wait(dev, cmd, param, op, timeout); + return mlx4_comm_cmd_poll(dev, cmd, param, timeout); +} + +static int cmd_pending(struct mlx4_dev *dev) +{ + u32 status; + + if (pci_channel_offline(dev->persist->pdev)) + return -EIO; + + status = readl(mlx4_priv(dev)->cmd.hcr + HCR_STATUS_OFFSET); + + return (status & swab32(1 << HCR_GO_BIT)) || + (mlx4_priv(dev)->cmd.toggle == + !!(status & swab32(1 << HCR_T_BIT))); +} + +static int mlx4_cmd_post(struct mlx4_dev *dev, u64 in_param, u64 out_param, + u32 in_modifier, u8 op_modifier, u16 op, u16 token, + int event) +{ + struct mlx4_cmd *cmd = &mlx4_priv(dev)->cmd; + u32 __iomem *hcr = cmd->hcr; + int ret = -EIO; + unsigned long end; + + mutex_lock(&dev->persist->device_state_mutex); + /* To avoid writing to unknown addresses after the device state was + * changed to internal error and the chip was reset, + * check the INTERNAL_ERROR flag which is updated under + * device_state_mutex lock. + */ + if (pci_channel_offline(dev->persist->pdev) || + (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR)) { + /* + * Device is going through error recovery + * and cannot accept commands. + */ + goto out; + } + + end = jiffies; + if (event) + end += msecs_to_jiffies(GO_BIT_TIMEOUT_MSECS); + + while (cmd_pending(dev)) { + if (pci_channel_offline(dev->persist->pdev)) { + /* + * Device is going through error recovery + * and cannot accept commands. + */ + goto out; + } + + if (time_after_eq(jiffies, end)) { + mlx4_err(dev, "%s:cmd_pending failed\n", __func__); + goto out; + } + cond_resched(); + } + + /* + * We use writel (instead of something like memcpy_toio) + * because writes of less than 32 bits to the HCR don't work + * (and some architectures such as ia64 implement memcpy_toio + * in terms of writeb). + */ + __raw_writel((__force u32) cpu_to_be32(in_param >> 32), hcr + 0); + __raw_writel((__force u32) cpu_to_be32(in_param & 0xfffffffful), hcr + 1); + __raw_writel((__force u32) cpu_to_be32(in_modifier), hcr + 2); + __raw_writel((__force u32) cpu_to_be32(out_param >> 32), hcr + 3); + __raw_writel((__force u32) cpu_to_be32(out_param & 0xfffffffful), hcr + 4); + __raw_writel((__force u32) cpu_to_be32(token << 16), hcr + 5); + + /* __raw_writel may not order writes. */ + wmb(); + + __raw_writel((__force u32) cpu_to_be32((1 << HCR_GO_BIT) | + (cmd->toggle << HCR_T_BIT) | + (event ? (1 << HCR_E_BIT) : 0) | + (op_modifier << HCR_OPMOD_SHIFT) | + op), hcr + 6); + + /* + * Make sure that our HCR writes don't get mixed in with + * writes from another CPU starting a FW command. + */ + mmiowb(); + + cmd->toggle = cmd->toggle ^ 1; + + ret = 0; + +out: + if (ret) + mlx4_warn(dev, "Could not post command 0x%x: ret=%d, in_param=0x%llx, in_mod=0x%x, op_mod=0x%x\n", + op, ret, (long long)in_param, in_modifier, op_modifier); + mutex_unlock(&dev->persist->device_state_mutex); + + return ret; +} + +static int mlx4_slave_cmd(struct mlx4_dev *dev, u64 in_param, u64 *out_param, + int out_is_imm, u32 in_modifier, u8 op_modifier, + u16 op, unsigned long timeout) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_vhcr_cmd *vhcr = priv->mfunc.vhcr; + int ret; + + mutex_lock(&priv->cmd.slave_cmd_mutex); + + vhcr->in_param = cpu_to_be64(in_param); + vhcr->out_param = out_param ? cpu_to_be64(*out_param) : 0; + vhcr->in_modifier = cpu_to_be32(in_modifier); + vhcr->opcode = cpu_to_be16((((u16) op_modifier) << 12) | (op & 0xfff)); + vhcr->token = cpu_to_be16(CMD_POLL_TOKEN); + vhcr->status = 0; + vhcr->flags = !!(priv->cmd.use_events) << 6; + + if (mlx4_is_master(dev)) { + ret = mlx4_master_process_vhcr(dev, dev->caps.function, vhcr); + if (!ret) { + if (out_is_imm) { + if (out_param) + *out_param = + be64_to_cpu(vhcr->out_param); + else { + mlx4_err(dev, "response expected while output mailbox is NULL for command 0x%x\n", + op); + vhcr->status = CMD_STAT_BAD_PARAM; + } + } + ret = mlx4_status_to_errno(vhcr->status); + } + if (ret && + dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) + ret = mlx4_internal_err_ret_value(dev, op, op_modifier); + } else { + ret = mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR_POST, 0, op, + MLX4_COMM_TIME + timeout); + if (!ret) { + if (out_is_imm) { + if (out_param) + *out_param = + be64_to_cpu(vhcr->out_param); + else { + mlx4_err(dev, "response expected while output mailbox is NULL for command 0x%x\n", + op); + vhcr->status = CMD_STAT_BAD_PARAM; + } + } + ret = mlx4_status_to_errno(vhcr->status); + } else { + if (dev->persist->state & + MLX4_DEVICE_STATE_INTERNAL_ERROR) + ret = mlx4_internal_err_ret_value(dev, op, + op_modifier); + else + mlx4_err(dev, "failed execution of VHCR_POST command opcode 0x%x\n", op); + } + } + + mutex_unlock(&priv->cmd.slave_cmd_mutex); + return ret; +} + +static int mlx4_cmd_poll(struct mlx4_dev *dev, u64 in_param, u64 *out_param, + int out_is_imm, u32 in_modifier, u8 op_modifier, + u16 op, unsigned long timeout) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + void __iomem *hcr = priv->cmd.hcr; + int err = 0; + unsigned long end; + u32 stat; + + down(&priv->cmd.poll_sem); + + if (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) { + /* + * Device is going through error recovery + * and cannot accept commands. + */ + err = mlx4_internal_err_ret_value(dev, op, op_modifier); + goto out; + } + + if (out_is_imm && !out_param) { + mlx4_err(dev, "response expected while output mailbox is NULL for command 0x%x\n", + op); + err = -EINVAL; + goto out; + } + + err = mlx4_cmd_post(dev, in_param, out_param ? *out_param : 0, + in_modifier, op_modifier, op, CMD_POLL_TOKEN, 0); + if (err) + goto out_reset; + + end = msecs_to_jiffies(timeout) + jiffies; + while (cmd_pending(dev) && time_before(jiffies, end)) { + if (pci_channel_offline(dev->persist->pdev)) { + /* + * Device is going through error recovery + * and cannot accept commands. + */ + err = -EIO; + goto out_reset; + } + + if (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) { + err = mlx4_internal_err_ret_value(dev, op, op_modifier); + goto out; + } + + cond_resched(); + } + + if (cmd_pending(dev)) { + mlx4_warn(dev, "command 0x%x timed out (go bit not cleared)\n", + op); + err = -EIO; + goto out_reset; + } + + if (out_is_imm) + *out_param = + (u64) be32_to_cpu((__force __be32) + __raw_readl(hcr + HCR_OUT_PARAM_OFFSET)) << 32 | + (u64) be32_to_cpu((__force __be32) + __raw_readl(hcr + HCR_OUT_PARAM_OFFSET + 4)); + stat = be32_to_cpu((__force __be32) + __raw_readl(hcr + HCR_STATUS_OFFSET)) >> 24; + err = mlx4_status_to_errno(stat); + if (err) { + mlx4_err(dev, "command 0x%x failed: fw status = 0x%x\n", + op, stat); + if (mlx4_closing_cmd_fatal_error(op, stat)) + goto out_reset; + goto out; + } + +out_reset: + if (err) + err = mlx4_cmd_reset_flow(dev, op, op_modifier, err); +out: + up(&priv->cmd.poll_sem); + return err; +} + +void mlx4_cmd_event(struct mlx4_dev *dev, u16 token, u8 status, u64 out_param) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_cmd_context *context = + &priv->cmd.context[token & priv->cmd.token_mask]; + + /* previously timed out command completing at long last */ + if (token != context->token) + return; + + context->fw_status = status; + context->result = mlx4_status_to_errno(status); + context->out_param = out_param; + + complete(&context->done); +} + +static int mlx4_cmd_wait(struct mlx4_dev *dev, u64 in_param, u64 *out_param, + int out_is_imm, u32 in_modifier, u8 op_modifier, + u16 op, unsigned long timeout) +{ + struct mlx4_cmd *cmd = &mlx4_priv(dev)->cmd; + struct mlx4_cmd_context *context; + long ret_wait; + int err = 0; + + down(&cmd->event_sem); + + spin_lock(&cmd->context_lock); + BUG_ON(cmd->free_head < 0); + context = &cmd->context[cmd->free_head]; + context->token += cmd->token_mask + 1; + cmd->free_head = context->next; + spin_unlock(&cmd->context_lock); + + if (out_is_imm && !out_param) { + mlx4_err(dev, "response expected while output mailbox is NULL for command 0x%x\n", + op); + err = -EINVAL; + goto out; + } + + reinit_completion(&context->done); + + err = mlx4_cmd_post(dev, in_param, out_param ? *out_param : 0, + in_modifier, op_modifier, op, context->token, 1); + if (err) + goto out_reset; + + if (op == MLX4_CMD_SENSE_PORT) { + ret_wait = + wait_for_completion_interruptible_timeout(&context->done, + msecs_to_jiffies(timeout)); + if (ret_wait < 0) { + context->fw_status = 0; + context->out_param = 0; + context->result = 0; + } + } else { + ret_wait = (long)wait_for_completion_timeout(&context->done, + msecs_to_jiffies(timeout)); + } + if (!ret_wait) { + mlx4_warn(dev, "command 0x%x timed out (go bit not cleared)\n", + op); + if (op == MLX4_CMD_NOP) { + err = -EBUSY; + goto out; + } else { + err = -EIO; + goto out_reset; + } + } + + err = context->result; + if (err) { + /* Since we do not want to have this error message always + * displayed at driver start when there are ConnectX2 HCAs + * on the host, we deprecate the error message for this + * specific command/input_mod/opcode_mod/fw-status to be debug. + */ + if (op == MLX4_CMD_SET_PORT && + (in_modifier == 1 || in_modifier == 2) && + op_modifier == MLX4_SET_PORT_IB_OPCODE && + context->fw_status == CMD_STAT_BAD_SIZE) + mlx4_dbg(dev, "command 0x%x failed: fw status = 0x%x\n", + op, context->fw_status); + else + mlx4_err(dev, "command 0x%x failed: fw status = 0x%x\n", + op, context->fw_status); + if (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) + err = mlx4_internal_err_ret_value(dev, op, op_modifier); + else if (mlx4_closing_cmd_fatal_error(op, context->fw_status)) + goto out_reset; + + goto out; + } + + if (out_is_imm) + *out_param = context->out_param; + +out_reset: + if (err) + err = mlx4_cmd_reset_flow(dev, op, op_modifier, err); +out: + spin_lock(&cmd->context_lock); + context->next = cmd->free_head; + cmd->free_head = context - cmd->context; + spin_unlock(&cmd->context_lock); + + up(&cmd->event_sem); + return err; +} + +int __mlx4_cmd(struct mlx4_dev *dev, u64 in_param, u64 *out_param, + int out_is_imm, u32 in_modifier, u8 op_modifier, + u16 op, unsigned long timeout, int native) +{ + if (pci_channel_offline(dev->persist->pdev)) + return mlx4_cmd_reset_flow(dev, op, op_modifier, -EIO); + + if (!mlx4_is_mfunc(dev) || (native && mlx4_is_master(dev))) { + int ret; + + if (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) + return mlx4_internal_err_ret_value(dev, op, + op_modifier); + down_read(&mlx4_priv(dev)->cmd.switch_sem); + if (mlx4_priv(dev)->cmd.use_events) + ret = mlx4_cmd_wait(dev, in_param, out_param, + out_is_imm, in_modifier, + op_modifier, op, timeout); + else + ret = mlx4_cmd_poll(dev, in_param, out_param, + out_is_imm, in_modifier, + op_modifier, op, timeout); + + up_read(&mlx4_priv(dev)->cmd.switch_sem); + return ret; + } + return mlx4_slave_cmd(dev, in_param, out_param, out_is_imm, + in_modifier, op_modifier, op, timeout); +} +EXPORT_SYMBOL_GPL(__mlx4_cmd); + + +int mlx4_ARM_COMM_CHANNEL(struct mlx4_dev *dev) +{ + return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_ARM_COMM_CHANNEL, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); +} + +static int mlx4_ACCESS_MEM(struct mlx4_dev *dev, u64 master_addr, + int slave, u64 slave_addr, + int size, int is_read) +{ + u64 in_param; + u64 out_param; + + if ((slave_addr & 0xfff) | (master_addr & 0xfff) | + (slave & ~0x7f) | (size & 0xff)) { + mlx4_err(dev, "Bad access mem params - slave_addr:0x%llx " + "master_addr:0x%llx slave_id:%d size:%d\n", + (unsigned long long)slave_addr, + (unsigned long long)master_addr, slave, size); + return -EINVAL; + } + + if (is_read) { + in_param = (u64) slave | slave_addr; + out_param = (u64) dev->caps.function | master_addr; + } else { + in_param = (u64) dev->caps.function | master_addr; + out_param = (u64) slave | slave_addr; + } + + return mlx4_cmd_imm(dev, in_param, &out_param, size, 0, + MLX4_CMD_ACCESS_MEM, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); +} + +static int query_pkey_block(struct mlx4_dev *dev, u8 port, u16 index, u16 *pkey, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox) +{ + struct ib_smp *in_mad = (struct ib_smp *)(inbox->buf); + struct ib_smp *out_mad = (struct ib_smp *)(outbox->buf); + int err; + int i; + + if (index & 0x1f) + return -EINVAL; + + in_mad->attr_mod = cpu_to_be32(index / 32); + + err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, port, 3, + MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C, + MLX4_CMD_NATIVE); + if (err) + return err; + + for (i = 0; i < 32; ++i) + pkey[i] = be16_to_cpu(((__be16 *) out_mad->data)[i]); + + return err; +} + +static int get_full_pkey_table(struct mlx4_dev *dev, u8 port, u16 *table, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox) +{ + int i; + int err; + + for (i = 0; i < dev->caps.pkey_table_len[port]; i += 32) { + err = query_pkey_block(dev, port, i, table + i, inbox, outbox); + if (err) + return err; + } + + return 0; +} +#define PORT_CAPABILITY_LOCATION_IN_SMP 20 +#define PORT_STATE_OFFSET 32 + +static enum ib_port_state vf_port_state(struct mlx4_dev *dev, int port, int vf) +{ + if (mlx4_get_slave_port_state(dev, vf, port) == SLAVE_PORT_UP) + return IB_PORT_ACTIVE; + else + return IB_PORT_DOWN; +} + +static int mlx4_MAD_IFC_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct ib_smp *smp = inbox->buf; + u32 index; + u8 port, slave_port; + u8 opcode_modifier; + u16 *table; + int err; + int vidx, pidx; + int network_view; + struct mlx4_priv *priv = mlx4_priv(dev); + struct ib_smp *outsmp = outbox->buf; + __be16 *outtab = (__be16 *)(outsmp->data); + __be32 slave_cap_mask; + __be64 slave_node_guid; + + slave_port = vhcr->in_modifier; + port = mlx4_slave_convert_port(dev, slave, slave_port); + + /* network-view bit is for driver use only, and should not be passed to FW */ + opcode_modifier = vhcr->op_modifier & ~0x8; /* clear netw view bit */ + network_view = !!(vhcr->op_modifier & 0x8); + + if (smp->base_version == 1 && + smp->mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED && + smp->class_version == 1) { + /* host view is paravirtualized */ + if (!network_view && smp->method == IB_MGMT_METHOD_GET) { + if (smp->attr_id == IB_SMP_ATTR_PKEY_TABLE) { + index = be32_to_cpu(smp->attr_mod); + if (port < 1 || port > dev->caps.num_ports) + return -EINVAL; + table = kcalloc((dev->caps.pkey_table_len[port] / 32) + 1, + sizeof(*table) * 32, GFP_KERNEL); + + if (!table) + return -ENOMEM; + /* need to get the full pkey table because the paravirtualized + * pkeys may be scattered among several pkey blocks. + */ + err = get_full_pkey_table(dev, port, table, inbox, outbox); + if (!err) { + for (vidx = index * 32; vidx < (index + 1) * 32; ++vidx) { + pidx = priv->virt2phys_pkey[slave][port - 1][vidx]; + outtab[vidx % 32] = cpu_to_be16(table[pidx]); + } + } + kfree(table); + return err; + } + if (smp->attr_id == IB_SMP_ATTR_PORT_INFO) { + /*get the slave specific caps:*/ + /*do the command */ + smp->attr_mod = cpu_to_be32(port); + err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, + port, opcode_modifier, + vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); + /* modify the response for slaves */ + if (!err && slave != mlx4_master_func_num(dev)) { + u8 *state = outsmp->data + PORT_STATE_OFFSET; + + *state = (*state & 0xf0) | vf_port_state(dev, port, slave); + slave_cap_mask = priv->mfunc.master.slave_state[slave].ib_cap_mask[port]; + memcpy(outsmp->data + PORT_CAPABILITY_LOCATION_IN_SMP, &slave_cap_mask, 4); + } + return err; + } + if (smp->attr_id == IB_SMP_ATTR_GUID_INFO) { + __be64 guid = mlx4_get_admin_guid(dev, slave, + port); + + /* set the PF admin guid to the FW/HW burned + * GUID, if it wasn't yet set + */ + if (slave == 0 && guid == 0) { + smp->attr_mod = 0; + err = mlx4_cmd_box(dev, + inbox->dma, + outbox->dma, + vhcr->in_modifier, + opcode_modifier, + vhcr->op, + MLX4_CMD_TIME_CLASS_C, + MLX4_CMD_NATIVE); + if (err) + return err; + mlx4_set_admin_guid(dev, + *(__be64 *)outsmp-> + data, slave, port); + } else { + memcpy(outsmp->data, &guid, 8); + } + + /* clean all other gids */ + memset(outsmp->data + 8, 0, 56); + return 0; + } + if (smp->attr_id == IB_SMP_ATTR_NODE_INFO) { + err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, + port, opcode_modifier, + vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); + if (!err) { + slave_node_guid = mlx4_get_slave_node_guid(dev, slave); + memcpy(outsmp->data + 12, &slave_node_guid, 8); + } + return err; + } + } + } + + /* Non-privileged VFs are only allowed "host" view LID-routed 'Get' MADs. + * These are the MADs used by ib verbs (such as ib_query_gids). + */ + if (slave != mlx4_master_func_num(dev) && + !mlx4_vf_smi_enabled(dev, slave, port)) { + if (!(smp->mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED && + smp->method == IB_MGMT_METHOD_GET) || network_view) { + mlx4_err(dev, "Unprivileged slave %d is trying to execute a Subnet MGMT MAD, class 0x%x, method 0x%x, view=%s for attr 0x%x. Rejecting\n", + slave, smp->mgmt_class, smp->method, + network_view ? "Network" : "Host", + be16_to_cpu(smp->attr_id)); + return -EPERM; + } + } + + return mlx4_cmd_box(dev, inbox->dma, outbox->dma, + vhcr->in_modifier, opcode_modifier, + vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); +} + +static int mlx4_CMD_EPERM_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + return -EPERM; +} + +int mlx4_DMA_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + u64 in_param; + u64 out_param; + int err; + + in_param = cmd->has_inbox ? (u64) inbox->dma : vhcr->in_param; + out_param = cmd->has_outbox ? (u64) outbox->dma : vhcr->out_param; + if (cmd->encode_slave_id) { + in_param &= 0xffffffffffffff00ll; + in_param |= slave; + } + + err = __mlx4_cmd(dev, in_param, &out_param, cmd->out_is_imm, + vhcr->in_modifier, vhcr->op_modifier, vhcr->op, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + + if (cmd->out_is_imm) + vhcr->out_param = out_param; + + return err; +} + +static struct mlx4_cmd_info cmd_info[] = { + { + .opcode = MLX4_CMD_QUERY_FW, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QUERY_FW_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_HCA, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = NULL + }, + { + .opcode = MLX4_CMD_QUERY_DEV_CAP, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QUERY_DEV_CAP_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_FUNC_CAP, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QUERY_FUNC_CAP_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_ADAPTER, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = NULL + }, + { + .opcode = MLX4_CMD_INIT_PORT, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_INIT_PORT_wrapper + }, + { + .opcode = MLX4_CMD_CLOSE_PORT, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_CLOSE_PORT_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_PORT, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QUERY_PORT_wrapper + }, + { + .opcode = MLX4_CMD_SET_PORT, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_SET_PORT_wrapper + }, + { + .opcode = MLX4_CMD_MAP_EQ, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_MAP_EQ_wrapper + }, + { + .opcode = MLX4_CMD_SW2HW_EQ, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = true, + .verify = NULL, + .wrapper = mlx4_SW2HW_EQ_wrapper + }, + { + .opcode = MLX4_CMD_HW_HEALTH_CHECK, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = NULL + }, + { + .opcode = MLX4_CMD_NOP, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = NULL + }, + { + .opcode = MLX4_CMD_CONFIG_DEV, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_CONFIG_DEV_wrapper + }, + { + .opcode = MLX4_CMD_ALLOC_RES, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = true, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_ALLOC_RES_wrapper + }, + { + .opcode = MLX4_CMD_FREE_RES, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_FREE_RES_wrapper + }, + { + .opcode = MLX4_CMD_SW2HW_MPT, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = true, + .verify = NULL, + .wrapper = mlx4_SW2HW_MPT_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_MPT, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QUERY_MPT_wrapper + }, + { + .opcode = MLX4_CMD_HW2SW_MPT, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_HW2SW_MPT_wrapper + }, + { + .opcode = MLX4_CMD_READ_MTT, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = NULL + }, + { + .opcode = MLX4_CMD_WRITE_MTT, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_WRITE_MTT_wrapper + }, + { + .opcode = MLX4_CMD_SYNC_TPT, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = NULL + }, + { + .opcode = MLX4_CMD_HW2SW_EQ, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = true, + .verify = NULL, + .wrapper = mlx4_HW2SW_EQ_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_EQ, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = true, + .verify = NULL, + .wrapper = mlx4_QUERY_EQ_wrapper + }, + { + .opcode = MLX4_CMD_SW2HW_CQ, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = true, + .verify = NULL, + .wrapper = mlx4_SW2HW_CQ_wrapper + }, + { + .opcode = MLX4_CMD_HW2SW_CQ, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_HW2SW_CQ_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_CQ, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QUERY_CQ_wrapper + }, + { + .opcode = MLX4_CMD_MODIFY_CQ, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = true, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_MODIFY_CQ_wrapper + }, + { + .opcode = MLX4_CMD_SW2HW_SRQ, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = true, + .verify = NULL, + .wrapper = mlx4_SW2HW_SRQ_wrapper + }, + { + .opcode = MLX4_CMD_HW2SW_SRQ, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_HW2SW_SRQ_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_SRQ, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QUERY_SRQ_wrapper + }, + { + .opcode = MLX4_CMD_ARM_SRQ, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_ARM_SRQ_wrapper + }, + { + .opcode = MLX4_CMD_RST2INIT_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = true, + .verify = NULL, + .wrapper = mlx4_RST2INIT_QP_wrapper + }, + { + .opcode = MLX4_CMD_INIT2INIT_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_INIT2INIT_QP_wrapper + }, + { + .opcode = MLX4_CMD_INIT2RTR_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_INIT2RTR_QP_wrapper + }, + { + .opcode = MLX4_CMD_RTR2RTS_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_RTR2RTS_QP_wrapper + }, + { + .opcode = MLX4_CMD_RTS2RTS_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_RTS2RTS_QP_wrapper + }, + { + .opcode = MLX4_CMD_SQERR2RTS_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_SQERR2RTS_QP_wrapper + }, + { + .opcode = MLX4_CMD_2ERR_QP, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_GEN_QP_wrapper + }, + { + .opcode = MLX4_CMD_RTS2SQD_QP, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_GEN_QP_wrapper + }, + { + .opcode = MLX4_CMD_SQD2SQD_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_SQD2SQD_QP_wrapper + }, + { + .opcode = MLX4_CMD_SQD2RTS_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_SQD2RTS_QP_wrapper + }, + { + .opcode = MLX4_CMD_2RST_QP, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_2RST_QP_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_QP, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_GEN_QP_wrapper + }, + { + .opcode = MLX4_CMD_SUSPEND_QP, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_GEN_QP_wrapper + }, + { + .opcode = MLX4_CMD_UNSUSPEND_QP, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_GEN_QP_wrapper + }, + { + .opcode = MLX4_CMD_UPDATE_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_UPDATE_QP_wrapper + }, + { + .opcode = MLX4_CMD_GET_OP_REQ, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_CMD_EPERM_wrapper, + }, + { + .opcode = MLX4_CMD_ALLOCATE_VPP, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_CMD_EPERM_wrapper, + }, + { + .opcode = MLX4_CMD_SET_VPORT_QOS, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_CMD_EPERM_wrapper, + }, + { + .opcode = MLX4_CMD_CONF_SPECIAL_QP, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, /* XXX verify: only demux can do this */ + .wrapper = NULL + }, + { + .opcode = MLX4_CMD_MAD_IFC, + .has_inbox = true, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_MAD_IFC_wrapper + }, + { + .opcode = MLX4_CMD_MAD_DEMUX, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_CMD_EPERM_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_IF_STAT, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QUERY_IF_STAT_wrapper + }, + { + .opcode = MLX4_CMD_ACCESS_REG, + .has_inbox = true, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_ACCESS_REG_wrapper, + }, + { + .opcode = MLX4_CMD_CONGESTION_CTRL_OPCODE, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_CMD_EPERM_wrapper, + }, + /* Native multicast commands are not available for guests */ + { + .opcode = MLX4_CMD_QP_ATTACH, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QP_ATTACH_wrapper + }, + { + .opcode = MLX4_CMD_PROMISC, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_PROMISC_wrapper + }, + /* Ethernet specific commands */ + { + .opcode = MLX4_CMD_SET_VLAN_FLTR, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_SET_VLAN_FLTR_wrapper + }, + { + .opcode = MLX4_CMD_SET_MCAST_FLTR, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_SET_MCAST_FLTR_wrapper + }, + { + .opcode = MLX4_CMD_DUMP_ETH_STATS, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_DUMP_ETH_STATS_wrapper + }, + { + .opcode = MLX4_CMD_INFORM_FLR_DONE, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = NULL + }, + /* flow steering commands */ + { + .opcode = MLX4_QP_FLOW_STEERING_ATTACH, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = true, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QP_FLOW_STEERING_ATTACH_wrapper + }, + { + .opcode = MLX4_QP_FLOW_STEERING_DETACH, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QP_FLOW_STEERING_DETACH_wrapper + }, + { + .opcode = MLX4_FLOW_STEERING_IB_UC_QP_RANGE, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_CMD_EPERM_wrapper + }, + { + .opcode = MLX4_CMD_VIRT_PORT_MAP, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_CMD_EPERM_wrapper + }, +}; + +static int mlx4_master_process_vhcr(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr_cmd *in_vhcr) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_cmd_info *cmd = NULL; + struct mlx4_vhcr_cmd *vhcr_cmd = in_vhcr ? in_vhcr : priv->mfunc.vhcr; + struct mlx4_vhcr *vhcr; + struct mlx4_cmd_mailbox *inbox = NULL; + struct mlx4_cmd_mailbox *outbox = NULL; + u64 in_param; + u64 out_param; + int ret = 0; + int i; + int err = 0; + + /* Create sw representation of Virtual HCR */ + vhcr = kzalloc(sizeof(struct mlx4_vhcr), GFP_KERNEL); + if (!vhcr) + return -ENOMEM; + + /* DMA in the vHCR */ + if (!in_vhcr) { + ret = mlx4_ACCESS_MEM(dev, priv->mfunc.vhcr_dma, slave, + priv->mfunc.master.slave_state[slave].vhcr_dma, + ALIGN(sizeof(struct mlx4_vhcr_cmd), + MLX4_ACCESS_MEM_ALIGN), 1); + if (ret) { + if (!(dev->persist->state & + MLX4_DEVICE_STATE_INTERNAL_ERROR)) + mlx4_err(dev, "%s: Failed reading vhcr ret: 0x%x\n", + __func__, ret); + kfree(vhcr); + return ret; + } + } + + /* Fill SW VHCR fields */ + vhcr->in_param = be64_to_cpu(vhcr_cmd->in_param); + vhcr->out_param = be64_to_cpu(vhcr_cmd->out_param); + vhcr->in_modifier = be32_to_cpu(vhcr_cmd->in_modifier); + vhcr->token = be16_to_cpu(vhcr_cmd->token); + vhcr->op = be16_to_cpu(vhcr_cmd->opcode) & 0xfff; + vhcr->op_modifier = (u8) (be16_to_cpu(vhcr_cmd->opcode) >> 12); + vhcr->e_bit = vhcr_cmd->flags & (1 << 6); + + /* Lookup command */ + for (i = 0; i < ARRAY_SIZE(cmd_info); ++i) { + if (vhcr->op == cmd_info[i].opcode) { + cmd = &cmd_info[i]; + break; + } + } + if (!cmd) { + mlx4_err(dev, "Unknown command:0x%x accepted from slave:%d\n", + vhcr->op, slave); + vhcr_cmd->status = CMD_STAT_BAD_PARAM; + goto out_status; + } + + /* Read inbox */ + if (cmd->has_inbox) { + vhcr->in_param &= INBOX_MASK; + inbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(inbox)) { + vhcr_cmd->status = CMD_STAT_BAD_SIZE; + inbox = NULL; + goto out_status; + } + + ret = mlx4_ACCESS_MEM(dev, inbox->dma, slave, + vhcr->in_param, + MLX4_MAILBOX_SIZE, 1); + if (ret) { + if (!(dev->persist->state & + MLX4_DEVICE_STATE_INTERNAL_ERROR)) + mlx4_err(dev, "%s: Failed reading inbox (cmd:0x%x)\n", + __func__, cmd->opcode); + vhcr_cmd->status = CMD_STAT_INTERNAL_ERR; + goto out_status; + } + } + + /* Apply permission and bound checks if applicable */ + if (cmd->verify && cmd->verify(dev, slave, vhcr, inbox)) { + mlx4_warn(dev, "Command:0x%x from slave: %d failed protection checks for resource_id:%d\n", + vhcr->op, slave, vhcr->in_modifier); + vhcr_cmd->status = CMD_STAT_BAD_OP; + goto out_status; + } + + /* Allocate outbox */ + if (cmd->has_outbox) { + outbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(outbox)) { + vhcr_cmd->status = CMD_STAT_BAD_SIZE; + outbox = NULL; + goto out_status; + } + } + + /* Execute the command! */ + if (cmd->wrapper) { + err = cmd->wrapper(dev, slave, vhcr, inbox, outbox, + cmd); + if (cmd->out_is_imm) + vhcr_cmd->out_param = cpu_to_be64(vhcr->out_param); + } else { + in_param = cmd->has_inbox ? (u64) inbox->dma : + vhcr->in_param; + out_param = cmd->has_outbox ? (u64) outbox->dma : + vhcr->out_param; + err = __mlx4_cmd(dev, in_param, &out_param, + cmd->out_is_imm, vhcr->in_modifier, + vhcr->op_modifier, vhcr->op, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + + if (cmd->out_is_imm) { + vhcr->out_param = out_param; + vhcr_cmd->out_param = cpu_to_be64(vhcr->out_param); + } + } + + if (err) { + if (!(dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR)) + mlx4_warn(dev, "vhcr command:0x%x slave:%d failed with error:%d, status %d\n", + vhcr->op, slave, vhcr->errno, err); + vhcr_cmd->status = mlx4_errno_to_status(err); + goto out_status; + } + + + /* Write outbox if command completed successfully */ + if (cmd->has_outbox && !vhcr_cmd->status) { + ret = mlx4_ACCESS_MEM(dev, outbox->dma, slave, + vhcr->out_param, + MLX4_MAILBOX_SIZE, MLX4_CMD_WRAPPED); + if (ret) { + /* If we failed to write back the outbox after the + *command was successfully executed, we must fail this + * slave, as it is now in undefined state */ + if (!(dev->persist->state & + MLX4_DEVICE_STATE_INTERNAL_ERROR)) + mlx4_err(dev, "%s:Failed writing outbox\n", __func__); + goto out; + } + } + +out_status: + /* DMA back vhcr result */ + if (!in_vhcr) { + ret = mlx4_ACCESS_MEM(dev, priv->mfunc.vhcr_dma, slave, + priv->mfunc.master.slave_state[slave].vhcr_dma, + ALIGN(sizeof(struct mlx4_vhcr), + MLX4_ACCESS_MEM_ALIGN), + MLX4_CMD_WRAPPED); + if (ret) + mlx4_err(dev, "%s:Failed writing vhcr result\n", + __func__); + else if (vhcr->e_bit && + mlx4_GEN_EQE(dev, slave, &priv->mfunc.master.cmd_eqe)) + mlx4_warn(dev, "Failed to generate command completion eqe for slave %d\n", + slave); + } + +out: + kfree(vhcr); + mlx4_free_cmd_mailbox(dev, inbox); + mlx4_free_cmd_mailbox(dev, outbox); + return ret; +} + +static int mlx4_master_immediate_activate_vlan_qos(struct mlx4_priv *priv, + int slave, int port) +{ + struct mlx4_vport_oper_state *vp_oper; + struct mlx4_vport_state *vp_admin; + struct mlx4_vf_immed_vlan_work *work; + struct mlx4_dev *dev = &priv->dev; + int err; + int admin_vlan_ix = NO_INDX; + + vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; + vp_admin = &priv->mfunc.master.vf_admin[slave].vport[port]; + + if (vp_oper->state.default_vlan == vp_admin->default_vlan && + vp_oper->state.default_qos == vp_admin->default_qos && + vp_oper->state.vlan_proto == vp_admin->vlan_proto && + vp_oper->state.qos_vport == vp_admin->qos_vport) + return 0; + + if (!(priv->mfunc.master.slave_state[slave].active && + dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_UPDATE_QP)) { + /* even if the UPDATE_QP command isn't supported, we still want + * to set this VF link according to the admin directive + */ + return -1; + } + + mlx4_dbg(dev, "updating immediately admin params slave %d port %d\n", + slave, port); + mlx4_dbg(dev, "vlan %d QoS %d link down\n", + vp_admin->default_vlan, vp_admin->default_qos); + + work = kzalloc(sizeof(*work), GFP_KERNEL); + if (!work) + return -ENOMEM; + + if (vp_oper->state.default_vlan != vp_admin->default_vlan) { + if (MLX4_VGT != vp_admin->default_vlan) { + err = __mlx4_register_vlan(&priv->dev, port, + vp_admin->default_vlan, + &admin_vlan_ix); + if (err) { + kfree(work); + mlx4_warn(&priv->dev, + "No vlan resources slave %d, port %d\n", + slave, port); + return err; + } + } else { + admin_vlan_ix = NO_INDX; + } + work->flags |= MLX4_VF_IMMED_VLAN_FLAG_VLAN; + mlx4_dbg(&priv->dev, + "alloc vlan %d idx %d slave %d port %d\n", + (int)(vp_admin->default_vlan), + admin_vlan_ix, slave, port); + } + + /* save original vlan ix and vlan id */ + work->orig_vlan_id = vp_oper->state.default_vlan; + work->orig_vlan_ix = vp_oper->vlan_idx; + + /* handle new qos */ + if (vp_oper->state.default_qos != vp_admin->default_qos) + work->flags |= MLX4_VF_IMMED_VLAN_FLAG_QOS; + + if (work->flags & MLX4_VF_IMMED_VLAN_FLAG_VLAN) + vp_oper->vlan_idx = admin_vlan_ix; + + vp_oper->state.default_vlan = vp_admin->default_vlan; + vp_oper->state.default_qos = vp_admin->default_qos; + vp_oper->state.vlan_proto = vp_admin->vlan_proto; + vp_oper->state.qos_vport = vp_admin->qos_vport; + + if (1 /* vp_admin->link_state == IFLA_VF_LINK_STATE_DISABLE */) + work->flags |= MLX4_VF_IMMED_VLAN_FLAG_LINK_DISABLE; + + /* iterate over QPs owned by this slave, using UPDATE_QP */ + work->port = port; + work->slave = slave; + work->qos = vp_oper->state.default_qos; + work->qos_vport = vp_oper->state.qos_vport; + work->vlan_id = vp_oper->state.default_vlan; + work->vlan_ix = vp_oper->vlan_idx; + work->vlan_proto = vp_oper->state.vlan_proto; + work->priv = priv; + INIT_WORK(&work->work, mlx4_vf_immed_vlan_work_handler); + queue_work(priv->mfunc.master.comm_wq, &work->work); + + return 0; +} + +static void mlx4_set_default_port_qos(struct mlx4_dev *dev, int port) +{ + struct mlx4_qos_manager *port_qos_ctl; + struct mlx4_priv *priv = mlx4_priv(dev); + + port_qos_ctl = &priv->mfunc.master.qos_ctl[port]; + bitmap_zero(port_qos_ctl->priority_bm, MLX4_NUM_UP); + + /* Enable only default prio at PF init routine */ + set_bit(MLX4_DEFAULT_QOS_PRIO, port_qos_ctl->priority_bm); +} + +static void mlx4_allocate_port_vpps(struct mlx4_dev *dev, int port) +{ + int i; + int err; + int num_vfs; + u16 availible_vpp; + u8 vpp_param[MLX4_NUM_UP]; + struct mlx4_qos_manager *port_qos; + struct mlx4_priv *priv = mlx4_priv(dev); + + err = mlx4_ALLOCATE_VPP_get(dev, port, &availible_vpp, vpp_param); + if (err) { + mlx4_info(dev, "Failed query availible VPPs\n"); + return; + } + + port_qos = &priv->mfunc.master.qos_ctl[port]; + num_vfs = (availible_vpp / + bitmap_weight(port_qos->priority_bm, MLX4_NUM_UP)); + + for (i = 0; i < MLX4_NUM_UP; i++) { + if (test_bit(i, port_qos->priority_bm)) + vpp_param[i] = num_vfs; + } + + err = mlx4_ALLOCATE_VPP_set(dev, port, vpp_param); + if (err) { + mlx4_info(dev, "Failed allocating VPPs\n"); + return; + } + + /* Query actual allocated VPP, just to make sure */ + err = mlx4_ALLOCATE_VPP_get(dev, port, &availible_vpp, vpp_param); + if (err) { + mlx4_info(dev, "Failed query availible VPPs\n"); + return; + } + + port_qos->num_of_qos_vfs = num_vfs; + mlx4_dbg(dev, "Port %d Availible VPPs %d\n", port, availible_vpp); + + for (i = 0; i < MLX4_NUM_UP; i++) + mlx4_dbg(dev, "Port %d UP %d Allocated %d VPPs\n", port, i, + vpp_param[i]); +} + +static int mlx4_master_activate_admin_state(struct mlx4_priv *priv, int slave) +{ + int port, err; + struct mlx4_vport_state *vp_admin; + struct mlx4_vport_oper_state *vp_oper; + struct mlx4_slave_state *slave_state = + &priv->mfunc.master.slave_state[slave]; + struct mlx4_active_ports actv_ports = mlx4_get_active_ports( + &priv->dev, slave); + int min_port = find_first_bit(actv_ports.ports, + priv->dev.caps.num_ports) + 1; + int max_port = min_port - 1 + + bitmap_weight(actv_ports.ports, priv->dev.caps.num_ports); + + for (port = min_port; port <= max_port; port++) { + if (!test_bit(port - 1, actv_ports.ports)) + continue; + priv->mfunc.master.vf_oper[slave].smi_enabled[port] = + priv->mfunc.master.vf_admin[slave].enable_smi[port]; + vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; + vp_admin = &priv->mfunc.master.vf_admin[slave].vport[port]; + if (vp_admin->vlan_proto != htons(ETH_P_8021AD) || + slave_state->vst_qinq_supported) { + vp_oper->state.vlan_proto = vp_admin->vlan_proto; + vp_oper->state.default_vlan = vp_admin->default_vlan; + vp_oper->state.default_qos = vp_admin->default_qos; + } + vp_oper->state.mac = vp_admin->mac; + vp_oper->state.spoofchk = vp_admin->spoofchk; + vp_oper->state.tx_rate = vp_admin->tx_rate; + vp_oper->state.qos_vport = vp_admin->qos_vport; + vp_oper->state.guid = vp_admin->guid; + + if (MLX4_VGT != vp_admin->default_vlan) { + err = __mlx4_register_vlan(&priv->dev, port, + vp_admin->default_vlan, &(vp_oper->vlan_idx)); + if (err) { + vp_oper->vlan_idx = NO_INDX; + vp_oper->state.default_vlan = MLX4_VGT; + vp_oper->state.vlan_proto = htons(ETH_P_8021Q); + mlx4_warn(&priv->dev, + "No vlan resources slave %d, port %d\n", + slave, port); + return err; + } + mlx4_dbg(&priv->dev, "alloc vlan %d idx %d slave %d port %d\n", + (int)(vp_oper->state.default_vlan), + vp_oper->vlan_idx, slave, port); + } + if (vp_admin->spoofchk) { + vp_oper->mac_idx = __mlx4_register_mac(&priv->dev, + port, + vp_admin->mac); + if (0 > vp_oper->mac_idx) { + err = vp_oper->mac_idx; + vp_oper->mac_idx = NO_INDX; + mlx4_warn(&priv->dev, + "No mac resources slave %d, port %d\n", + slave, port); + return err; + } + mlx4_dbg(&priv->dev, "alloc mac %llx idx %d slave %d port %d\n", + (unsigned long long) vp_oper->state.mac, vp_oper->mac_idx, slave, port); + } + } + return 0; +} + +static void mlx4_master_deactivate_admin_state(struct mlx4_priv *priv, int slave) +{ + int port; + struct mlx4_vport_oper_state *vp_oper; + struct mlx4_active_ports actv_ports = mlx4_get_active_ports( + &priv->dev, slave); + int min_port = find_first_bit(actv_ports.ports, + priv->dev.caps.num_ports) + 1; + int max_port = min_port - 1 + + bitmap_weight(actv_ports.ports, priv->dev.caps.num_ports); + + + for (port = min_port; port <= max_port; port++) { + if (!test_bit(port - 1, actv_ports.ports)) + continue; + priv->mfunc.master.vf_oper[slave].smi_enabled[port] = + MLX4_VF_SMI_DISABLED; + vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; + if (NO_INDX != vp_oper->vlan_idx) { + __mlx4_unregister_vlan(&priv->dev, + port, vp_oper->state.default_vlan); + vp_oper->vlan_idx = NO_INDX; + } + if (NO_INDX != vp_oper->mac_idx) { + __mlx4_unregister_mac(&priv->dev, port, vp_oper->state.mac); + vp_oper->mac_idx = NO_INDX; + } + } + return; +} + +static void mlx4_master_do_cmd(struct mlx4_dev *dev, int slave, u8 cmd, + u16 param, u8 toggle) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state; + u32 reply; + u8 is_going_down = 0; + int i; + unsigned long flags; + + slave_state[slave].comm_toggle ^= 1; + reply = (u32) slave_state[slave].comm_toggle << 31; + if (toggle != slave_state[slave].comm_toggle) { + mlx4_warn(dev, "Incorrect toggle %d from slave %d. *** MASTER STATE COMPROMISED ***\n", + toggle, slave); + goto reset_slave; + } + if (cmd == MLX4_COMM_CMD_RESET) { + mlx4_warn(dev, "Received reset from slave:%d\n", slave); + slave_state[slave].active = false; + slave_state[slave].old_vlan_api = false; + slave_state[slave].vst_qinq_supported = false; + mlx4_master_deactivate_admin_state(priv, slave); + for (i = 0; i < MLX4_EVENT_TYPES_NUM; ++i) { + slave_state[slave].event_eq[i].eqn = -1; + slave_state[slave].event_eq[i].token = 0; + } + /*check if we are in the middle of FLR process, + if so return "retry" status to the slave*/ + if (MLX4_COMM_CMD_FLR == slave_state[slave].last_cmd) + goto inform_slave_state; + + mlx4_dispatch_event(dev, MLX4_DEV_EVENT_SLAVE_SHUTDOWN, slave); + + /* write the version in the event field */ + reply |= mlx4_comm_get_version(); + + goto reset_slave; + } + /*command from slave in the middle of FLR*/ + if (cmd != MLX4_COMM_CMD_RESET && + MLX4_COMM_CMD_FLR == slave_state[slave].last_cmd) { + mlx4_warn(dev, "slave:%d is Trying to run cmd(0x%x) in the middle of FLR\n", + slave, cmd); + return; + } + + switch (cmd) { + case MLX4_COMM_CMD_VHCR0: + if (slave_state[slave].last_cmd != MLX4_COMM_CMD_RESET) + goto reset_slave; + slave_state[slave].vhcr_dma = ((u64) param) << 48; + priv->mfunc.master.slave_state[slave].cookie = 0; + break; + case MLX4_COMM_CMD_VHCR1: + if (slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR0) + goto reset_slave; + slave_state[slave].vhcr_dma |= ((u64) param) << 32; + break; + case MLX4_COMM_CMD_VHCR2: + if (slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR1) + goto reset_slave; + slave_state[slave].vhcr_dma |= ((u64) param) << 16; + break; + case MLX4_COMM_CMD_VHCR_EN: + if (slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR2) + goto reset_slave; + slave_state[slave].vhcr_dma |= param; + if (mlx4_master_activate_admin_state(priv, slave)) + goto reset_slave; + slave_state[slave].active = true; + mlx4_dispatch_event(dev, MLX4_DEV_EVENT_SLAVE_INIT, slave); + break; + case MLX4_COMM_CMD_VHCR_POST: + if ((slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR_EN) && + (slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR_POST)) { + mlx4_warn(dev, "slave:%d is out of sync, cmd=0x%x, last command=0x%x, reset is needed\n", + slave, cmd, slave_state[slave].last_cmd); + goto reset_slave; + } + + mutex_lock(&priv->cmd.slave_cmd_mutex); + if (mlx4_master_process_vhcr(dev, slave, NULL)) { + mlx4_err(dev, "Failed processing vhcr for slave:%d, resetting slave\n", + slave); + mutex_unlock(&priv->cmd.slave_cmd_mutex); + goto reset_slave; + } + mutex_unlock(&priv->cmd.slave_cmd_mutex); + break; + default: + mlx4_warn(dev, "Bad comm cmd:%d from slave:%d\n", cmd, slave); + goto reset_slave; + } + spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags); + if (!slave_state[slave].is_slave_going_down) + slave_state[slave].last_cmd = cmd; + else + is_going_down = 1; + spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags); + if (is_going_down) { + mlx4_warn(dev, "Slave is going down aborting command(%d) executing from slave:%d\n", + cmd, slave); + return; + } + __raw_writel((__force u32) cpu_to_be32(reply), + &priv->mfunc.comm[slave].slave_read); + mmiowb(); + + return; + +reset_slave: + /* cleanup any slave resources */ + if (dev->persist->interface_state & MLX4_INTERFACE_STATE_UP) + mlx4_delete_all_resources_for_slave(dev, slave); + + if (cmd != MLX4_COMM_CMD_RESET) { + mlx4_warn(dev, "Turn on internal error to force reset, slave=%d, cmd=0x%x\n", + slave, cmd); + /* Turn on internal error letting slave reset itself immeditaly, + * otherwise it might take till timeout on command is passed + */ + reply |= ((u32)COMM_CHAN_EVENT_INTERNAL_ERR); + } + + spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags); + if (!slave_state[slave].is_slave_going_down) + slave_state[slave].last_cmd = MLX4_COMM_CMD_RESET; + spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags); + /*with slave in the middle of flr, no need to clean resources again.*/ +inform_slave_state: + memset(&slave_state[slave].event_eq, 0, + sizeof(struct mlx4_slave_event_eq_info)); + __raw_writel((__force u32) cpu_to_be32(reply), + &priv->mfunc.comm[slave].slave_read); + wmb(); +} + +/* master command processing */ +void mlx4_master_comm_channel(struct work_struct *work) +{ + struct mlx4_mfunc_master_ctx *master = + container_of(work, + struct mlx4_mfunc_master_ctx, + comm_work); + struct mlx4_mfunc *mfunc = + container_of(master, struct mlx4_mfunc, master); + struct mlx4_priv *priv = + container_of(mfunc, struct mlx4_priv, mfunc); + struct mlx4_dev *dev = &priv->dev; + __be32 *bit_vec; + u32 comm_cmd; + u32 vec; + int i, j, slave; + int toggle; + int served = 0; + int reported = 0; + u32 slt; + + bit_vec = master->comm_arm_bit_vector; + for (i = 0; i < COMM_CHANNEL_BIT_ARRAY_SIZE; i++) { + vec = be32_to_cpu(bit_vec[i]); + for (j = 0; j < 32; j++) { + if (!(vec & (1 << j))) + continue; + ++reported; + slave = (i * 32) + j; + comm_cmd = swab32(readl( + &mfunc->comm[slave].slave_write)); + slt = swab32(readl(&mfunc->comm[slave].slave_read)) + >> 31; + toggle = comm_cmd >> 31; + if (toggle != slt) { + if (master->slave_state[slave].comm_toggle + != slt) { + pr_info("slave %d out of sync. read toggle %d, state toggle %d. Resynching.\n", + slave, slt, + master->slave_state[slave].comm_toggle); + master->slave_state[slave].comm_toggle = + slt; + } + mlx4_master_do_cmd(dev, slave, + comm_cmd >> 16 & 0xff, + comm_cmd & 0xffff, toggle); + ++served; + } + } + } + + if (reported && reported != served) + mlx4_warn(dev, "Got command event with bitmask from %d slaves but %d were served\n", + reported, served); + + if (mlx4_ARM_COMM_CHANNEL(dev)) + mlx4_warn(dev, "Failed to arm comm channel events\n"); +} + +static int sync_toggles(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + u32 wr_toggle; + u32 rd_toggle; + unsigned long end; + + wr_toggle = swab32(readl(&priv->mfunc.comm->slave_write)); + if (wr_toggle == 0xffffffff) + end = jiffies + msecs_to_jiffies(30000); + else + end = jiffies + msecs_to_jiffies(5000); + + while (time_before(jiffies, end)) { + rd_toggle = swab32(readl(&priv->mfunc.comm->slave_read)); + if (wr_toggle == 0xffffffff || rd_toggle == 0xffffffff) { + /* PCI might be offline */ + msleep(100); + wr_toggle = swab32(readl(&priv->mfunc.comm-> + slave_write)); + continue; + } + + if (rd_toggle >> 31 == wr_toggle >> 31) { + priv->cmd.comm_toggle = rd_toggle >> 31; + return 0; + } + + cond_resched(); + } + + /* + * we could reach here if for example the previous VM using this + * function misbehaved and left the channel with unsynced state. We + * should fix this here and give this VM a chance to use a properly + * synced channel + */ + mlx4_warn(dev, "recovering from previously mis-behaved VM\n"); + __raw_writel((__force u32) 0, &priv->mfunc.comm->slave_read); + __raw_writel((__force u32) 0, &priv->mfunc.comm->slave_write); + priv->cmd.comm_toggle = 0; + + return 0; +} + +int mlx4_multi_func_init(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *s_state; + int i, j, err, port; + + if (mlx4_is_master(dev)) + priv->mfunc.comm = + ioremap(pci_resource_start(dev->persist->pdev, + priv->fw.comm_bar) + + priv->fw.comm_base, MLX4_COMM_PAGESIZE); + else + priv->mfunc.comm = + ioremap(pci_resource_start(dev->persist->pdev, 2) + + MLX4_SLAVE_COMM_BASE, MLX4_COMM_PAGESIZE); + if (!priv->mfunc.comm) { + mlx4_err(dev, "Couldn't map communication vector\n"); + goto err_vhcr; + } + + if (mlx4_is_master(dev)) { + struct mlx4_vf_oper_state *vf_oper; + struct mlx4_vf_admin_state *vf_admin; + + priv->mfunc.master.slave_state = + kzalloc(dev->num_slaves * + sizeof(struct mlx4_slave_state), GFP_KERNEL); + if (!priv->mfunc.master.slave_state) + goto err_comm; + + priv->mfunc.master.vf_admin = + kzalloc(dev->num_slaves * + sizeof(struct mlx4_vf_admin_state), GFP_KERNEL); + if (!priv->mfunc.master.vf_admin) + goto err_comm_admin; + + priv->mfunc.master.vf_oper = + kzalloc(dev->num_slaves * + sizeof(struct mlx4_vf_oper_state), GFP_KERNEL); + if (!priv->mfunc.master.vf_oper) + goto err_comm_oper; + + for (i = 0; i < dev->num_slaves; ++i) { + vf_admin = &priv->mfunc.master.vf_admin[i]; + vf_oper = &priv->mfunc.master.vf_oper[i]; + s_state = &priv->mfunc.master.slave_state[i]; + s_state->last_cmd = MLX4_COMM_CMD_RESET; + s_state->vst_qinq_supported = false; + mutex_init(&priv->mfunc.master.gen_eqe_mutex[i]); + for (j = 0; j < MLX4_EVENT_TYPES_NUM; ++j) + s_state->event_eq[j].eqn = -1; + __raw_writel((__force u32) 0, + &priv->mfunc.comm[i].slave_write); + __raw_writel((__force u32) 0, + &priv->mfunc.comm[i].slave_read); + mmiowb(); + for (port = 1; port <= MLX4_MAX_PORTS; port++) { + struct mlx4_vport_state *admin_vport; + struct mlx4_vport_state *oper_vport; + + s_state->vlan_filter[port] = + kzalloc(sizeof(struct mlx4_vlan_fltr), + GFP_KERNEL); + if (!s_state->vlan_filter[port]) { + if (--port) + kfree(s_state->vlan_filter[port]); + goto err_slaves; + } + + admin_vport = &vf_admin->vport[port]; + oper_vport = &vf_oper->vport[port].state; + INIT_LIST_HEAD(&s_state->mcast_filters[port]); + admin_vport->default_vlan = MLX4_VGT; + oper_vport->default_vlan = MLX4_VGT; + admin_vport->qos_vport = + MLX4_VPP_DEFAULT_VPORT; + oper_vport->qos_vport = MLX4_VPP_DEFAULT_VPORT; + admin_vport->vlan_proto = htons(ETH_P_8021Q); + oper_vport->vlan_proto = htons(ETH_P_8021Q); + vf_oper->vport[port].vlan_idx = NO_INDX; + vf_oper->vport[port].mac_idx = NO_INDX; + mlx4_set_random_admin_guid(dev, i, port); + } + spin_lock_init(&s_state->lock); + } + + if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_QOS_VPP) { + for (port = 1; port <= dev->caps.num_ports; port++) { + if (mlx4_is_eth(dev, port)) { + mlx4_set_default_port_qos(dev, port); + mlx4_allocate_port_vpps(dev, port); + } + } + } + + memset(&priv->mfunc.master.cmd_eqe, 0, sizeof(struct mlx4_eqe)); + priv->mfunc.master.cmd_eqe.type = MLX4_EVENT_TYPE_CMD; + INIT_WORK(&priv->mfunc.master.comm_work, + mlx4_master_comm_channel); + INIT_WORK(&priv->mfunc.master.slave_event_work, + mlx4_gen_slave_eqe); + INIT_WORK(&priv->mfunc.master.slave_flr_event_work, + mlx4_master_handle_slave_flr); + spin_lock_init(&priv->mfunc.master.slave_state_lock); + spin_lock_init(&priv->mfunc.master.slave_eq.event_lock); + priv->mfunc.master.comm_wq = + create_singlethread_workqueue("mlx4_comm"); + if (!priv->mfunc.master.comm_wq) + goto err_slaves; + + if (mlx4_init_resource_tracker(dev)) + goto err_thread; + + } else { + err = sync_toggles(dev); + if (err) { + mlx4_err(dev, "Couldn't sync toggles\n"); + goto err_comm; + } + } + return 0; + +err_thread: + flush_workqueue(priv->mfunc.master.comm_wq); + destroy_workqueue(priv->mfunc.master.comm_wq); +err_slaves: + while (i--) { + for (port = 1; port <= MLX4_MAX_PORTS; port++) + kfree(priv->mfunc.master.slave_state[i].vlan_filter[port]); + } + kfree(priv->mfunc.master.vf_oper); +err_comm_oper: + kfree(priv->mfunc.master.vf_admin); +err_comm_admin: + kfree(priv->mfunc.master.slave_state); +err_comm: + iounmap(priv->mfunc.comm); + priv->mfunc.comm = NULL; +err_vhcr: + dma_free_coherent(&dev->persist->pdev->dev, PAGE_SIZE, + priv->mfunc.vhcr, + priv->mfunc.vhcr_dma); + priv->mfunc.vhcr = NULL; + return -ENOMEM; +} + +int mlx4_cmd_init(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int flags = 0; + + if (!priv->cmd.initialized) { + init_rwsem(&priv->cmd.switch_sem); + mutex_init(&priv->cmd.slave_cmd_mutex); + sema_init(&priv->cmd.poll_sem, 1); + priv->cmd.use_events = 0; + priv->cmd.toggle = 1; + priv->cmd.initialized = 1; + flags |= MLX4_CMD_CLEANUP_STRUCT; + } + + if (!mlx4_is_slave(dev) && !priv->cmd.hcr) { + priv->cmd.hcr = ioremap(pci_resource_start(dev->persist->pdev, + 0) + MLX4_HCR_BASE, MLX4_HCR_SIZE); + if (!priv->cmd.hcr) { + mlx4_err(dev, "Couldn't map command register\n"); + goto err; + } + flags |= MLX4_CMD_CLEANUP_HCR; + } + + if (mlx4_is_mfunc(dev) && !priv->mfunc.vhcr) { + priv->mfunc.vhcr = dma_alloc_coherent(&dev->persist->pdev->dev, + PAGE_SIZE, + &priv->mfunc.vhcr_dma, + GFP_KERNEL); + if (!priv->mfunc.vhcr) + goto err; + + flags |= MLX4_CMD_CLEANUP_VHCR; + } + + if (!priv->cmd.pool) { + priv->cmd.pool = pci_pool_create("mlx4_cmd", + dev->persist->pdev, + MLX4_MAILBOX_SIZE, + MLX4_MAILBOX_SIZE, 0); + if (!priv->cmd.pool) + goto err; + + flags |= MLX4_CMD_CLEANUP_POOL; + } + + return 0; + +err: + mlx4_cmd_cleanup(dev, flags); + return -ENOMEM; +} + +void mlx4_report_internal_err_comm_event(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int slave; + u32 slave_read; + + /* If the comm channel has not yet been initialized, + * skip reporting the internal error event to all + * the communication channels. + */ + if (!priv->mfunc.comm) + return; + + /* Report an internal error event to all + * communication channels. + */ + for (slave = 0; slave < dev->num_slaves; slave++) { + slave_read = swab32(readl(&priv->mfunc.comm[slave].slave_read)); + slave_read |= (u32)COMM_CHAN_EVENT_INTERNAL_ERR; + __raw_writel((__force u32)cpu_to_be32(slave_read), + &priv->mfunc.comm[slave].slave_read); + /* Make sure that our comm channel write doesn't + * get mixed in with writes from another CPU. + */ + mmiowb(); + } +} + +void mlx4_multi_func_cleanup(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int i, port; + + if (mlx4_is_master(dev)) { + flush_workqueue(priv->mfunc.master.comm_wq); + destroy_workqueue(priv->mfunc.master.comm_wq); + for (i = 0; i < dev->num_slaves; i++) { + for (port = 1; port <= MLX4_MAX_PORTS; port++) + kfree(priv->mfunc.master.slave_state[i].vlan_filter[port]); + } + kfree(priv->mfunc.master.slave_state); + kfree(priv->mfunc.master.vf_admin); + kfree(priv->mfunc.master.vf_oper); + dev->num_slaves = 0; + } + + iounmap(priv->mfunc.comm); + priv->mfunc.comm = NULL; +} + +void mlx4_cmd_cleanup(struct mlx4_dev *dev, int cleanup_mask) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + if (priv->cmd.pool && (cleanup_mask & MLX4_CMD_CLEANUP_POOL)) { + pci_pool_destroy(priv->cmd.pool); + priv->cmd.pool = NULL; + } + + if (!mlx4_is_slave(dev) && priv->cmd.hcr && + (cleanup_mask & MLX4_CMD_CLEANUP_HCR)) { + iounmap(priv->cmd.hcr); + priv->cmd.hcr = NULL; + } + if (mlx4_is_mfunc(dev) && priv->mfunc.vhcr && + (cleanup_mask & MLX4_CMD_CLEANUP_VHCR)) { + dma_free_coherent(&dev->persist->pdev->dev, PAGE_SIZE, + priv->mfunc.vhcr, priv->mfunc.vhcr_dma); + priv->mfunc.vhcr = NULL; + } + if (priv->cmd.initialized && (cleanup_mask & MLX4_CMD_CLEANUP_STRUCT)) + priv->cmd.initialized = 0; +} + +/* + * Switch to using events to issue FW commands (can only be called + * after event queue for command events has been initialized). + */ +int mlx4_cmd_use_events(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int i; + int err = 0; + + priv->cmd.context = kmalloc(priv->cmd.max_cmds * + sizeof (struct mlx4_cmd_context), + GFP_KERNEL); + if (!priv->cmd.context) + return -ENOMEM; + + down_write(&priv->cmd.switch_sem); + for (i = 0; i < priv->cmd.max_cmds; ++i) { + priv->cmd.context[i].token = i; + priv->cmd.context[i].next = i + 1; + /* To support fatal error flow, initialize all + * cmd contexts to allow simulating completions + * with complete() at any time. + */ + init_completion(&priv->cmd.context[i].done); + } + + priv->cmd.context[priv->cmd.max_cmds - 1].next = -1; + priv->cmd.free_head = 0; + + sema_init(&priv->cmd.event_sem, priv->cmd.max_cmds); + + for (priv->cmd.token_mask = 1; + priv->cmd.token_mask < priv->cmd.max_cmds; + priv->cmd.token_mask <<= 1) + ; /* nothing */ + --priv->cmd.token_mask; + + down(&priv->cmd.poll_sem); + priv->cmd.use_events = 1; + up_write(&priv->cmd.switch_sem); + + return err; +} + +/* + * Switch back to polling (used when shutting down the device) + */ +void mlx4_cmd_use_polling(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int i; + + down_write(&priv->cmd.switch_sem); + priv->cmd.use_events = 0; + + for (i = 0; i < priv->cmd.max_cmds; ++i) + down(&priv->cmd.event_sem); + + kfree(priv->cmd.context); + + up(&priv->cmd.poll_sem); + up_write(&priv->cmd.switch_sem); +} + +struct mlx4_cmd_mailbox *mlx4_alloc_cmd_mailbox(struct mlx4_dev *dev) +{ + struct mlx4_cmd_mailbox *mailbox; + + mailbox = kmalloc(sizeof *mailbox, GFP_KERNEL); + if (!mailbox) + return ERR_PTR(-ENOMEM); + + mailbox->buf = pci_pool_alloc(mlx4_priv(dev)->cmd.pool, GFP_KERNEL, + &mailbox->dma); + if (!mailbox->buf) { + kfree(mailbox); + return ERR_PTR(-ENOMEM); + } + + memset(mailbox->buf, 0, MLX4_MAILBOX_SIZE); + + return mailbox; +} +EXPORT_SYMBOL_GPL(mlx4_alloc_cmd_mailbox); + +void mlx4_free_cmd_mailbox(struct mlx4_dev *dev, + struct mlx4_cmd_mailbox *mailbox) +{ + if (!mailbox) + return; + + pci_pool_free(mlx4_priv(dev)->cmd.pool, mailbox->buf, mailbox->dma); + kfree(mailbox); +} +EXPORT_SYMBOL_GPL(mlx4_free_cmd_mailbox); + +u32 mlx4_comm_get_version(void) +{ + return ((u32) CMD_CHAN_IF_REV << 8) | (u32) CMD_CHAN_VER; +} + +static int mlx4_get_slave_indx(struct mlx4_dev *dev, int vf) +{ + if ((vf < 0) || (vf >= dev->persist->num_vfs)) { + mlx4_err(dev, "Bad vf number:%d (number of activated vf: %d)\n", + vf, dev->persist->num_vfs); + return -EINVAL; + } + + return vf+1; +} + +int mlx4_get_vf_indx(struct mlx4_dev *dev, int slave) +{ + if (slave < 1 || slave > dev->persist->num_vfs) { + mlx4_err(dev, + "Bad slave number:%d (number of activated slaves: %lu)\n", + slave, dev->num_slaves); + return -EINVAL; + } + return slave - 1; +} + +void mlx4_cmd_wake_completions(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_cmd_context *context; + int i; + + spin_lock(&priv->cmd.context_lock); + if (priv->cmd.context) { + for (i = 0; i < priv->cmd.max_cmds; ++i) { + context = &priv->cmd.context[i]; + context->fw_status = CMD_STAT_INTERNAL_ERR; + context->result = + mlx4_status_to_errno(CMD_STAT_INTERNAL_ERR); + complete(&context->done); + } + } + spin_unlock(&priv->cmd.context_lock); +} + +struct mlx4_active_ports mlx4_get_active_ports(struct mlx4_dev *dev, int slave) +{ + struct mlx4_active_ports actv_ports; + int vf; + + bitmap_zero(actv_ports.ports, MLX4_MAX_PORTS); + + if (slave == 0) { + bitmap_fill(actv_ports.ports, dev->caps.num_ports); + return actv_ports; + } + + vf = mlx4_get_vf_indx(dev, slave); + if (vf < 0) + return actv_ports; + + bitmap_set(actv_ports.ports, dev->dev_vfs[vf].min_port - 1, + min((int)dev->dev_vfs[mlx4_get_vf_indx(dev, slave)].n_ports, + dev->caps.num_ports)); + + return actv_ports; +} +EXPORT_SYMBOL_GPL(mlx4_get_active_ports); + +int mlx4_slave_convert_port(struct mlx4_dev *dev, int slave, int port) +{ + unsigned n; + struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, slave); + unsigned m = bitmap_weight(actv_ports.ports, dev->caps.num_ports); + + if (port <= 0 || port > m) + return -EINVAL; + + n = find_first_bit(actv_ports.ports, dev->caps.num_ports); + if (port <= n) + port = n + 1; + + return port; +} +EXPORT_SYMBOL_GPL(mlx4_slave_convert_port); + +int mlx4_phys_to_slave_port(struct mlx4_dev *dev, int slave, int port) +{ + struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, slave); + if (test_bit(port - 1, actv_ports.ports)) + return port - + find_first_bit(actv_ports.ports, dev->caps.num_ports); + + return -1; +} +EXPORT_SYMBOL_GPL(mlx4_phys_to_slave_port); + +struct mlx4_slaves_pport mlx4_phys_to_slaves_pport(struct mlx4_dev *dev, + int port) +{ + unsigned i; + struct mlx4_slaves_pport slaves_pport; + + bitmap_zero(slaves_pport.slaves, MLX4_MFUNC_MAX); + + if (port <= 0 || port > dev->caps.num_ports) + return slaves_pport; + + for (i = 0; i < dev->persist->num_vfs + 1; i++) { + struct mlx4_active_ports actv_ports = + mlx4_get_active_ports(dev, i); + if (test_bit(port - 1, actv_ports.ports)) + set_bit(i, slaves_pport.slaves); + } + + return slaves_pport; +} +EXPORT_SYMBOL_GPL(mlx4_phys_to_slaves_pport); + +struct mlx4_slaves_pport mlx4_phys_to_slaves_pport_actv( + struct mlx4_dev *dev, + const struct mlx4_active_ports *crit_ports) +{ + unsigned i; + struct mlx4_slaves_pport slaves_pport; + + bitmap_zero(slaves_pport.slaves, MLX4_MFUNC_MAX); + + for (i = 0; i < dev->persist->num_vfs + 1; i++) { + struct mlx4_active_ports actv_ports = + mlx4_get_active_ports(dev, i); + if (bitmap_equal(crit_ports->ports, actv_ports.ports, + dev->caps.num_ports)) + set_bit(i, slaves_pport.slaves); + } + + return slaves_pport; +} +EXPORT_SYMBOL_GPL(mlx4_phys_to_slaves_pport_actv); + +static int mlx4_slaves_closest_port(struct mlx4_dev *dev, int slave, int port) +{ + struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, slave); + int min_port = find_first_bit(actv_ports.ports, dev->caps.num_ports) + + 1; + int max_port = min_port + + bitmap_weight(actv_ports.ports, dev->caps.num_ports); + + if (port < min_port) + port = min_port; + else if (port >= max_port) + port = max_port - 1; + + return port; +} + +static int mlx4_set_vport_qos(struct mlx4_priv *priv, int slave, int port, + int max_tx_rate) +{ + int i; + int err; + struct mlx4_qos_manager *port_qos; + struct mlx4_dev *dev = &priv->dev; + struct mlx4_vport_qos_param vpp_qos[MLX4_NUM_UP]; + + port_qos = &priv->mfunc.master.qos_ctl[port]; + memset(vpp_qos, 0, sizeof(struct mlx4_vport_qos_param) * MLX4_NUM_UP); + + if (slave > port_qos->num_of_qos_vfs) { + mlx4_info(dev, "No availible VPP resources for this VF\n"); + return -EINVAL; + } + + /* Query for default QoS values from Vport 0 is needed */ + err = mlx4_SET_VPORT_QOS_get(dev, port, 0, vpp_qos); + if (err) { + mlx4_info(dev, "Failed to query Vport 0 QoS values\n"); + return err; + } + + for (i = 0; i < MLX4_NUM_UP; i++) { + if (test_bit(i, port_qos->priority_bm) && max_tx_rate) { + vpp_qos[i].max_avg_bw = max_tx_rate; + vpp_qos[i].enable = 1; + } else { + /* if user supplied tx_rate == 0, meaning no rate limit + * configuration is required. so we are leaving the + * value of max_avg_bw as queried from Vport 0. + */ + vpp_qos[i].enable = 0; + } + } + + err = mlx4_SET_VPORT_QOS_set(dev, port, slave, vpp_qos); + if (err) { + mlx4_info(dev, "Failed to set Vport %d QoS values\n", slave); + return err; + } + + return 0; +} + +static bool mlx4_is_vf_vst_and_prio_qos(struct mlx4_dev *dev, int port, + struct mlx4_vport_state *vf_admin) +{ + struct mlx4_qos_manager *info; + struct mlx4_priv *priv = mlx4_priv(dev); + + if (!mlx4_is_master(dev) || + !(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_QOS_VPP)) + return false; + + info = &priv->mfunc.master.qos_ctl[port]; + + if (vf_admin->default_vlan != MLX4_VGT && + test_bit(vf_admin->default_qos, info->priority_bm)) + return true; + + return false; +} + +static bool mlx4_valid_vf_state_change(struct mlx4_dev *dev, int port, + struct mlx4_vport_state *vf_admin, + int vlan, int qos) +{ + struct mlx4_vport_state dummy_admin = {0}; + + if (!mlx4_is_vf_vst_and_prio_qos(dev, port, vf_admin) || + !vf_admin->tx_rate) + return true; + + dummy_admin.default_qos = qos; + dummy_admin.default_vlan = vlan; + + /* VF wants to move to other VST state which is valid with current + * rate limit. Either differnt default vlan in VST or other + * supported QoS priority. Otherwise we don't allow this change when + * the TX rate is still configured. + */ + if (mlx4_is_vf_vst_and_prio_qos(dev, port, &dummy_admin)) + return true; + + mlx4_info(dev, "Cannot change VF state to %s while rate is set\n", + (vlan == MLX4_VGT) ? "VGT" : "VST"); + + if (vlan != MLX4_VGT) + mlx4_info(dev, "VST priority %d not supported for QoS\n", qos); + + mlx4_info(dev, "Please set rate to 0 prior to this VF state change\n"); + + return false; +} + +int mlx4_set_vf_mac(struct mlx4_dev *dev, int port, int vf, u64 mac) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_vport_state *s_info; + int slave; + + if (!mlx4_is_master(dev)) + return -EPROTONOSUPPORT; + + slave = mlx4_get_slave_indx(dev, vf); + if (slave < 0) + return -EINVAL; + + port = mlx4_slaves_closest_port(dev, slave, port); + s_info = &priv->mfunc.master.vf_admin[slave].vport[port]; + s_info->mac = mac; + mlx4_info(dev, "default mac on vf %d port %d to %llX will take effect only after vf restart\n", + vf, port, (unsigned long long)s_info->mac); + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_set_vf_mac); + + +int mlx4_set_vf_vlan(struct mlx4_dev *dev, int port, int vf, u16 vlan, u8 qos, + __be16 proto) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_vport_state *vf_admin; + struct mlx4_slave_state *slave_state; + struct mlx4_vport_oper_state *vf_oper; + int slave; + + if ((!mlx4_is_master(dev)) || + !(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_VLAN_CONTROL)) + return -EPROTONOSUPPORT; + + if ((vlan > 4095) || (qos > 7)) + return -EINVAL; + + if (proto == htons(ETH_P_8021AD) && + !(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SVLAN_BY_QP)) + return -EPROTONOSUPPORT; + + if (proto != htons(ETH_P_8021Q) && + proto != htons(ETH_P_8021AD)) + return -EINVAL; + + if ((proto == htons(ETH_P_8021AD)) && + ((vlan == 0) || (vlan == MLX4_VGT))) + return -EINVAL; + + slave = mlx4_get_slave_indx(dev, vf); + if (slave < 0) + return -EINVAL; + + slave_state = &priv->mfunc.master.slave_state[slave]; + if ((proto == htons(ETH_P_8021AD)) && (slave_state->active) && + (!slave_state->vst_qinq_supported)) { + mlx4_err(dev, "vf %d does not support VST QinQ mode\n", vf); + return -EPROTONOSUPPORT; + } + port = mlx4_slaves_closest_port(dev, slave, port); + vf_admin = &priv->mfunc.master.vf_admin[slave].vport[port]; + vf_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; + + if (!mlx4_valid_vf_state_change(dev, port, vf_admin, vlan, qos)) + return -EPERM; + + if ((0 == vlan) && (0 == qos)) + vf_admin->default_vlan = MLX4_VGT; + else + vf_admin->default_vlan = vlan; + vf_admin->default_qos = qos; + vf_admin->vlan_proto = proto; + + /* If rate was configured prior to VST, we saved the configured rate + * in vf_admin->rate and now, if priority supported we enforce the QoS + */ + if (mlx4_is_vf_vst_and_prio_qos(dev, port, vf_admin) && + vf_admin->tx_rate) + vf_admin->qos_vport = slave; + + /* Try to activate new vf state without restart, + * this option is not supported while moving to VST QinQ mode. + */ + if ((proto == htons(ETH_P_8021AD) && + vf_oper->state.vlan_proto != proto) || + mlx4_master_immediate_activate_vlan_qos(priv, slave, port)) + mlx4_info(dev, + "updating vf %d port %d config will take effect on next VF restart\n", + vf, port); + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_set_vf_vlan); + +int mlx4_set_vf_rate(struct mlx4_dev *dev, int port, int vf, int min_tx_rate, + int max_tx_rate) +{ + int err; + int slave; + struct mlx4_vport_state *vf_admin; + struct mlx4_priv *priv = mlx4_priv(dev); + + if (!mlx4_is_master(dev) || + !(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_QOS_VPP)) + return -EPROTONOSUPPORT; + + if (min_tx_rate) { + mlx4_info(dev, "Minimum BW share not supported\n"); + return -EPROTONOSUPPORT; + } + + slave = mlx4_get_slave_indx(dev, vf); + if (slave < 0) + return -EINVAL; + + port = mlx4_slaves_closest_port(dev, slave, port); + vf_admin = &priv->mfunc.master.vf_admin[slave].vport[port]; + + err = mlx4_set_vport_qos(priv, slave, port, max_tx_rate); + if (err) { + mlx4_info(dev, "vf %d failed to set rate %d\n", vf, + max_tx_rate); + return err; + } + + vf_admin->tx_rate = max_tx_rate; + /* if VF is not in supported mode (VST with supported prio), + * we do not change vport configuration for its QPs, but save + * the rate, so it will be enforced when it moves to supported + * mode next time. + */ + if (!mlx4_is_vf_vst_and_prio_qos(dev, port, vf_admin)) { + mlx4_info(dev, + "rate set for VF %d when not in valid state\n", vf); + + if (vf_admin->default_vlan != MLX4_VGT) + mlx4_info(dev, "VST priority not supported by QoS\n"); + else + mlx4_info(dev, "VF in VGT mode (needed VST)\n"); + + mlx4_info(dev, + "rate %d take affect when VF moves to valid state\n", + max_tx_rate); + return 0; + } + + /* If user sets rate 0 assigning default vport for its QPs */ + vf_admin->qos_vport = max_tx_rate ? slave : MLX4_VPP_DEFAULT_VPORT; + + if (priv->mfunc.master.slave_state[slave].active && + dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_UPDATE_QP) + mlx4_master_immediate_activate_vlan_qos(priv, slave, port); + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_set_vf_rate); + + /* mlx4_get_slave_default_vlan - + * return true if VST ( default vlan) + * if VST, will return vlan & qos (if not NULL) + */ +bool mlx4_get_slave_default_vlan(struct mlx4_dev *dev, int port, int slave, + u16 *vlan, u8 *qos) +{ + struct mlx4_vport_oper_state *vp_oper; + struct mlx4_priv *priv; + + priv = mlx4_priv(dev); + port = mlx4_slaves_closest_port(dev, slave, port); + vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; + + if (MLX4_VGT != vp_oper->state.default_vlan) { + if (vlan) + *vlan = vp_oper->state.default_vlan; + if (qos) + *qos = vp_oper->state.default_qos; + return true; + } + return false; +} +EXPORT_SYMBOL_GPL(mlx4_get_slave_default_vlan); + +int mlx4_set_vf_spoofchk(struct mlx4_dev *dev, int port, int vf, bool setting) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_vport_state *s_info; + int slave; + + if ((!mlx4_is_master(dev)) || + !(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FSM)) + return -EPROTONOSUPPORT; + + slave = mlx4_get_slave_indx(dev, vf); + if (slave < 0) + return -EINVAL; + + port = mlx4_slaves_closest_port(dev, slave, port); + s_info = &priv->mfunc.master.vf_admin[slave].vport[port]; + s_info->spoofchk = setting; + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_set_vf_spoofchk); + +int mlx4_get_counter_stats(struct mlx4_dev *dev, int counter_index, + struct mlx4_counter *counter_stats, int reset) +{ + struct mlx4_cmd_mailbox *mailbox = NULL; + struct mlx4_counter *tmp_counter; + int err; + u32 if_stat_in_mod; + + if (!counter_stats) + return -EINVAL; + + if (counter_index == MLX4_SINK_COUNTER_INDEX(dev)) + return 0; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + memset(mailbox->buf, 0, sizeof(struct mlx4_counter)); + if_stat_in_mod = counter_index; + if (reset) + if_stat_in_mod |= MLX4_QUERY_IF_STAT_RESET; + err = mlx4_cmd_box(dev, 0, mailbox->dma, + if_stat_in_mod, 0, + MLX4_CMD_QUERY_IF_STAT, + MLX4_CMD_TIME_CLASS_C, + MLX4_CMD_NATIVE); + if (err) { + mlx4_dbg(dev, "%s: failed to read statistics for counter index %d\n", + __func__, counter_index); + goto if_stat_out; + } + tmp_counter = (struct mlx4_counter *)mailbox->buf; + counter_stats->counter_mode = tmp_counter->counter_mode; + if (counter_stats->counter_mode == 0) { + counter_stats->rx_frames = + cpu_to_be64(be64_to_cpu(counter_stats->rx_frames) + + be64_to_cpu(tmp_counter->rx_frames)); + counter_stats->tx_frames = + cpu_to_be64(be64_to_cpu(counter_stats->tx_frames) + + be64_to_cpu(tmp_counter->tx_frames)); + counter_stats->rx_bytes = + cpu_to_be64(be64_to_cpu(counter_stats->rx_bytes) + + be64_to_cpu(tmp_counter->rx_bytes)); + counter_stats->tx_bytes = + cpu_to_be64(be64_to_cpu(counter_stats->tx_bytes) + + be64_to_cpu(tmp_counter->tx_bytes)); + } + +if_stat_out: + mlx4_free_cmd_mailbox(dev, mailbox); + + return err; +} +EXPORT_SYMBOL_GPL(mlx4_get_counter_stats); + +int mlx4_vf_smi_enabled(struct mlx4_dev *dev, int slave, int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + if (slave < 1 || slave >= dev->num_slaves || + port < 1 || port > MLX4_MAX_PORTS) + return 0; + + return priv->mfunc.master.vf_oper[slave].smi_enabled[port] == + MLX4_VF_SMI_ENABLED; +} +EXPORT_SYMBOL_GPL(mlx4_vf_smi_enabled); + +int mlx4_vf_get_enable_smi_admin(struct mlx4_dev *dev, int slave, int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + if (slave == mlx4_master_func_num(dev)) + return 1; + + if (slave < 1 || slave >= dev->num_slaves || + port < 1 || port > MLX4_MAX_PORTS) + return 0; + + return priv->mfunc.master.vf_admin[slave].enable_smi[port] == + MLX4_VF_SMI_ENABLED; +} +EXPORT_SYMBOL_GPL(mlx4_vf_get_enable_smi_admin); + +int mlx4_vf_set_enable_smi_admin(struct mlx4_dev *dev, int slave, int port, + int enabled) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_active_ports actv_ports = mlx4_get_active_ports( + &priv->dev, slave); + int min_port = find_first_bit(actv_ports.ports, + priv->dev.caps.num_ports) + 1; + int max_port = min_port - 1 + + bitmap_weight(actv_ports.ports, priv->dev.caps.num_ports); + + if (slave == mlx4_master_func_num(dev)) + return 0; + + if (slave < 1 || slave >= dev->num_slaves || + port < 1 || port > MLX4_MAX_PORTS || + enabled < 0 || enabled > 1) + return -EINVAL; + + if (min_port == max_port && dev->caps.num_ports > 1) { + mlx4_info(dev, "SMI access disallowed for single ported VFs\n"); + return -EPROTONOSUPPORT; + } + + priv->mfunc.master.vf_admin[slave].enable_smi[port] = enabled; + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_vf_set_enable_smi_admin); + Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_cmd.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_cq.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_cq.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_cq.c (revision 329159) @@ -0,0 +1,363 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. + * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * Copyright (c) 2004 Voltaire, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include + +#include "mlx4.h" +#include "icm.h" + +#define MLX4_CQ_STATUS_OK ( 0 << 28) +#define MLX4_CQ_STATUS_OVERFLOW ( 9 << 28) +#define MLX4_CQ_STATUS_WRITE_FAIL (10 << 28) +#define MLX4_CQ_FLAG_CC ( 1 << 18) +#define MLX4_CQ_FLAG_OI ( 1 << 17) +#define MLX4_CQ_STATE_ARMED ( 9 << 8) +#define MLX4_CQ_STATE_ARMED_SOL ( 6 << 8) +#define MLX4_EQ_STATE_FIRED (10 << 8) + +void mlx4_cq_completion(struct mlx4_dev *dev, u32 cqn) +{ + struct mlx4_cq *cq; + + cq = radix_tree_lookup(&mlx4_priv(dev)->cq_table.tree, + cqn & (dev->caps.num_cqs - 1)); + if (!cq) { + mlx4_dbg(dev, "Completion event for bogus CQ %08x\n", cqn); + return; + } + + ++cq->arm_sn; + + cq->comp(cq); +} + +void mlx4_cq_event(struct mlx4_dev *dev, u32 cqn, int event_type) +{ + struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table; + struct mlx4_cq *cq; + + spin_lock(&cq_table->lock); + + cq = radix_tree_lookup(&cq_table->tree, cqn & (dev->caps.num_cqs - 1)); + if (cq) + atomic_inc(&cq->refcount); + + spin_unlock(&cq_table->lock); + + if (!cq) { + mlx4_warn(dev, "Async event for bogus CQ %08x\n", cqn); + return; + } + + cq->event(cq, event_type); + + if (atomic_dec_and_test(&cq->refcount)) + complete(&cq->free); +} + +static int mlx4_SW2HW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, + int cq_num) +{ + return mlx4_cmd(dev, mailbox->dma, cq_num, 0, + MLX4_CMD_SW2HW_CQ, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); +} + +static int mlx4_MODIFY_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, + int cq_num, u32 opmod) +{ + return mlx4_cmd(dev, mailbox->dma, cq_num, opmod, MLX4_CMD_MODIFY_CQ, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); +} + +static int mlx4_HW2SW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, + int cq_num) +{ + return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, + cq_num, mailbox ? 0 : 1, MLX4_CMD_HW2SW_CQ, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); +} + +int mlx4_cq_modify(struct mlx4_dev *dev, struct mlx4_cq *cq, + u16 count, u16 period) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_cq_context *cq_context; + int err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + cq_context = mailbox->buf; + cq_context->cq_max_count = cpu_to_be16(count); + cq_context->cq_period = cpu_to_be16(period); + + err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 1); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_cq_modify); + +int mlx4_cq_resize(struct mlx4_dev *dev, struct mlx4_cq *cq, + int entries, struct mlx4_mtt *mtt) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_cq_context *cq_context; + u64 mtt_addr; + int err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + cq_context = mailbox->buf; + cq_context->logsize_usrpage = cpu_to_be32(ilog2(entries) << 24); + cq_context->log_page_size = mtt->page_shift - 12; + mtt_addr = mlx4_mtt_addr(dev, mtt); + cq_context->mtt_base_addr_h = mtt_addr >> 32; + cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); + + err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 0); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_cq_resize); + +int __mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_cq_table *cq_table = &priv->cq_table; + int err; + + *cqn = mlx4_bitmap_alloc(&cq_table->bitmap); + if (*cqn == -1) + return -ENOMEM; + + err = mlx4_table_get(dev, &cq_table->table, *cqn, GFP_KERNEL); + if (err) + goto err_out; + + err = mlx4_table_get(dev, &cq_table->cmpt_table, *cqn, GFP_KERNEL); + if (err) + goto err_put; + return 0; + +err_put: + mlx4_table_put(dev, &cq_table->table, *cqn); + +err_out: + mlx4_bitmap_free(&cq_table->bitmap, *cqn, MLX4_NO_RR); + return err; +} + +static int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn) +{ + u64 out_param; + int err; + + if (mlx4_is_mfunc(dev)) { + err = mlx4_cmd_imm(dev, 0, &out_param, RES_CQ, + RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (err) + return err; + else { + *cqn = get_param_l(&out_param); + return 0; + } + } + return __mlx4_cq_alloc_icm(dev, cqn); +} + +void __mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_cq_table *cq_table = &priv->cq_table; + + mlx4_table_put(dev, &cq_table->cmpt_table, cqn); + mlx4_table_put(dev, &cq_table->table, cqn); + mlx4_bitmap_free(&cq_table->bitmap, cqn, MLX4_NO_RR); +} + +static void mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn) +{ + u64 in_param = 0; + int err; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, cqn); + err = mlx4_cmd(dev, in_param, RES_CQ, RES_OP_RESERVE_AND_MAP, + MLX4_CMD_FREE_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (err) + mlx4_warn(dev, "Failed freeing cq:%d\n", cqn); + } else + __mlx4_cq_free_icm(dev, cqn); +} + +int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, + struct mlx4_mtt *mtt, struct mlx4_uar *uar, u64 db_rec, + struct mlx4_cq *cq, unsigned vector, int collapsed, + int timestamp_en) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_cq_table *cq_table = &priv->cq_table; + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_cq_context *cq_context; + u64 mtt_addr; + int err; + + if (vector >= dev->caps.num_comp_vectors) + return -EINVAL; + + cq->vector = vector; + + err = mlx4_cq_alloc_icm(dev, &cq->cqn); + if (err) + return err; + + spin_lock_irq(&cq_table->lock); + err = radix_tree_insert(&cq_table->tree, cq->cqn, cq); + spin_unlock_irq(&cq_table->lock); + if (err) + goto err_icm; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + err = PTR_ERR(mailbox); + goto err_radix; + } + + cq_context = mailbox->buf; + cq_context->flags = cpu_to_be32(!!collapsed << 18); + if (timestamp_en) + cq_context->flags |= cpu_to_be32(1 << 19); + + cq_context->logsize_usrpage = + cpu_to_be32((ilog2(nent) << 24) | + mlx4_to_hw_uar_index(dev, uar->index)); + cq_context->comp_eqn = priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(vector)].eqn; + cq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT; + + mtt_addr = mlx4_mtt_addr(dev, mtt); + cq_context->mtt_base_addr_h = mtt_addr >> 32; + cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); + cq_context->db_rec_addr = cpu_to_be64(db_rec); + + err = mlx4_SW2HW_CQ(dev, mailbox, cq->cqn); + mlx4_free_cmd_mailbox(dev, mailbox); + if (err) + goto err_radix; + + cq->cons_index = 0; + cq->arm_sn = 1; + cq->uar = uar; + atomic_set(&cq->refcount, 1); + init_completion(&cq->free); + + cq->irq = priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(vector)].irq; + return 0; + +err_radix: + spin_lock_irq(&cq_table->lock); + radix_tree_delete(&cq_table->tree, cq->cqn); + spin_unlock_irq(&cq_table->lock); + +err_icm: + mlx4_cq_free_icm(dev, cq->cqn); + + return err; +} +EXPORT_SYMBOL_GPL(mlx4_cq_alloc); + +void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_cq_table *cq_table = &priv->cq_table; + int err; + + err = mlx4_HW2SW_CQ(dev, NULL, cq->cqn); + if (err) + mlx4_warn(dev, "HW2SW_CQ failed (%d) for CQN %06x\n", err, cq->cqn); + + synchronize_irq(priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(cq->vector)].irq); + if (priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(cq->vector)].irq != + priv->eq_table.eq[MLX4_EQ_ASYNC].irq) + synchronize_irq(priv->eq_table.eq[MLX4_EQ_ASYNC].irq); + + spin_lock_irq(&cq_table->lock); + radix_tree_delete(&cq_table->tree, cq->cqn); + spin_unlock_irq(&cq_table->lock); + + if (atomic_dec_and_test(&cq->refcount)) + complete(&cq->free); + wait_for_completion(&cq->free); + + mlx4_cq_free_icm(dev, cq->cqn); +} +EXPORT_SYMBOL_GPL(mlx4_cq_free); + +int mlx4_init_cq_table(struct mlx4_dev *dev) +{ + struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table; + int err; + + spin_lock_init(&cq_table->lock); + INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC); + if (mlx4_is_slave(dev)) + return 0; + + err = mlx4_bitmap_init(&cq_table->bitmap, dev->caps.num_cqs, + dev->caps.num_cqs - 1, dev->caps.reserved_cqs, 0); + if (err) + return err; + + return 0; +} + +void mlx4_cleanup_cq_table(struct mlx4_dev *dev) +{ + if (mlx4_is_slave(dev)) + return; + /* Nothing to do to clean up radix_tree */ + mlx4_bitmap_cleanup(&mlx4_priv(dev)->cq_table.bitmap); +} Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_cq.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_eq.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_eq.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_eq.c (revision 329159) @@ -0,0 +1,1537 @@ +/* + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include "mlx4.h" +#include "fw.h" + +enum { + MLX4_IRQNAME_SIZE = 32 +}; + +enum { + MLX4_NUM_ASYNC_EQE = 0x100, + MLX4_NUM_SPARE_EQE = 0x80, + MLX4_EQ_ENTRY_SIZE = 0x20 +}; + +#define MLX4_EQ_STATUS_OK ( 0 << 28) +#define MLX4_EQ_STATUS_WRITE_FAIL (10 << 28) +#define MLX4_EQ_OWNER_SW ( 0 << 24) +#define MLX4_EQ_OWNER_HW ( 1 << 24) +#define MLX4_EQ_FLAG_EC ( 1 << 18) +#define MLX4_EQ_FLAG_OI ( 1 << 17) +#define MLX4_EQ_STATE_ARMED ( 9 << 8) +#define MLX4_EQ_STATE_FIRED (10 << 8) +#define MLX4_EQ_STATE_ALWAYS_ARMED (11 << 8) + +#define MLX4_ASYNC_EVENT_MASK ((1ull << MLX4_EVENT_TYPE_PATH_MIG) | \ + (1ull << MLX4_EVENT_TYPE_COMM_EST) | \ + (1ull << MLX4_EVENT_TYPE_SQ_DRAINED) | \ + (1ull << MLX4_EVENT_TYPE_CQ_ERROR) | \ + (1ull << MLX4_EVENT_TYPE_WQ_CATAS_ERROR) | \ + (1ull << MLX4_EVENT_TYPE_EEC_CATAS_ERROR) | \ + (1ull << MLX4_EVENT_TYPE_PATH_MIG_FAILED) | \ + (1ull << MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \ + (1ull << MLX4_EVENT_TYPE_WQ_ACCESS_ERROR) | \ + (1ull << MLX4_EVENT_TYPE_PORT_CHANGE) | \ + (1ull << MLX4_EVENT_TYPE_ECC_DETECT) | \ + (1ull << MLX4_EVENT_TYPE_SRQ_CATAS_ERROR) | \ + (1ull << MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE) | \ + (1ull << MLX4_EVENT_TYPE_SRQ_LIMIT) | \ + (1ull << MLX4_EVENT_TYPE_CMD) | \ + (1ull << MLX4_EVENT_TYPE_OP_REQUIRED) | \ + (1ull << MLX4_EVENT_TYPE_COMM_CHANNEL) | \ + (1ull << MLX4_EVENT_TYPE_FLR_EVENT) | \ + (1ull << MLX4_EVENT_TYPE_FATAL_WARNING)) + +static u64 get_async_ev_mask(struct mlx4_dev *dev) +{ + u64 async_ev_mask = MLX4_ASYNC_EVENT_MASK; + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV) + async_ev_mask |= (1ull << MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT); + if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RECOVERABLE_ERROR_EVENT) + async_ev_mask |= (1ull << MLX4_EVENT_TYPE_RECOVERABLE_ERROR_EVENT); + + return async_ev_mask; +} + +static void eq_set_ci(struct mlx4_eq *eq, int req_not) +{ + __raw_writel((__force u32) cpu_to_be32((eq->cons_index & 0xffffff) | + req_not << 31), + eq->doorbell); + /* We still want ordering, just not swabbing, so add a barrier */ + mb(); +} + +static struct mlx4_eqe *get_eqe(struct mlx4_eq *eq, u32 entry, u8 eqe_factor, + u8 eqe_size) +{ + /* (entry & (eq->nent - 1)) gives us a cyclic array */ + unsigned long offset = (entry & (eq->nent - 1)) * eqe_size; + /* CX3 is capable of extending the EQE from 32 to 64 bytes with + * strides of 64B,128B and 256B. + * When 64B EQE is used, the first (in the lower addresses) + * 32 bytes in the 64 byte EQE are reserved and the next 32 bytes + * contain the legacy EQE information. + * In all other cases, the first 32B contains the legacy EQE info. + */ + return eq->page_list[offset / PAGE_SIZE].buf + (offset + (eqe_factor ? MLX4_EQ_ENTRY_SIZE : 0)) % PAGE_SIZE; +} + +static struct mlx4_eqe *next_eqe_sw(struct mlx4_eq *eq, u8 eqe_factor, u8 size) +{ + struct mlx4_eqe *eqe = get_eqe(eq, eq->cons_index, eqe_factor, size); + return !!(eqe->owner & 0x80) ^ !!(eq->cons_index & eq->nent) ? NULL : eqe; +} + +static struct mlx4_eqe *next_slave_event_eqe(struct mlx4_slave_event_eq *slave_eq) +{ + struct mlx4_eqe *eqe = + &slave_eq->event_eqe[slave_eq->cons & (SLAVE_EVENT_EQ_SIZE - 1)]; + return (!!(eqe->owner & 0x80) ^ + !!(slave_eq->cons & SLAVE_EVENT_EQ_SIZE)) ? + eqe : NULL; +} + +void mlx4_gen_slave_eqe(struct work_struct *work) +{ + struct mlx4_mfunc_master_ctx *master = + container_of(work, struct mlx4_mfunc_master_ctx, + slave_event_work); + struct mlx4_mfunc *mfunc = + container_of(master, struct mlx4_mfunc, master); + struct mlx4_priv *priv = container_of(mfunc, struct mlx4_priv, mfunc); + struct mlx4_dev *dev = &priv->dev; + struct mlx4_slave_event_eq *slave_eq = &mfunc->master.slave_eq; + struct mlx4_eqe *eqe; + u8 slave; + int i, phys_port, slave_port; + + for (eqe = next_slave_event_eqe(slave_eq); eqe; + eqe = next_slave_event_eqe(slave_eq)) { + slave = eqe->slave_id; + + if (eqe->type == MLX4_EVENT_TYPE_PORT_CHANGE && + eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_DOWN && + mlx4_is_bonded(dev)) { + struct mlx4_port_cap port_cap; + + if (!mlx4_QUERY_PORT(dev, 1, &port_cap) && port_cap.link_state) + goto consume; + + if (!mlx4_QUERY_PORT(dev, 2, &port_cap) && port_cap.link_state) + goto consume; + } + /* All active slaves need to receive the event */ + if (slave == ALL_SLAVES) { + for (i = 0; i <= dev->persist->num_vfs; i++) { + phys_port = 0; + if (eqe->type == MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT && + eqe->subtype == MLX4_DEV_PMC_SUBTYPE_PORT_INFO) { + phys_port = eqe->event.port_mgmt_change.port; + slave_port = mlx4_phys_to_slave_port(dev, i, phys_port); + if (slave_port < 0) /* VF doesn't have this port */ + continue; + eqe->event.port_mgmt_change.port = slave_port; + } + if (mlx4_GEN_EQE(dev, i, eqe)) + mlx4_warn(dev, "Failed to generate event for slave %d\n", + i); + if (phys_port) + eqe->event.port_mgmt_change.port = phys_port; + } + } else { + if (mlx4_GEN_EQE(dev, slave, eqe)) + mlx4_warn(dev, "Failed to generate event for slave %d\n", + slave); + } +consume: + ++slave_eq->cons; + } +} + + +static void slave_event(struct mlx4_dev *dev, u8 slave, struct mlx4_eqe *eqe) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_event_eq *slave_eq = &priv->mfunc.master.slave_eq; + struct mlx4_eqe *s_eqe; + unsigned long flags; + + spin_lock_irqsave(&slave_eq->event_lock, flags); + s_eqe = &slave_eq->event_eqe[slave_eq->prod & (SLAVE_EVENT_EQ_SIZE - 1)]; + if ((!!(s_eqe->owner & 0x80)) ^ + (!!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE))) { + mlx4_warn(dev, "Master failed to generate an EQE for slave: %d. No free EQE on slave events queue\n", + slave); + spin_unlock_irqrestore(&slave_eq->event_lock, flags); + return; + } + + memcpy(s_eqe, eqe, sizeof(struct mlx4_eqe) - 1); + s_eqe->slave_id = slave; + /* ensure all information is written before setting the ownersip bit */ + wmb(); + s_eqe->owner = !!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE) ? 0x0 : 0x80; + ++slave_eq->prod; + + queue_work(priv->mfunc.master.comm_wq, + &priv->mfunc.master.slave_event_work); + spin_unlock_irqrestore(&slave_eq->event_lock, flags); +} + +static void mlx4_slave_event(struct mlx4_dev *dev, int slave, + struct mlx4_eqe *eqe) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + if (slave < 0 || slave > dev->persist->num_vfs || + slave == dev->caps.function || + !priv->mfunc.master.slave_state[slave].active) + return; + + slave_event(dev, slave, eqe); +} + +static void mlx4_set_eq_affinity_hint(struct mlx4_priv *priv, int vec) +{ + int hint_err; + struct mlx4_dev *dev = &priv->dev; + struct mlx4_eq *eq = &priv->eq_table.eq[vec]; + + hint_err = bind_irq_to_cpu(eq->irq, eq->affinity_cpu_id); + + if (hint_err) + mlx4_warn(dev, "bind_irq_to_cpu failed, err %d\n", hint_err); +} + +int mlx4_gen_pkey_eqe(struct mlx4_dev *dev, int slave, u8 port) +{ + struct mlx4_eqe eqe; + + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *s_slave = &priv->mfunc.master.slave_state[slave]; + + if (!s_slave->active) + return 0; + + memset(&eqe, 0, sizeof eqe); + + eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT; + eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PKEY_TABLE; + eqe.event.port_mgmt_change.port = mlx4_phys_to_slave_port(dev, slave, port); + + return mlx4_GEN_EQE(dev, slave, &eqe); +} +EXPORT_SYMBOL(mlx4_gen_pkey_eqe); + +int mlx4_gen_guid_change_eqe(struct mlx4_dev *dev, int slave, u8 port) +{ + struct mlx4_eqe eqe; + + /*don't send if we don't have the that slave */ + if (dev->persist->num_vfs < slave) + return 0; + memset(&eqe, 0, sizeof eqe); + + eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT; + eqe.subtype = MLX4_DEV_PMC_SUBTYPE_GUID_INFO; + eqe.event.port_mgmt_change.port = mlx4_phys_to_slave_port(dev, slave, port); + + return mlx4_GEN_EQE(dev, slave, &eqe); +} +EXPORT_SYMBOL(mlx4_gen_guid_change_eqe); + +int mlx4_gen_port_state_change_eqe(struct mlx4_dev *dev, int slave, u8 port, + u8 port_subtype_change) +{ + struct mlx4_eqe eqe; + u8 slave_port = mlx4_phys_to_slave_port(dev, slave, port); + + /*don't send if we don't have the that slave */ + if (dev->persist->num_vfs < slave) + return 0; + memset(&eqe, 0, sizeof eqe); + + eqe.type = MLX4_EVENT_TYPE_PORT_CHANGE; + eqe.subtype = port_subtype_change; + eqe.event.port_change.port = cpu_to_be32(slave_port << 28); + + mlx4_dbg(dev, "%s: sending: %d to slave: %d on port: %d\n", __func__, + port_subtype_change, slave, port); + return mlx4_GEN_EQE(dev, slave, &eqe); +} +EXPORT_SYMBOL(mlx4_gen_port_state_change_eqe); + +enum slave_port_state mlx4_get_slave_port_state(struct mlx4_dev *dev, int slave, u8 port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *s_state = priv->mfunc.master.slave_state; + struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, slave); + + if (slave >= dev->num_slaves || port > dev->caps.num_ports || + port <= 0 || !test_bit(port - 1, actv_ports.ports)) { + pr_err("%s: Error: asking for slave:%d, port:%d\n", + __func__, slave, port); + return SLAVE_PORT_DOWN; + } + return s_state[slave].port_state[port]; +} +EXPORT_SYMBOL(mlx4_get_slave_port_state); + +static int mlx4_set_slave_port_state(struct mlx4_dev *dev, int slave, u8 port, + enum slave_port_state state) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *s_state = priv->mfunc.master.slave_state; + struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, slave); + + if (slave >= dev->num_slaves || port > dev->caps.num_ports || + port <= 0 || !test_bit(port - 1, actv_ports.ports)) { + pr_err("%s: Error: asking for slave:%d, port:%d\n", + __func__, slave, port); + return -1; + } + s_state[slave].port_state[port] = state; + + return 0; +} + +static void set_all_slave_state(struct mlx4_dev *dev, u8 port, int event) +{ + int i; + enum slave_port_gen_event gen_event; + struct mlx4_slaves_pport slaves_pport = mlx4_phys_to_slaves_pport(dev, + port); + + for (i = 0; i < dev->persist->num_vfs + 1; i++) + if (test_bit(i, slaves_pport.slaves)) + set_and_calc_slave_port_state(dev, i, port, + event, &gen_event); +} +/************************************************************************** + The function get as input the new event to that port, + and according to the prev state change the slave's port state. + The events are: + MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN, + MLX4_PORT_STATE_DEV_EVENT_PORT_UP + MLX4_PORT_STATE_IB_EVENT_GID_VALID + MLX4_PORT_STATE_IB_EVENT_GID_INVALID +***************************************************************************/ +int set_and_calc_slave_port_state(struct mlx4_dev *dev, int slave, + u8 port, int event, + enum slave_port_gen_event *gen_event) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *ctx = NULL; + unsigned long flags; + int ret = -1; + struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, slave); + enum slave_port_state cur_state = + mlx4_get_slave_port_state(dev, slave, port); + + *gen_event = SLAVE_PORT_GEN_EVENT_NONE; + + if (slave >= dev->num_slaves || port > dev->caps.num_ports || + port <= 0 || !test_bit(port - 1, actv_ports.ports)) { + pr_err("%s: Error: asking for slave:%d, port:%d\n", + __func__, slave, port); + return ret; + } + + ctx = &priv->mfunc.master.slave_state[slave]; + spin_lock_irqsave(&ctx->lock, flags); + + switch (cur_state) { + case SLAVE_PORT_DOWN: + if (MLX4_PORT_STATE_DEV_EVENT_PORT_UP == event) + mlx4_set_slave_port_state(dev, slave, port, + SLAVE_PENDING_UP); + break; + case SLAVE_PENDING_UP: + if (MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN == event) + mlx4_set_slave_port_state(dev, slave, port, + SLAVE_PORT_DOWN); + else if (MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID == event) { + mlx4_set_slave_port_state(dev, slave, port, + SLAVE_PORT_UP); + *gen_event = SLAVE_PORT_GEN_EVENT_UP; + } + break; + case SLAVE_PORT_UP: + if (MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN == event) { + mlx4_set_slave_port_state(dev, slave, port, + SLAVE_PORT_DOWN); + *gen_event = SLAVE_PORT_GEN_EVENT_DOWN; + } else if (MLX4_PORT_STATE_IB_EVENT_GID_INVALID == + event) { + mlx4_set_slave_port_state(dev, slave, port, + SLAVE_PENDING_UP); + *gen_event = SLAVE_PORT_GEN_EVENT_DOWN; + } + break; + default: + pr_err("%s: BUG!!! UNKNOWN state: slave:%d, port:%d\n", + __func__, slave, port); + goto out; + } + ret = mlx4_get_slave_port_state(dev, slave, port); + +out: + spin_unlock_irqrestore(&ctx->lock, flags); + return ret; +} + +EXPORT_SYMBOL(set_and_calc_slave_port_state); + +int mlx4_gen_slaves_port_mgt_ev(struct mlx4_dev *dev, u8 port, int attr) +{ + struct mlx4_eqe eqe; + + memset(&eqe, 0, sizeof eqe); + + eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT; + eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PORT_INFO; + eqe.event.port_mgmt_change.port = port; + eqe.event.port_mgmt_change.params.port_info.changed_attr = + cpu_to_be32((u32) attr); + + slave_event(dev, ALL_SLAVES, &eqe); + return 0; +} +EXPORT_SYMBOL(mlx4_gen_slaves_port_mgt_ev); + +void mlx4_master_handle_slave_flr(struct work_struct *work) +{ + struct mlx4_mfunc_master_ctx *master = + container_of(work, struct mlx4_mfunc_master_ctx, + slave_flr_event_work); + struct mlx4_mfunc *mfunc = + container_of(master, struct mlx4_mfunc, master); + struct mlx4_priv *priv = + container_of(mfunc, struct mlx4_priv, mfunc); + struct mlx4_dev *dev = &priv->dev; + struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state; + int i; + int err; + unsigned long flags; + + mlx4_dbg(dev, "mlx4_handle_slave_flr\n"); + + for (i = 0 ; i < dev->num_slaves; i++) { + + if (MLX4_COMM_CMD_FLR == slave_state[i].last_cmd) { + mlx4_dbg(dev, "mlx4_handle_slave_flr: clean slave: %d\n", + i); + /* In case of 'Reset flow' FLR can be generated for + * a slave before mlx4_load_one is done. + * make sure interface is up before trying to delete + * slave resources which weren't allocated yet. + */ + if (dev->persist->interface_state & + MLX4_INTERFACE_STATE_UP) + mlx4_delete_all_resources_for_slave(dev, i); + /*return the slave to running mode*/ + spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags); + slave_state[i].last_cmd = MLX4_COMM_CMD_RESET; + slave_state[i].is_slave_going_down = 0; + spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags); + /*notify the FW:*/ + err = mlx4_cmd(dev, 0, i, 0, MLX4_CMD_INFORM_FLR_DONE, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (err) + mlx4_warn(dev, "Failed to notify FW on FLR done (slave:%d)\n", + i); + } + } +} + +static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_eqe *eqe; + int cqn = -1; + int eqes_found = 0; + int set_ci = 0; + int port; + int slave = 0; + int ret; + u32 flr_slave; + u8 update_slave_state; + int i; + enum slave_port_gen_event gen_event; + unsigned long flags; + struct mlx4_vport_state *s_info; + int eqe_size = dev->caps.eqe_size; + + while ((eqe = next_eqe_sw(eq, dev->caps.eqe_factor, eqe_size))) { + /* + * Make sure we read EQ entry contents after we've + * checked the ownership bit. + */ + rmb(); + + switch (eqe->type) { + case MLX4_EVENT_TYPE_COMP: + cqn = be32_to_cpu(eqe->event.comp.cqn) & 0xffffff; + mlx4_cq_completion(dev, cqn); + break; + + case MLX4_EVENT_TYPE_PATH_MIG: + case MLX4_EVENT_TYPE_COMM_EST: + case MLX4_EVENT_TYPE_SQ_DRAINED: + case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE: + case MLX4_EVENT_TYPE_WQ_CATAS_ERROR: + case MLX4_EVENT_TYPE_PATH_MIG_FAILED: + case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR: + mlx4_dbg(dev, "event %d arrived\n", eqe->type); + if (mlx4_is_master(dev)) { + /* forward only to slave owning the QP */ + ret = mlx4_get_slave_from_resource_id(dev, + RES_QP, + be32_to_cpu(eqe->event.qp.qpn) + & 0xffffff, &slave); + if (ret && ret != -ENOENT) { + mlx4_dbg(dev, "QP event %02x(%02x) on EQ %d at index %u: could not get slave id (%d)\n", + eqe->type, eqe->subtype, + eq->eqn, eq->cons_index, ret); + break; + } + + if (!ret && slave != dev->caps.function) { + mlx4_slave_event(dev, slave, eqe); + break; + } + + } + mlx4_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & + 0xffffff, eqe->type); + break; + + case MLX4_EVENT_TYPE_SRQ_LIMIT: + mlx4_dbg(dev, "%s: MLX4_EVENT_TYPE_SRQ_LIMIT\n", + __func__); + case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR: + if (mlx4_is_master(dev)) { + /* forward only to slave owning the SRQ */ + ret = mlx4_get_slave_from_resource_id(dev, + RES_SRQ, + be32_to_cpu(eqe->event.srq.srqn) + & 0xffffff, + &slave); + if (ret && ret != -ENOENT) { + mlx4_warn(dev, "SRQ event %02x(%02x) on EQ %d at index %u: could not get slave id (%d)\n", + eqe->type, eqe->subtype, + eq->eqn, eq->cons_index, ret); + break; + } + mlx4_warn(dev, "%s: slave:%d, srq_no:0x%x, event: %02x(%02x)\n", + __func__, slave, + be32_to_cpu(eqe->event.srq.srqn), + eqe->type, eqe->subtype); + + if (!ret && slave != dev->caps.function) { + mlx4_warn(dev, "%s: sending event %02x(%02x) to slave:%d\n", + __func__, eqe->type, + eqe->subtype, slave); + mlx4_slave_event(dev, slave, eqe); + break; + } + } + mlx4_srq_event(dev, be32_to_cpu(eqe->event.srq.srqn) & + 0xffffff, eqe->type); + break; + + case MLX4_EVENT_TYPE_CMD: + mlx4_cmd_event(dev, + be16_to_cpu(eqe->event.cmd.token), + eqe->event.cmd.status, + be64_to_cpu(eqe->event.cmd.out_param)); + break; + + case MLX4_EVENT_TYPE_PORT_CHANGE: { + struct mlx4_slaves_pport slaves_port; + port = be32_to_cpu(eqe->event.port_change.port) >> 28; + slaves_port = mlx4_phys_to_slaves_pport(dev, port); + if (eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_DOWN) { + mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_DOWN, + port); + mlx4_priv(dev)->sense.do_sense_port[port] = 1; + if (!mlx4_is_master(dev)) + break; + for (i = 0; i < dev->persist->num_vfs + 1; + i++) { + int reported_port = mlx4_is_bonded(dev) ? 1 : mlx4_phys_to_slave_port(dev, i, port); + + if (!test_bit(i, slaves_port.slaves) && !mlx4_is_bonded(dev)) + continue; + if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) { + if (i == mlx4_master_func_num(dev)) + continue; + mlx4_dbg(dev, "%s: Sending MLX4_PORT_CHANGE_SUBTYPE_DOWN to slave: %d, port:%d\n", + __func__, i, port); + s_info = &priv->mfunc.master.vf_oper[i].vport[port].state; + if (0 /*IFLA_VF_LINK_STATE_AUTO == s_info->link_state*/) { + eqe->event.port_change.port = + cpu_to_be32( + (be32_to_cpu(eqe->event.port_change.port) & 0xFFFFFFF) + | (reported_port << 28)); + mlx4_slave_event(dev, i, eqe); + } + } else { /* IB port */ + set_and_calc_slave_port_state(dev, i, port, + MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN, + &gen_event); + /*we can be in pending state, then do not send port_down event*/ + if (SLAVE_PORT_GEN_EVENT_DOWN == gen_event) { + if (i == mlx4_master_func_num(dev)) + continue; + eqe->event.port_change.port = + cpu_to_be32( + (be32_to_cpu(eqe->event.port_change.port) & 0xFFFFFFF) + | (mlx4_phys_to_slave_port(dev, i, port) << 28)); + mlx4_slave_event(dev, i, eqe); + } + } + } + } else { + mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_UP, port); + + mlx4_priv(dev)->sense.do_sense_port[port] = 0; + + if (!mlx4_is_master(dev)) + break; + if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) + for (i = 0; + i < dev->persist->num_vfs + 1; + i++) { + int reported_port = mlx4_is_bonded(dev) ? 1 : mlx4_phys_to_slave_port(dev, i, port); + + if (!test_bit(i, slaves_port.slaves) && !mlx4_is_bonded(dev)) + continue; + if (i == mlx4_master_func_num(dev)) + continue; + s_info = &priv->mfunc.master.vf_oper[i].vport[port].state; + if (0 /*IFLA_VF_LINK_STATE_AUTO == s_info->link_state*/) { + eqe->event.port_change.port = + cpu_to_be32( + (be32_to_cpu(eqe->event.port_change.port) & 0xFFFFFFF) + | (reported_port << 28)); + mlx4_slave_event(dev, i, eqe); + } + } + else /* IB port */ + /* port-up event will be sent to a slave when the + * slave's alias-guid is set. This is done in alias_GUID.c + */ + set_all_slave_state(dev, port, MLX4_DEV_EVENT_PORT_UP); + } + break; + } + + case MLX4_EVENT_TYPE_CQ_ERROR: + mlx4_warn(dev, "CQ %s on CQN %06x\n", + eqe->event.cq_err.syndrome == 1 ? + "overrun" : "access violation", + be32_to_cpu(eqe->event.cq_err.cqn) & 0xffffff); + if (mlx4_is_master(dev)) { + ret = mlx4_get_slave_from_resource_id(dev, + RES_CQ, + be32_to_cpu(eqe->event.cq_err.cqn) + & 0xffffff, &slave); + if (ret && ret != -ENOENT) { + mlx4_dbg(dev, "CQ event %02x(%02x) on EQ %d at index %u: could not get slave id (%d)\n", + eqe->type, eqe->subtype, + eq->eqn, eq->cons_index, ret); + break; + } + + if (!ret && slave != dev->caps.function) { + mlx4_slave_event(dev, slave, eqe); + break; + } + } + mlx4_cq_event(dev, + be32_to_cpu(eqe->event.cq_err.cqn) + & 0xffffff, + eqe->type); + break; + + case MLX4_EVENT_TYPE_EQ_OVERFLOW: + mlx4_warn(dev, "EQ overrun on EQN %d\n", eq->eqn); + break; + + case MLX4_EVENT_TYPE_OP_REQUIRED: + atomic_inc(&priv->opreq_count); + /* FW commands can't be executed from interrupt context + * working in deferred task + */ + queue_work(mlx4_wq, &priv->opreq_task); + break; + + case MLX4_EVENT_TYPE_COMM_CHANNEL: + if (!mlx4_is_master(dev)) { + mlx4_warn(dev, "Received comm channel event for non master device\n"); + break; + } + memcpy(&priv->mfunc.master.comm_arm_bit_vector, + eqe->event.comm_channel_arm.bit_vec, + sizeof eqe->event.comm_channel_arm.bit_vec); + queue_work(priv->mfunc.master.comm_wq, + &priv->mfunc.master.comm_work); + break; + + case MLX4_EVENT_TYPE_FLR_EVENT: + flr_slave = be32_to_cpu(eqe->event.flr_event.slave_id); + if (!mlx4_is_master(dev)) { + mlx4_warn(dev, "Non-master function received FLR event\n"); + break; + } + + mlx4_dbg(dev, "FLR event for slave: %d\n", flr_slave); + + if (flr_slave >= dev->num_slaves) { + mlx4_warn(dev, + "Got FLR for unknown function: %d\n", + flr_slave); + update_slave_state = 0; + } else + update_slave_state = 1; + + spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags); + if (update_slave_state) { + priv->mfunc.master.slave_state[flr_slave].active = false; + priv->mfunc.master.slave_state[flr_slave].last_cmd = MLX4_COMM_CMD_FLR; + priv->mfunc.master.slave_state[flr_slave].is_slave_going_down = 1; + } + spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags); + mlx4_dispatch_event(dev, MLX4_DEV_EVENT_SLAVE_SHUTDOWN, + flr_slave); + queue_work(priv->mfunc.master.comm_wq, + &priv->mfunc.master.slave_flr_event_work); + break; + + case MLX4_EVENT_TYPE_FATAL_WARNING: + if (eqe->subtype == MLX4_FATAL_WARNING_SUBTYPE_WARMING) { + if (mlx4_is_master(dev)) + for (i = 0; i < dev->num_slaves; i++) { + mlx4_dbg(dev, "%s: Sending MLX4_FATAL_WARNING_SUBTYPE_WARMING to slave: %d\n", + __func__, i); + if (i == dev->caps.function) + continue; + mlx4_slave_event(dev, i, eqe); + } + mlx4_err(dev, "Temperature Threshold was reached! Threshold: %d celsius degrees; Current Temperature: %d\n", + be16_to_cpu(eqe->event.warming.warning_threshold), + be16_to_cpu(eqe->event.warming.current_temperature)); + } else + mlx4_warn(dev, "Unhandled event FATAL WARNING (%02x), subtype %02x on EQ %d at index %u. owner=%x, nent=0x%x, slave=%x, ownership=%s\n", + eqe->type, eqe->subtype, eq->eqn, + eq->cons_index, eqe->owner, eq->nent, + eqe->slave_id, + !!(eqe->owner & 0x80) ^ + !!(eq->cons_index & eq->nent) ? "HW" : "SW"); + + break; + + case MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT: + mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_MGMT_CHANGE, + (unsigned long) eqe); + break; + + case MLX4_EVENT_TYPE_RECOVERABLE_ERROR_EVENT: + switch (eqe->subtype) { + case MLX4_RECOVERABLE_ERROR_EVENT_SUBTYPE_BAD_CABLE: + mlx4_warn(dev, "Bad cable detected on port %u\n", + eqe->event.bad_cable.port); + break; + case MLX4_RECOVERABLE_ERROR_EVENT_SUBTYPE_UNSUPPORTED_CABLE: + mlx4_warn(dev, "Unsupported cable detected\n"); + break; + default: + mlx4_dbg(dev, + "Unhandled recoverable error event detected: %02x(%02x) on EQ %d at index %u. owner=%x, nent=0x%x, ownership=%s\n", + eqe->type, eqe->subtype, eq->eqn, + eq->cons_index, eqe->owner, eq->nent, + !!(eqe->owner & 0x80) ^ + !!(eq->cons_index & eq->nent) ? "HW" : "SW"); + break; + } + break; + + case MLX4_EVENT_TYPE_EEC_CATAS_ERROR: + case MLX4_EVENT_TYPE_ECC_DETECT: + default: + mlx4_warn(dev, "Unhandled event %02x(%02x) on EQ %d at index %u. owner=%x, nent=0x%x, slave=%x, ownership=%s\n", + eqe->type, eqe->subtype, eq->eqn, + eq->cons_index, eqe->owner, eq->nent, + eqe->slave_id, + !!(eqe->owner & 0x80) ^ + !!(eq->cons_index & eq->nent) ? "HW" : "SW"); + break; + } + + ++eq->cons_index; + eqes_found = 1; + ++set_ci; + + /* + * The HCA will think the queue has overflowed if we + * don't tell it we've been processing events. We + * create our EQs with MLX4_NUM_SPARE_EQE extra + * entries, so we must update our consumer index at + * least that often. + */ + if (unlikely(set_ci >= MLX4_NUM_SPARE_EQE)) { + eq_set_ci(eq, 0); + set_ci = 0; + } + } + + eq_set_ci(eq, 1); + + return eqes_found; +} + +static irqreturn_t mlx4_interrupt(int irq, void *dev_ptr) +{ + struct mlx4_dev *dev = dev_ptr; + struct mlx4_priv *priv = mlx4_priv(dev); + int work = 0; + int i; + + writel(priv->eq_table.clr_mask, priv->eq_table.clr_int); + + for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) + work |= mlx4_eq_int(dev, &priv->eq_table.eq[i]); + + return IRQ_RETVAL(work); +} + +static irqreturn_t mlx4_msi_x_interrupt(int irq, void *eq_ptr) +{ + struct mlx4_eq *eq = eq_ptr; + struct mlx4_dev *dev = eq->dev; + + mlx4_eq_int(dev, eq); + + /* MSI-X vectors always belong to us */ + return IRQ_HANDLED; +} + +int mlx4_MAP_EQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_event_eq_info *event_eq = + priv->mfunc.master.slave_state[slave].event_eq; + u32 in_modifier = vhcr->in_modifier; + u32 eqn = in_modifier & 0x3FF; + u64 in_param = vhcr->in_param; + int err = 0; + int i; + + if (slave == dev->caps.function) + err = mlx4_cmd(dev, in_param, (in_modifier & 0x80000000) | eqn, + 0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); + if (!err) + for (i = 0; i < MLX4_EVENT_TYPES_NUM; ++i) + if (in_param & (1LL << i)) + event_eq[i].eqn = in_modifier >> 31 ? -1 : eqn; + + return err; +} + +static int mlx4_MAP_EQ(struct mlx4_dev *dev, u64 event_mask, int unmap, + int eq_num) +{ + return mlx4_cmd(dev, event_mask, (unmap << 31) | eq_num, + 0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); +} + +static int mlx4_SW2HW_EQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, + int eq_num) +{ + return mlx4_cmd(dev, mailbox->dma, eq_num, 0, + MLX4_CMD_SW2HW_EQ, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); +} + +static int mlx4_HW2SW_EQ(struct mlx4_dev *dev, int eq_num) +{ + return mlx4_cmd(dev, 0, eq_num, 1, MLX4_CMD_HW2SW_EQ, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); +} + +static int mlx4_num_eq_uar(struct mlx4_dev *dev) +{ + /* + * Each UAR holds 4 EQ doorbells. To figure out how many UARs + * we need to map, take the difference of highest index and + * the lowest index we'll use and add 1. + */ + return (dev->caps.num_comp_vectors + 1 + dev->caps.reserved_eqs) / 4 - + dev->caps.reserved_eqs / 4 + 1; +} + +static void __iomem *mlx4_get_eq_uar(struct mlx4_dev *dev, struct mlx4_eq *eq) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int index; + + index = eq->eqn / 4 - dev->caps.reserved_eqs / 4; + + if (!priv->eq_table.uar_map[index]) { + priv->eq_table.uar_map[index] = + ioremap(pci_resource_start(dev->persist->pdev, 2) + + ((eq->eqn / 4) << (dev->uar_page_shift)), + (1 << (dev->uar_page_shift))); + if (!priv->eq_table.uar_map[index]) { + mlx4_err(dev, "Couldn't map EQ doorbell for EQN 0x%06x\n", + eq->eqn); + return NULL; + } + } + + return priv->eq_table.uar_map[index] + 0x800 + 8 * (eq->eqn % 4); +} + +static void mlx4_unmap_uar(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int i; + + for (i = 0; i < mlx4_num_eq_uar(dev); ++i) + if (priv->eq_table.uar_map[i]) { + iounmap(priv->eq_table.uar_map[i]); + priv->eq_table.uar_map[i] = NULL; + } +} + +static int mlx4_create_eq(struct mlx4_dev *dev, int nent, + u8 intr, struct mlx4_eq *eq) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_eq_context *eq_context; + int npages; + u64 *dma_list = NULL; + dma_addr_t t; + u64 mtt_addr; + int err = -ENOMEM; + int i; + + eq->dev = dev; + eq->nent = roundup_pow_of_two(max(nent, 2)); + /* CX3 is capable of extending the CQE/EQE from 32 to 64 bytes, with + * strides of 64B,128B and 256B. + */ + npages = PAGE_ALIGN(eq->nent * dev->caps.eqe_size) / PAGE_SIZE; + + eq->page_list = kmalloc(npages * sizeof *eq->page_list, + GFP_KERNEL); + if (!eq->page_list) + goto err_out; + + for (i = 0; i < npages; ++i) + eq->page_list[i].buf = NULL; + + dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL); + if (!dma_list) + goto err_out_free; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + goto err_out_free; + eq_context = mailbox->buf; + + for (i = 0; i < npages; ++i) { + eq->page_list[i].buf = dma_alloc_coherent(&dev->persist-> + pdev->dev, + PAGE_SIZE, &t, + GFP_KERNEL); + if (!eq->page_list[i].buf) + goto err_out_free_pages; + + dma_list[i] = t; + eq->page_list[i].map = t; + + memset(eq->page_list[i].buf, 0, PAGE_SIZE); + } + + eq->eqn = mlx4_bitmap_alloc(&priv->eq_table.bitmap); + if (eq->eqn == -1) + goto err_out_free_pages; + + eq->doorbell = mlx4_get_eq_uar(dev, eq); + if (!eq->doorbell) { + err = -ENOMEM; + goto err_out_free_eq; + } + + err = mlx4_mtt_init(dev, npages, PAGE_SHIFT, &eq->mtt); + if (err) + goto err_out_free_eq; + + err = mlx4_write_mtt(dev, &eq->mtt, 0, npages, dma_list); + if (err) + goto err_out_free_mtt; + + eq_context->flags = cpu_to_be32(MLX4_EQ_STATUS_OK | + MLX4_EQ_STATE_ARMED); + eq_context->log_eq_size = ilog2(eq->nent); + eq_context->intr = intr; + eq_context->log_page_size = PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT; + + mtt_addr = mlx4_mtt_addr(dev, &eq->mtt); + eq_context->mtt_base_addr_h = mtt_addr >> 32; + eq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); + + err = mlx4_SW2HW_EQ(dev, mailbox, eq->eqn); + if (err) { + mlx4_warn(dev, "SW2HW_EQ failed (%d)\n", err); + goto err_out_free_mtt; + } + + kfree(dma_list); + mlx4_free_cmd_mailbox(dev, mailbox); + + eq->cons_index = 0; + + return err; + +err_out_free_mtt: + mlx4_mtt_cleanup(dev, &eq->mtt); + +err_out_free_eq: + mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn, MLX4_USE_RR); + +err_out_free_pages: + for (i = 0; i < npages; ++i) + if (eq->page_list[i].buf) + dma_free_coherent(&dev->persist->pdev->dev, PAGE_SIZE, + eq->page_list[i].buf, + eq->page_list[i].map); + + mlx4_free_cmd_mailbox(dev, mailbox); + +err_out_free: + kfree(eq->page_list); + kfree(dma_list); + +err_out: + return err; +} + +static void mlx4_free_eq(struct mlx4_dev *dev, + struct mlx4_eq *eq) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int err; + int i; + /* CX3 is capable of extending the CQE/EQE from 32 to 64 bytes, with + * strides of 64B,128B and 256B + */ + int npages = PAGE_ALIGN(dev->caps.eqe_size * eq->nent) / PAGE_SIZE; + + err = mlx4_HW2SW_EQ(dev, eq->eqn); + if (err) + mlx4_warn(dev, "HW2SW_EQ failed (%d)\n", err); + + synchronize_irq(eq->irq); + + mlx4_mtt_cleanup(dev, &eq->mtt); + for (i = 0; i < npages; ++i) + dma_free_coherent(&dev->persist->pdev->dev, PAGE_SIZE, + eq->page_list[i].buf, + eq->page_list[i].map); + + kfree(eq->page_list); + mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn, MLX4_USE_RR); +} + +static void mlx4_free_irqs(struct mlx4_dev *dev) +{ + struct mlx4_eq_table *eq_table = &mlx4_priv(dev)->eq_table; + int i; + + if (eq_table->have_irq) + free_irq(dev->persist->pdev->irq, dev); + + for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) + if (eq_table->eq[i].have_irq) { + eq_table->eq[i].affinity_cpu_id = NOCPU; + free_irq(eq_table->eq[i].irq, eq_table->eq + i); + eq_table->eq[i].have_irq = 0; + } + + kfree(eq_table->irq_names); +} + +static int mlx4_map_clr_int(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + priv->clr_base = ioremap(pci_resource_start(dev->persist->pdev, + priv->fw.clr_int_bar) + + priv->fw.clr_int_base, MLX4_CLR_INT_SIZE); + if (!priv->clr_base) { + mlx4_err(dev, "Couldn't map interrupt clear register, aborting\n"); + return -ENOMEM; + } + + return 0; +} + +static void mlx4_unmap_clr_int(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + iounmap(priv->clr_base); +} + +int mlx4_alloc_eq_table(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + priv->eq_table.eq = kcalloc(dev->caps.num_eqs - dev->caps.reserved_eqs, + sizeof *priv->eq_table.eq, GFP_KERNEL); + if (!priv->eq_table.eq) + return -ENOMEM; + + return 0; +} + +void mlx4_free_eq_table(struct mlx4_dev *dev) +{ + kfree(mlx4_priv(dev)->eq_table.eq); +} + +int mlx4_init_eq_table(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int err; + int i; + + priv->eq_table.uar_map = kcalloc(mlx4_num_eq_uar(dev), + sizeof *priv->eq_table.uar_map, + GFP_KERNEL); + if (!priv->eq_table.uar_map) { + err = -ENOMEM; + goto err_out_free; + } + + err = mlx4_bitmap_init(&priv->eq_table.bitmap, + roundup_pow_of_two(dev->caps.num_eqs), + dev->caps.num_eqs - 1, + dev->caps.reserved_eqs, + roundup_pow_of_two(dev->caps.num_eqs) - + dev->caps.num_eqs); + if (err) + goto err_out_free; + + for (i = 0; i < mlx4_num_eq_uar(dev); ++i) + priv->eq_table.uar_map[i] = NULL; + + if (!mlx4_is_slave(dev)) { + err = mlx4_map_clr_int(dev); + if (err) + goto err_out_bitmap; + + priv->eq_table.clr_mask = + swab32(1 << (priv->eq_table.inta_pin & 31)); + priv->eq_table.clr_int = priv->clr_base + + (priv->eq_table.inta_pin < 32 ? 4 : 0); + } + + priv->eq_table.irq_names = + kmalloc(MLX4_IRQNAME_SIZE * (dev->caps.num_comp_vectors + 1), + GFP_KERNEL); + if (!priv->eq_table.irq_names) { + err = -ENOMEM; + goto err_out_clr_int; + } + + for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) { + if (i == MLX4_EQ_ASYNC) { + err = mlx4_create_eq(dev, + MLX4_NUM_ASYNC_EQE + MLX4_NUM_SPARE_EQE, + 0, &priv->eq_table.eq[MLX4_EQ_ASYNC]); + } else { + struct mlx4_eq *eq = &priv->eq_table.eq[i]; +#ifdef CONFIG_RFS_ACCEL + int port = find_first_bit(eq->actv_ports.ports, + dev->caps.num_ports) + 1; + + if (port <= dev->caps.num_ports) { + struct mlx4_port_info *info = + &mlx4_priv(dev)->port[port]; + + if (!info->rmap) { + info->rmap = alloc_irq_cpu_rmap( + mlx4_get_eqs_per_port(dev, port)); + if (!info->rmap) { + mlx4_warn(dev, "Failed to allocate cpu rmap\n"); + err = -ENOMEM; + goto err_out_unmap; + } + } + + err = irq_cpu_rmap_add( + info->rmap, eq->irq); + if (err) + mlx4_warn(dev, "Failed adding irq rmap\n"); + } +#endif + err = mlx4_create_eq(dev, dev->quotas.cq + + MLX4_NUM_SPARE_EQE, + (dev->flags & MLX4_FLAG_MSI_X) ? + i + 1 - !!(i > MLX4_EQ_ASYNC) : 0, + eq); + } + if (err) + goto err_out_unmap; + } + + if (dev->flags & MLX4_FLAG_MSI_X) { + const char *eq_name; + + snprintf(priv->eq_table.irq_names + + MLX4_EQ_ASYNC * MLX4_IRQNAME_SIZE, + MLX4_IRQNAME_SIZE, + "mlx4-async@pci:%s", + pci_name(dev->persist->pdev)); + eq_name = priv->eq_table.irq_names + + MLX4_EQ_ASYNC * MLX4_IRQNAME_SIZE; + + err = request_irq(priv->eq_table.eq[MLX4_EQ_ASYNC].irq, + mlx4_msi_x_interrupt, 0, eq_name, + priv->eq_table.eq + MLX4_EQ_ASYNC); + if (err) + goto err_out_unmap; + + priv->eq_table.eq[MLX4_EQ_ASYNC].have_irq = 1; + } else { + snprintf(priv->eq_table.irq_names, + MLX4_IRQNAME_SIZE, + DRV_NAME "@pci:%s", + pci_name(dev->persist->pdev)); + err = request_irq(dev->persist->pdev->irq, mlx4_interrupt, + IRQF_SHARED, priv->eq_table.irq_names, dev); + if (err) + goto err_out_unmap; + + priv->eq_table.have_irq = 1; + } + + err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, + priv->eq_table.eq[MLX4_EQ_ASYNC].eqn); + if (err) + mlx4_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n", + priv->eq_table.eq[MLX4_EQ_ASYNC].eqn, err); + + /* arm ASYNC eq */ + eq_set_ci(&priv->eq_table.eq[MLX4_EQ_ASYNC], 1); + + return 0; + +err_out_unmap: + while (i > 0) + mlx4_free_eq(dev, &priv->eq_table.eq[--i]); +#ifdef CONFIG_RFS_ACCEL + for (i = 1; i <= dev->caps.num_ports; i++) { + if (mlx4_priv(dev)->port[i].rmap) { + free_irq_cpu_rmap(mlx4_priv(dev)->port[i].rmap); + mlx4_priv(dev)->port[i].rmap = NULL; + } + } +#endif + mlx4_free_irqs(dev); + +err_out_clr_int: + if (!mlx4_is_slave(dev)) + mlx4_unmap_clr_int(dev); + +err_out_bitmap: + mlx4_unmap_uar(dev); + mlx4_bitmap_cleanup(&priv->eq_table.bitmap); + +err_out_free: + kfree(priv->eq_table.uar_map); + + return err; +} + +void mlx4_cleanup_eq_table(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int i; + + mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 1, + priv->eq_table.eq[MLX4_EQ_ASYNC].eqn); + +#ifdef CONFIG_RFS_ACCEL + for (i = 1; i <= dev->caps.num_ports; i++) { + if (mlx4_priv(dev)->port[i].rmap) { + free_irq_cpu_rmap(mlx4_priv(dev)->port[i].rmap); + mlx4_priv(dev)->port[i].rmap = NULL; + } + } +#endif + mlx4_free_irqs(dev); + + for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) + mlx4_free_eq(dev, &priv->eq_table.eq[i]); + + if (!mlx4_is_slave(dev)) + mlx4_unmap_clr_int(dev); + + mlx4_unmap_uar(dev); + mlx4_bitmap_cleanup(&priv->eq_table.bitmap); + + kfree(priv->eq_table.uar_map); +} + +/* A test that verifies that we can accept interrupts + * on the vector allocated for asynchronous events + */ +int mlx4_test_async(struct mlx4_dev *dev) +{ + return mlx4_NOP(dev); +} +EXPORT_SYMBOL(mlx4_test_async); + +/* A test that verifies that we can accept interrupts + * on the given irq vector of the tested port. + * Interrupts are checked using the NOP command. + */ +int mlx4_test_interrupt(struct mlx4_dev *dev, int vector) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int err; + + /* Temporary use polling for command completions */ + mlx4_cmd_use_polling(dev); + + /* Map the new eq to handle all asynchronous events */ + err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, + priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(vector)].eqn); + if (err) { + mlx4_warn(dev, "Failed mapping eq for interrupt test\n"); + goto out; + } + + /* Go back to using events */ + mlx4_cmd_use_events(dev); + err = mlx4_NOP(dev); + + /* Return to default */ + mlx4_cmd_use_polling(dev); +out: + mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, + priv->eq_table.eq[MLX4_EQ_ASYNC].eqn); + mlx4_cmd_use_events(dev); + + return err; +} +EXPORT_SYMBOL(mlx4_test_interrupt); + +bool mlx4_is_eq_vector_valid(struct mlx4_dev *dev, u8 port, int vector) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + vector = MLX4_CQ_TO_EQ_VECTOR(vector); + if (vector < 0 || (vector >= dev->caps.num_comp_vectors + 1) || + (vector == MLX4_EQ_ASYNC)) + return false; + + return test_bit(port - 1, priv->eq_table.eq[vector].actv_ports.ports); +} +EXPORT_SYMBOL(mlx4_is_eq_vector_valid); + +u32 mlx4_get_eqs_per_port(struct mlx4_dev *dev, u8 port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + unsigned int i; + unsigned int sum = 0; + + for (i = 0; i < dev->caps.num_comp_vectors + 1; i++) + sum += !!test_bit(port - 1, + priv->eq_table.eq[i].actv_ports.ports); + + return sum; +} +EXPORT_SYMBOL(mlx4_get_eqs_per_port); + +int mlx4_is_eq_shared(struct mlx4_dev *dev, int vector) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + vector = MLX4_CQ_TO_EQ_VECTOR(vector); + if (vector <= 0 || (vector >= dev->caps.num_comp_vectors + 1)) + return -EINVAL; + + return !!(bitmap_weight(priv->eq_table.eq[vector].actv_ports.ports, + dev->caps.num_ports) > 1); +} +EXPORT_SYMBOL(mlx4_is_eq_shared); + +int mlx4_assign_eq(struct mlx4_dev *dev, u8 port, int *vector) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int err = 0, i = 0; + u32 min_ref_count_val = (u32)-1; + int requested_vector = MLX4_CQ_TO_EQ_VECTOR(*vector); + int *prequested_vector = NULL; + + + mutex_lock(&priv->msix_ctl.pool_lock); + if (requested_vector < (dev->caps.num_comp_vectors + 1) && + (requested_vector >= 0) && + (requested_vector != MLX4_EQ_ASYNC)) { + if (test_bit(port - 1, + priv->eq_table.eq[requested_vector].actv_ports.ports)) { + prequested_vector = &requested_vector; + } else { + struct mlx4_eq *eq; + + for (i = 1; i < port; + requested_vector += mlx4_get_eqs_per_port(dev, i++)) + ; + + eq = &priv->eq_table.eq[requested_vector]; + if (requested_vector < dev->caps.num_comp_vectors + 1 && + test_bit(port - 1, eq->actv_ports.ports)) { + prequested_vector = &requested_vector; + } + } + } + + if (!prequested_vector) { + requested_vector = -1; + for (i = 0; min_ref_count_val && i < dev->caps.num_comp_vectors + 1; + i++) { + struct mlx4_eq *eq = &priv->eq_table.eq[i]; + + if (min_ref_count_val > eq->ref_count && + test_bit(port - 1, eq->actv_ports.ports)) { + min_ref_count_val = eq->ref_count; + requested_vector = i; + } + } + + if (requested_vector < 0) { + err = -ENOSPC; + goto err_unlock; + } + + prequested_vector = &requested_vector; + } + + if (!test_bit(*prequested_vector, priv->msix_ctl.pool_bm) && + dev->flags & MLX4_FLAG_MSI_X) { + set_bit(*prequested_vector, priv->msix_ctl.pool_bm); + snprintf(priv->eq_table.irq_names + + *prequested_vector * MLX4_IRQNAME_SIZE, + MLX4_IRQNAME_SIZE, "mlx4-%d@%s", + *prequested_vector, dev_name(&dev->persist->pdev->dev)); + + err = request_irq(priv->eq_table.eq[*prequested_vector].irq, + mlx4_msi_x_interrupt, 0, + &priv->eq_table.irq_names[*prequested_vector << 5], + priv->eq_table.eq + *prequested_vector); + + if (err) { + clear_bit(*prequested_vector, priv->msix_ctl.pool_bm); + *prequested_vector = -1; + } else { + mlx4_set_eq_affinity_hint(priv, *prequested_vector); + eq_set_ci(&priv->eq_table.eq[*prequested_vector], 1); + priv->eq_table.eq[*prequested_vector].have_irq = 1; + } + } + + if (!err && *prequested_vector >= 0) + priv->eq_table.eq[*prequested_vector].ref_count++; + +err_unlock: + mutex_unlock(&priv->msix_ctl.pool_lock); + + if (!err && *prequested_vector >= 0) + *vector = MLX4_EQ_TO_CQ_VECTOR(*prequested_vector); + else + *vector = 0; + + return err; +} +EXPORT_SYMBOL(mlx4_assign_eq); + +int mlx4_eq_get_irq(struct mlx4_dev *dev, int cq_vec) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + return priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(cq_vec)].irq; +} +EXPORT_SYMBOL(mlx4_eq_get_irq); + +void mlx4_release_eq(struct mlx4_dev *dev, int vec) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int eq_vec = MLX4_CQ_TO_EQ_VECTOR(vec); + + mutex_lock(&priv->msix_ctl.pool_lock); + priv->eq_table.eq[eq_vec].ref_count--; + + /* once we allocated EQ, we don't release it because it might be binded + * to cpu_rmap. + */ + mutex_unlock(&priv->msix_ctl.pool_lock); +} +EXPORT_SYMBOL(mlx4_release_eq); + Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_eq.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_fw.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_fw.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_fw.c (revision 329159) @@ -0,0 +1,3053 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#define LINUXKPI_PARAM_PREFIX mlx4_ + +#include +#include +#include +#include + +#include + +#include "fw.h" +#include "icm.h" + +enum { + MLX4_COMMAND_INTERFACE_MIN_REV = 2, + MLX4_COMMAND_INTERFACE_MAX_REV = 3, + MLX4_COMMAND_INTERFACE_NEW_PORT_CMDS = 3, +}; + +extern void __buggy_use_of_MLX4_GET(void); +extern void __buggy_use_of_MLX4_PUT(void); + +static bool enable_qos; +module_param(enable_qos, bool, 0444); +MODULE_PARM_DESC(enable_qos, "Enable Enhanced QoS support (default: off)"); + +#define MLX4_GET(dest, source, offset) \ + do { \ + void *__p = (char *) (source) + (offset); \ + typedef struct { u64 value; } __packed u64_p_t; \ + u64 val; \ + switch (sizeof (dest)) { \ + case 1: (dest) = *(u8 *) __p; break; \ + case 2: (dest) = be16_to_cpup(__p); break; \ + case 4: (dest) = be32_to_cpup(__p); break; \ + case 8: val = ((u64_p_t *)__p)->value; \ + (dest) = be64_to_cpu(val); break; \ + default: __buggy_use_of_MLX4_GET(); \ + } \ + } while (0) + +#define MLX4_PUT(dest, source, offset) \ + do { \ + void *__d = ((char *) (dest) + (offset)); \ + switch (sizeof(source)) { \ + case 1: *(u8 *) __d = (source); break; \ + case 2: *(__be16 *) __d = cpu_to_be16(source); break; \ + case 4: *(__be32 *) __d = cpu_to_be32(source); break; \ + case 8: *(__be64 *) __d = cpu_to_be64(source); break; \ + default: __buggy_use_of_MLX4_PUT(); \ + } \ + } while (0) + +static void dump_dev_cap_flags(struct mlx4_dev *dev, u64 flags) +{ + static const char *fname[] = { + [ 0] = "RC transport", + [ 1] = "UC transport", + [ 2] = "UD transport", + [ 3] = "XRC transport", + [ 6] = "SRQ support", + [ 7] = "IPoIB checksum offload", + [ 8] = "P_Key violation counter", + [ 9] = "Q_Key violation counter", + [12] = "Dual Port Different Protocol (DPDP) support", + [15] = "Big LSO headers", + [16] = "MW support", + [17] = "APM support", + [18] = "Atomic ops support", + [19] = "Raw multicast support", + [20] = "Address vector port checking support", + [21] = "UD multicast support", + [30] = "IBoE support", + [32] = "Unicast loopback support", + [34] = "FCS header control", + [37] = "Wake On LAN (port1) support", + [38] = "Wake On LAN (port2) support", + [40] = "UDP RSS support", + [41] = "Unicast VEP steering support", + [42] = "Multicast VEP steering support", + [48] = "Counters support", + [52] = "RSS IP fragments support", + [53] = "Port ETS Scheduler support", + [55] = "Port link type sensing support", + [59] = "Port management change event support", + [61] = "64 byte EQE support", + [62] = "64 byte CQE support", + }; + int i; + + mlx4_dbg(dev, "DEV_CAP flags:\n"); + for (i = 0; i < ARRAY_SIZE(fname); ++i) + if (fname[i] && (flags & (1LL << i))) + mlx4_dbg(dev, " %s\n", fname[i]); +} + +static void dump_dev_cap_flags2(struct mlx4_dev *dev, u64 flags) +{ + static const char * const fname[] = { + [0] = "RSS support", + [1] = "RSS Toeplitz Hash Function support", + [2] = "RSS XOR Hash Function support", + [3] = "Device managed flow steering support", + [4] = "Automatic MAC reassignment support", + [5] = "Time stamping support", + [6] = "VST (control vlan insertion/stripping) support", + [7] = "FSM (MAC anti-spoofing) support", + [8] = "Dynamic QP updates support", + [9] = "Device managed flow steering IPoIB support", + [10] = "TCP/IP offloads/flow-steering for VXLAN support", + [11] = "MAD DEMUX (Secure-Host) support", + [12] = "Large cache line (>64B) CQE stride support", + [13] = "Large cache line (>64B) EQE stride support", + [14] = "Ethernet protocol control support", + [15] = "Ethernet Backplane autoneg support", + [16] = "CONFIG DEV support", + [17] = "Asymmetric EQs support", + [18] = "More than 80 VFs support", + [19] = "Performance optimized for limited rule configuration flow steering support", + [20] = "Recoverable error events support", + [21] = "Port Remap support", + [22] = "QCN support", + [23] = "QP rate limiting support", + [24] = "Ethernet Flow control statistics support", + [25] = "Granular QoS per VF support", + [26] = "Port ETS Scheduler support", + [27] = "Port beacon support", + [28] = "RX-ALL support", + [29] = "802.1ad offload support", + [31] = "Modifying loopback source checks using UPDATE_QP support", + [32] = "Loopback source checks support", + [33] = "RoCEv2 support", + [34] = "DMFS Sniffer support (UC & MC)", + [35] = "QinQ VST mode support", + [36] = "sl to vl mapping table change event support" + }; + int i; + + for (i = 0; i < ARRAY_SIZE(fname); ++i) + if (fname[i] && (flags & (1LL << i))) + mlx4_dbg(dev, " %s\n", fname[i]); +} + +int mlx4_MOD_STAT_CFG(struct mlx4_dev *dev, struct mlx4_mod_stat_cfg *cfg) +{ + struct mlx4_cmd_mailbox *mailbox; + u32 *inbox; + int err = 0; + +#define MOD_STAT_CFG_IN_SIZE 0x100 + +#define MOD_STAT_CFG_PG_SZ_M_OFFSET 0x002 +#define MOD_STAT_CFG_PG_SZ_OFFSET 0x003 + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + inbox = mailbox->buf; + + MLX4_PUT(inbox, cfg->log_pg_sz, MOD_STAT_CFG_PG_SZ_OFFSET); + MLX4_PUT(inbox, cfg->log_pg_sz_m, MOD_STAT_CFG_PG_SZ_M_OFFSET); + + err = mlx4_cmd(dev, mailbox->dma, 0, 0, MLX4_CMD_MOD_STAT_CFG, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +int mlx4_QUERY_FUNC(struct mlx4_dev *dev, struct mlx4_func *func, int slave) +{ + struct mlx4_cmd_mailbox *mailbox; + u32 *outbox; + u8 in_modifier; + u8 field; + u16 field16; + int err; + +#define QUERY_FUNC_BUS_OFFSET 0x00 +#define QUERY_FUNC_DEVICE_OFFSET 0x01 +#define QUERY_FUNC_FUNCTION_OFFSET 0x01 +#define QUERY_FUNC_PHYSICAL_FUNCTION_OFFSET 0x03 +#define QUERY_FUNC_RSVD_EQS_OFFSET 0x04 +#define QUERY_FUNC_MAX_EQ_OFFSET 0x06 +#define QUERY_FUNC_RSVD_UARS_OFFSET 0x0b + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + outbox = mailbox->buf; + + in_modifier = slave; + + err = mlx4_cmd_box(dev, 0, mailbox->dma, in_modifier, 0, + MLX4_CMD_QUERY_FUNC, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + goto out; + + MLX4_GET(field, outbox, QUERY_FUNC_BUS_OFFSET); + func->bus = field & 0xf; + MLX4_GET(field, outbox, QUERY_FUNC_DEVICE_OFFSET); + func->device = field & 0xf1; + MLX4_GET(field, outbox, QUERY_FUNC_FUNCTION_OFFSET); + func->function = field & 0x7; + MLX4_GET(field, outbox, QUERY_FUNC_PHYSICAL_FUNCTION_OFFSET); + func->physical_function = field & 0xf; + MLX4_GET(field16, outbox, QUERY_FUNC_RSVD_EQS_OFFSET); + func->rsvd_eqs = field16 & 0xffff; + MLX4_GET(field16, outbox, QUERY_FUNC_MAX_EQ_OFFSET); + func->max_eq = field16 & 0xffff; + MLX4_GET(field, outbox, QUERY_FUNC_RSVD_UARS_OFFSET); + func->rsvd_uars = field & 0x0f; + + mlx4_dbg(dev, "Bus: %d, Device: %d, Function: %d, Physical function: %d, Max EQs: %d, Reserved EQs: %d, Reserved UARs: %d\n", + func->bus, func->device, func->function, func->physical_function, + func->max_eq, func->rsvd_eqs, func->rsvd_uars); + +out: + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +static int mlx4_activate_vst_qinq(struct mlx4_priv *priv, int slave, int port) +{ + struct mlx4_vport_oper_state *vp_oper; + struct mlx4_vport_state *vp_admin; + int err; + + vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; + vp_admin = &priv->mfunc.master.vf_admin[slave].vport[port]; + + if (vp_admin->default_vlan != vp_oper->state.default_vlan) { + err = __mlx4_register_vlan(&priv->dev, port, + vp_admin->default_vlan, + &vp_oper->vlan_idx); + if (err) { + vp_oper->vlan_idx = NO_INDX; + mlx4_warn(&priv->dev, + "No vlan resources slave %d, port %d\n", + slave, port); + return err; + } + mlx4_dbg(&priv->dev, "alloc vlan %d idx %d slave %d port %d\n", + (int)(vp_oper->state.default_vlan), + vp_oper->vlan_idx, slave, port); + } + vp_oper->state.vlan_proto = vp_admin->vlan_proto; + vp_oper->state.default_vlan = vp_admin->default_vlan; + vp_oper->state.default_qos = vp_admin->default_qos; + + return 0; +} + +static int mlx4_handle_vst_qinq(struct mlx4_priv *priv, int slave, int port) +{ + struct mlx4_vport_oper_state *vp_oper; + struct mlx4_slave_state *slave_state; + struct mlx4_vport_state *vp_admin; + int err; + + vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; + vp_admin = &priv->mfunc.master.vf_admin[slave].vport[port]; + slave_state = &priv->mfunc.master.slave_state[slave]; + + if ((vp_admin->vlan_proto != htons(ETH_P_8021AD)) || + (!slave_state->active)) + return 0; + + if (vp_oper->state.vlan_proto == vp_admin->vlan_proto && + vp_oper->state.default_vlan == vp_admin->default_vlan && + vp_oper->state.default_qos == vp_admin->default_qos) + return 0; + + if (!slave_state->vst_qinq_supported) { + /* Warn and revert the request to set vst QinQ mode */ + vp_admin->vlan_proto = vp_oper->state.vlan_proto; + vp_admin->default_vlan = vp_oper->state.default_vlan; + vp_admin->default_qos = vp_oper->state.default_qos; + + mlx4_warn(&priv->dev, + "Slave %d does not support VST QinQ mode\n", slave); + return 0; + } + + err = mlx4_activate_vst_qinq(priv, slave, port); + return err; +} + +int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + u8 field, port; + u32 size, proxy_qp, qkey; + int err = 0; + struct mlx4_func func; + +#define QUERY_FUNC_CAP_FLAGS_OFFSET 0x0 +#define QUERY_FUNC_CAP_NUM_PORTS_OFFSET 0x1 +#define QUERY_FUNC_CAP_PF_BHVR_OFFSET 0x4 +#define QUERY_FUNC_CAP_FMR_OFFSET 0x8 +#define QUERY_FUNC_CAP_QP_QUOTA_OFFSET_DEP 0x10 +#define QUERY_FUNC_CAP_CQ_QUOTA_OFFSET_DEP 0x14 +#define QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET_DEP 0x18 +#define QUERY_FUNC_CAP_MPT_QUOTA_OFFSET_DEP 0x20 +#define QUERY_FUNC_CAP_MTT_QUOTA_OFFSET_DEP 0x24 +#define QUERY_FUNC_CAP_MCG_QUOTA_OFFSET_DEP 0x28 +#define QUERY_FUNC_CAP_MAX_EQ_OFFSET 0x2c +#define QUERY_FUNC_CAP_RESERVED_EQ_OFFSET 0x30 +#define QUERY_FUNC_CAP_QP_RESD_LKEY_OFFSET 0x48 + +#define QUERY_FUNC_CAP_QP_QUOTA_OFFSET 0x50 +#define QUERY_FUNC_CAP_CQ_QUOTA_OFFSET 0x54 +#define QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET 0x58 +#define QUERY_FUNC_CAP_MPT_QUOTA_OFFSET 0x60 +#define QUERY_FUNC_CAP_MTT_QUOTA_OFFSET 0x64 +#define QUERY_FUNC_CAP_MCG_QUOTA_OFFSET 0x68 + +#define QUERY_FUNC_CAP_EXTRA_FLAGS_OFFSET 0x6c + +#define QUERY_FUNC_CAP_FMR_FLAG 0x80 +#define QUERY_FUNC_CAP_FLAG_RDMA 0x40 +#define QUERY_FUNC_CAP_FLAG_ETH 0x80 +#define QUERY_FUNC_CAP_FLAG_QUOTAS 0x10 +#define QUERY_FUNC_CAP_FLAG_RESD_LKEY 0x08 +#define QUERY_FUNC_CAP_FLAG_VALID_MAILBOX 0x04 + +#define QUERY_FUNC_CAP_EXTRA_FLAGS_BF_QP_ALLOC_FLAG (1UL << 31) +#define QUERY_FUNC_CAP_EXTRA_FLAGS_A0_QP_ALLOC_FLAG (1UL << 30) + +/* when opcode modifier = 1 */ +#define QUERY_FUNC_CAP_PHYS_PORT_OFFSET 0x3 +#define QUERY_FUNC_CAP_PRIV_VF_QKEY_OFFSET 0x4 +#define QUERY_FUNC_CAP_FLAGS0_OFFSET 0x8 +#define QUERY_FUNC_CAP_FLAGS1_OFFSET 0xc + +#define QUERY_FUNC_CAP_QP0_TUNNEL 0x10 +#define QUERY_FUNC_CAP_QP0_PROXY 0x14 +#define QUERY_FUNC_CAP_QP1_TUNNEL 0x18 +#define QUERY_FUNC_CAP_QP1_PROXY 0x1c +#define QUERY_FUNC_CAP_PHYS_PORT_ID 0x28 + +#define QUERY_FUNC_CAP_FLAGS1_FORCE_MAC 0x40 +#define QUERY_FUNC_CAP_FLAGS1_FORCE_VLAN 0x80 +#define QUERY_FUNC_CAP_FLAGS1_NIC_INFO 0x10 +#define QUERY_FUNC_CAP_VF_ENABLE_QP0 0x08 + +#define QUERY_FUNC_CAP_FLAGS0_FORCE_PHY_WQE_GID 0x80 +#define QUERY_FUNC_CAP_PHV_BIT 0x40 +#define QUERY_FUNC_CAP_VLAN_OFFLOAD_DISABLE 0x20 + +#define QUERY_FUNC_CAP_SUPPORTS_VST_QINQ BIT(30) +#define QUERY_FUNC_CAP_SUPPORTS_NON_POWER_OF_2_NUM_EQS BIT(31) + + if (vhcr->op_modifier == 1) { + struct mlx4_active_ports actv_ports = + mlx4_get_active_ports(dev, slave); + int converted_port = mlx4_slave_convert_port( + dev, slave, vhcr->in_modifier); + struct mlx4_vport_oper_state *vp_oper; + + if (converted_port < 0) + return -EINVAL; + + vhcr->in_modifier = converted_port; + /* phys-port = logical-port */ + field = vhcr->in_modifier - + find_first_bit(actv_ports.ports, dev->caps.num_ports); + MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_PHYS_PORT_OFFSET); + + port = vhcr->in_modifier; + proxy_qp = dev->phys_caps.base_proxy_sqpn + 8 * slave + port - 1; + + /* Set nic_info bit to mark new fields support */ + field = QUERY_FUNC_CAP_FLAGS1_NIC_INFO; + + if (mlx4_vf_smi_enabled(dev, slave, port) && + !mlx4_get_parav_qkey(dev, proxy_qp, &qkey)) { + field |= QUERY_FUNC_CAP_VF_ENABLE_QP0; + MLX4_PUT(outbox->buf, qkey, + QUERY_FUNC_CAP_PRIV_VF_QKEY_OFFSET); + } + MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FLAGS1_OFFSET); + + /* size is now the QP number */ + size = dev->phys_caps.base_tunnel_sqpn + 8 * slave + port - 1; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP0_TUNNEL); + + size += 2; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP1_TUNNEL); + + MLX4_PUT(outbox->buf, proxy_qp, QUERY_FUNC_CAP_QP0_PROXY); + proxy_qp += 2; + MLX4_PUT(outbox->buf, proxy_qp, QUERY_FUNC_CAP_QP1_PROXY); + + MLX4_PUT(outbox->buf, dev->caps.phys_port_id[vhcr->in_modifier], + QUERY_FUNC_CAP_PHYS_PORT_ID); + + vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; + err = mlx4_handle_vst_qinq(priv, slave, port); + if (err) + return err; + + field = 0; + if (dev->caps.phv_bit[port]) + field |= QUERY_FUNC_CAP_PHV_BIT; + if (vp_oper->state.vlan_proto == htons(ETH_P_8021AD)) + field |= QUERY_FUNC_CAP_VLAN_OFFLOAD_DISABLE; + MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FLAGS0_OFFSET); + + } else if (vhcr->op_modifier == 0) { + struct mlx4_active_ports actv_ports = + mlx4_get_active_ports(dev, slave); + struct mlx4_slave_state *slave_state = + &priv->mfunc.master.slave_state[slave]; + + /* enable rdma and ethernet interfaces, new quota locations, + * and reserved lkey + */ + field = (QUERY_FUNC_CAP_FLAG_ETH | QUERY_FUNC_CAP_FLAG_RDMA | + QUERY_FUNC_CAP_FLAG_QUOTAS | QUERY_FUNC_CAP_FLAG_VALID_MAILBOX | + QUERY_FUNC_CAP_FLAG_RESD_LKEY); + MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FLAGS_OFFSET); + + field = min( + bitmap_weight(actv_ports.ports, dev->caps.num_ports), + dev->caps.num_ports); + MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_NUM_PORTS_OFFSET); + + size = dev->caps.function_caps; /* set PF behaviours */ + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_PF_BHVR_OFFSET); + + field = 0; /* protected FMR support not available as yet */ + MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FMR_OFFSET); + + size = priv->mfunc.master.res_tracker.res_alloc[RES_QP].quota[slave]; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP_QUOTA_OFFSET); + size = dev->caps.num_qps; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP_QUOTA_OFFSET_DEP); + + size = priv->mfunc.master.res_tracker.res_alloc[RES_SRQ].quota[slave]; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET); + size = dev->caps.num_srqs; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET_DEP); + + size = priv->mfunc.master.res_tracker.res_alloc[RES_CQ].quota[slave]; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET); + size = dev->caps.num_cqs; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET_DEP); + + if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) || + mlx4_QUERY_FUNC(dev, &func, slave)) { + size = vhcr->in_modifier & + QUERY_FUNC_CAP_SUPPORTS_NON_POWER_OF_2_NUM_EQS ? + dev->caps.num_eqs : + rounddown_pow_of_two(dev->caps.num_eqs); + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MAX_EQ_OFFSET); + size = dev->caps.reserved_eqs; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_RESERVED_EQ_OFFSET); + } else { + size = vhcr->in_modifier & + QUERY_FUNC_CAP_SUPPORTS_NON_POWER_OF_2_NUM_EQS ? + func.max_eq : + rounddown_pow_of_two(func.max_eq); + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MAX_EQ_OFFSET); + size = func.rsvd_eqs; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_RESERVED_EQ_OFFSET); + } + + size = priv->mfunc.master.res_tracker.res_alloc[RES_MPT].quota[slave]; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET); + size = dev->caps.num_mpts; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET_DEP); + + size = priv->mfunc.master.res_tracker.res_alloc[RES_MTT].quota[slave]; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET); + size = dev->caps.num_mtts; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET_DEP); + + size = dev->caps.num_mgms + dev->caps.num_amgms; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET); + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET_DEP); + + size = QUERY_FUNC_CAP_EXTRA_FLAGS_BF_QP_ALLOC_FLAG | + QUERY_FUNC_CAP_EXTRA_FLAGS_A0_QP_ALLOC_FLAG; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_EXTRA_FLAGS_OFFSET); + + size = dev->caps.reserved_lkey + ((slave << 8) & 0xFF00); + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP_RESD_LKEY_OFFSET); + + if (vhcr->in_modifier & QUERY_FUNC_CAP_SUPPORTS_VST_QINQ) + slave_state->vst_qinq_supported = true; + + } else + err = -EINVAL; + + return err; +} + +int mlx4_QUERY_FUNC_CAP(struct mlx4_dev *dev, u8 gen_or_port, + struct mlx4_func_cap *func_cap) +{ + struct mlx4_cmd_mailbox *mailbox; + u32 *outbox; + u8 field, op_modifier; + u32 size, qkey; + int err = 0, quotas = 0; + u32 in_modifier; + u32 slave_caps; + + op_modifier = !!gen_or_port; /* 0 = general, 1 = logical port */ + slave_caps = QUERY_FUNC_CAP_SUPPORTS_VST_QINQ | + QUERY_FUNC_CAP_SUPPORTS_NON_POWER_OF_2_NUM_EQS; + in_modifier = op_modifier ? gen_or_port : slave_caps; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + err = mlx4_cmd_box(dev, 0, mailbox->dma, in_modifier, op_modifier, + MLX4_CMD_QUERY_FUNC_CAP, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (err) + goto out; + + outbox = mailbox->buf; + + if (!op_modifier) { + MLX4_GET(field, outbox, QUERY_FUNC_CAP_FLAGS_OFFSET); + if (!(field & (QUERY_FUNC_CAP_FLAG_ETH | QUERY_FUNC_CAP_FLAG_RDMA))) { + mlx4_err(dev, "The host supports neither eth nor rdma interfaces\n"); + err = -EPROTONOSUPPORT; + goto out; + } + func_cap->flags = field; + quotas = !!(func_cap->flags & QUERY_FUNC_CAP_FLAG_QUOTAS); + + MLX4_GET(field, outbox, QUERY_FUNC_CAP_NUM_PORTS_OFFSET); + func_cap->num_ports = field; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_PF_BHVR_OFFSET); + func_cap->pf_context_behaviour = size; + + if (quotas) { + MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP_QUOTA_OFFSET); + func_cap->qp_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET); + func_cap->srq_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET); + func_cap->cq_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET); + func_cap->mpt_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET); + func_cap->mtt_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET); + func_cap->mcg_quota = size & 0xFFFFFF; + + } else { + MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP_QUOTA_OFFSET_DEP); + func_cap->qp_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET_DEP); + func_cap->srq_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET_DEP); + func_cap->cq_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET_DEP); + func_cap->mpt_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET_DEP); + func_cap->mtt_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET_DEP); + func_cap->mcg_quota = size & 0xFFFFFF; + } + MLX4_GET(size, outbox, QUERY_FUNC_CAP_MAX_EQ_OFFSET); + func_cap->max_eq = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_RESERVED_EQ_OFFSET); + func_cap->reserved_eq = size & 0xFFFFFF; + + if (func_cap->flags & QUERY_FUNC_CAP_FLAG_RESD_LKEY) { + MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP_RESD_LKEY_OFFSET); + func_cap->reserved_lkey = size; + } else { + func_cap->reserved_lkey = 0; + } + + func_cap->extra_flags = 0; + + /* Mailbox data from 0x6c and onward should only be treated if + * QUERY_FUNC_CAP_FLAG_VALID_MAILBOX is set in func_cap->flags + */ + if (func_cap->flags & QUERY_FUNC_CAP_FLAG_VALID_MAILBOX) { + MLX4_GET(size, outbox, QUERY_FUNC_CAP_EXTRA_FLAGS_OFFSET); + if (size & QUERY_FUNC_CAP_EXTRA_FLAGS_BF_QP_ALLOC_FLAG) + func_cap->extra_flags |= MLX4_QUERY_FUNC_FLAGS_BF_RES_QP; + if (size & QUERY_FUNC_CAP_EXTRA_FLAGS_A0_QP_ALLOC_FLAG) + func_cap->extra_flags |= MLX4_QUERY_FUNC_FLAGS_A0_RES_QP; + } + + goto out; + } + + /* logical port query */ + if (gen_or_port > dev->caps.num_ports) { + err = -EINVAL; + goto out; + } + + MLX4_GET(func_cap->flags1, outbox, QUERY_FUNC_CAP_FLAGS1_OFFSET); + if (dev->caps.port_type[gen_or_port] == MLX4_PORT_TYPE_ETH) { + if (func_cap->flags1 & QUERY_FUNC_CAP_FLAGS1_FORCE_VLAN) { + mlx4_err(dev, "VLAN is enforced on this port\n"); + err = -EPROTONOSUPPORT; + goto out; + } + + if (func_cap->flags1 & QUERY_FUNC_CAP_FLAGS1_FORCE_MAC) { + mlx4_err(dev, "Force mac is enabled on this port\n"); + err = -EPROTONOSUPPORT; + goto out; + } + } else if (dev->caps.port_type[gen_or_port] == MLX4_PORT_TYPE_IB) { + MLX4_GET(field, outbox, QUERY_FUNC_CAP_FLAGS0_OFFSET); + if (field & QUERY_FUNC_CAP_FLAGS0_FORCE_PHY_WQE_GID) { + mlx4_err(dev, "phy_wqe_gid is enforced on this ib port\n"); + err = -EPROTONOSUPPORT; + goto out; + } + } + + MLX4_GET(field, outbox, QUERY_FUNC_CAP_PHYS_PORT_OFFSET); + func_cap->physical_port = field; + if (func_cap->physical_port != gen_or_port) { + err = -ENOSYS; + goto out; + } + + if (func_cap->flags1 & QUERY_FUNC_CAP_VF_ENABLE_QP0) { + MLX4_GET(qkey, outbox, QUERY_FUNC_CAP_PRIV_VF_QKEY_OFFSET); + func_cap->qp0_qkey = qkey; + } else { + func_cap->qp0_qkey = 0; + } + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP0_TUNNEL); + func_cap->qp0_tunnel_qpn = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP0_PROXY); + func_cap->qp0_proxy_qpn = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP1_TUNNEL); + func_cap->qp1_tunnel_qpn = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP1_PROXY); + func_cap->qp1_proxy_qpn = size & 0xFFFFFF; + + if (func_cap->flags1 & QUERY_FUNC_CAP_FLAGS1_NIC_INFO) + MLX4_GET(func_cap->phys_port_id, outbox, + QUERY_FUNC_CAP_PHYS_PORT_ID); + + MLX4_GET(func_cap->flags0, outbox, QUERY_FUNC_CAP_FLAGS0_OFFSET); + + /* All other resources are allocated by the master, but we still report + * 'num' and 'reserved' capabilities as follows: + * - num remains the maximum resource index + * - 'num - reserved' is the total available objects of a resource, but + * resource indices may be less than 'reserved' + * TODO: set per-resource quotas */ + +out: + mlx4_free_cmd_mailbox(dev, mailbox); + + return err; +} + +static void disable_unsupported_roce_caps(void *buf); + +int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) +{ + struct mlx4_cmd_mailbox *mailbox; + u32 *outbox; + u8 field; + u32 field32, flags, ext_flags; + u16 size; + u16 stat_rate; + int err; + int i; + +#define QUERY_DEV_CAP_OUT_SIZE 0x100 +#define QUERY_DEV_CAP_MAX_SRQ_SZ_OFFSET 0x10 +#define QUERY_DEV_CAP_MAX_QP_SZ_OFFSET 0x11 +#define QUERY_DEV_CAP_RSVD_QP_OFFSET 0x12 +#define QUERY_DEV_CAP_MAX_QP_OFFSET 0x13 +#define QUERY_DEV_CAP_RSVD_SRQ_OFFSET 0x14 +#define QUERY_DEV_CAP_MAX_SRQ_OFFSET 0x15 +#define QUERY_DEV_CAP_RSVD_EEC_OFFSET 0x16 +#define QUERY_DEV_CAP_MAX_EEC_OFFSET 0x17 +#define QUERY_DEV_CAP_MAX_CQ_SZ_OFFSET 0x19 +#define QUERY_DEV_CAP_RSVD_CQ_OFFSET 0x1a +#define QUERY_DEV_CAP_MAX_CQ_OFFSET 0x1b +#define QUERY_DEV_CAP_MAX_MPT_OFFSET 0x1d +#define QUERY_DEV_CAP_RSVD_EQ_OFFSET 0x1e +#define QUERY_DEV_CAP_MAX_EQ_OFFSET 0x1f +#define QUERY_DEV_CAP_RSVD_MTT_OFFSET 0x20 +#define QUERY_DEV_CAP_MAX_MRW_SZ_OFFSET 0x21 +#define QUERY_DEV_CAP_RSVD_MRW_OFFSET 0x22 +#define QUERY_DEV_CAP_MAX_MTT_SEG_OFFSET 0x23 +#define QUERY_DEV_CAP_NUM_SYS_EQ_OFFSET 0x26 +#define QUERY_DEV_CAP_MAX_AV_OFFSET 0x27 +#define QUERY_DEV_CAP_MAX_REQ_QP_OFFSET 0x29 +#define QUERY_DEV_CAP_MAX_RES_QP_OFFSET 0x2b +#define QUERY_DEV_CAP_MAX_GSO_OFFSET 0x2d +#define QUERY_DEV_CAP_RSS_OFFSET 0x2e +#define QUERY_DEV_CAP_MAX_RDMA_OFFSET 0x2f +#define QUERY_DEV_CAP_RSZ_SRQ_OFFSET 0x33 +#define QUERY_DEV_CAP_PORT_BEACON_OFFSET 0x34 +#define QUERY_DEV_CAP_ACK_DELAY_OFFSET 0x35 +#define QUERY_DEV_CAP_MTU_WIDTH_OFFSET 0x36 +#define QUERY_DEV_CAP_VL_PORT_OFFSET 0x37 +#define QUERY_DEV_CAP_MAX_MSG_SZ_OFFSET 0x38 +#define QUERY_DEV_CAP_MAX_GID_OFFSET 0x3b +#define QUERY_DEV_CAP_RATE_SUPPORT_OFFSET 0x3c +#define QUERY_DEV_CAP_CQ_TS_SUPPORT_OFFSET 0x3e +#define QUERY_DEV_CAP_MAX_PKEY_OFFSET 0x3f +#define QUERY_DEV_CAP_EXT_FLAGS_OFFSET 0x40 +#define QUERY_DEV_CAP_FLAGS_OFFSET 0x44 +#define QUERY_DEV_CAP_RSVD_UAR_OFFSET 0x48 +#define QUERY_DEV_CAP_UAR_SZ_OFFSET 0x49 +#define QUERY_DEV_CAP_PAGE_SZ_OFFSET 0x4b +#define QUERY_DEV_CAP_BF_OFFSET 0x4c +#define QUERY_DEV_CAP_LOG_BF_REG_SZ_OFFSET 0x4d +#define QUERY_DEV_CAP_LOG_MAX_BF_REGS_PER_PAGE_OFFSET 0x4e +#define QUERY_DEV_CAP_LOG_MAX_BF_PAGES_OFFSET 0x4f +#define QUERY_DEV_CAP_MAX_SG_SQ_OFFSET 0x51 +#define QUERY_DEV_CAP_MAX_DESC_SZ_SQ_OFFSET 0x52 +#define QUERY_DEV_CAP_MAX_SG_RQ_OFFSET 0x55 +#define QUERY_DEV_CAP_MAX_DESC_SZ_RQ_OFFSET 0x56 +#define QUERY_DEV_CAP_SVLAN_BY_QP_OFFSET 0x5D +#define QUERY_DEV_CAP_MAX_QP_MCG_OFFSET 0x61 +#define QUERY_DEV_CAP_RSVD_MCG_OFFSET 0x62 +#define QUERY_DEV_CAP_MAX_MCG_OFFSET 0x63 +#define QUERY_DEV_CAP_RSVD_PD_OFFSET 0x64 +#define QUERY_DEV_CAP_MAX_PD_OFFSET 0x65 +#define QUERY_DEV_CAP_RSVD_XRC_OFFSET 0x66 +#define QUERY_DEV_CAP_MAX_XRC_OFFSET 0x67 +#define QUERY_DEV_CAP_MAX_COUNTERS_OFFSET 0x68 +#define QUERY_DEV_CAP_PORT_FLOWSTATS_COUNTERS_OFFSET 0x70 +#define QUERY_DEV_CAP_EXT_2_FLAGS_OFFSET 0x70 +#define QUERY_DEV_CAP_FLOW_STEERING_IPOIB_OFFSET 0x74 +#define QUERY_DEV_CAP_FLOW_STEERING_RANGE_EN_OFFSET 0x76 +#define QUERY_DEV_CAP_FLOW_STEERING_MAX_QP_OFFSET 0x77 +#define QUERY_DEV_CAP_SL2VL_EVENT_OFFSET 0x78 +#define QUERY_DEV_CAP_CQ_EQ_CACHE_LINE_STRIDE 0x7a +#define QUERY_DEV_CAP_ECN_QCN_VER_OFFSET 0x7b +#define QUERY_DEV_CAP_RDMARC_ENTRY_SZ_OFFSET 0x80 +#define QUERY_DEV_CAP_QPC_ENTRY_SZ_OFFSET 0x82 +#define QUERY_DEV_CAP_AUX_ENTRY_SZ_OFFSET 0x84 +#define QUERY_DEV_CAP_ALTC_ENTRY_SZ_OFFSET 0x86 +#define QUERY_DEV_CAP_EQC_ENTRY_SZ_OFFSET 0x88 +#define QUERY_DEV_CAP_CQC_ENTRY_SZ_OFFSET 0x8a +#define QUERY_DEV_CAP_SRQ_ENTRY_SZ_OFFSET 0x8c +#define QUERY_DEV_CAP_C_MPT_ENTRY_SZ_OFFSET 0x8e +#define QUERY_DEV_CAP_MTT_ENTRY_SZ_OFFSET 0x90 +#define QUERY_DEV_CAP_D_MPT_ENTRY_SZ_OFFSET 0x92 +#define QUERY_DEV_CAP_BMME_FLAGS_OFFSET 0x94 +#define QUERY_DEV_CAP_CONFIG_DEV_OFFSET 0x94 +#define QUERY_DEV_CAP_PHV_EN_OFFSET 0x96 +#define QUERY_DEV_CAP_RSVD_LKEY_OFFSET 0x98 +#define QUERY_DEV_CAP_MAX_ICM_SZ_OFFSET 0xa0 +#define QUERY_DEV_CAP_ETH_BACKPL_OFFSET 0x9c +#define QUERY_DEV_CAP_DIAG_RPRT_PER_PORT 0x9c +#define QUERY_DEV_CAP_FW_REASSIGN_MAC 0x9d +#define QUERY_DEV_CAP_VXLAN 0x9e +#define QUERY_DEV_CAP_MAD_DEMUX_OFFSET 0xb0 +#define QUERY_DEV_CAP_DMFS_HIGH_RATE_QPN_BASE_OFFSET 0xa8 +#define QUERY_DEV_CAP_DMFS_HIGH_RATE_QPN_RANGE_OFFSET 0xac +#define QUERY_DEV_CAP_QP_RATE_LIMIT_NUM_OFFSET 0xcc +#define QUERY_DEV_CAP_QP_RATE_LIMIT_MAX_OFFSET 0xd0 +#define QUERY_DEV_CAP_QP_RATE_LIMIT_MIN_OFFSET 0xd2 + + + dev_cap->flags2 = 0; + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + outbox = mailbox->buf; + + err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, MLX4_CMD_QUERY_DEV_CAP, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + if (err) + goto out; + + if (mlx4_is_mfunc(dev)) + disable_unsupported_roce_caps(outbox); + MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_QP_OFFSET); + dev_cap->reserved_qps = 1 << (field & 0xf); + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_QP_OFFSET); + dev_cap->max_qps = 1 << (field & 0x1f); + MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_SRQ_OFFSET); + dev_cap->reserved_srqs = 1 << (field >> 4); + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_SRQ_OFFSET); + dev_cap->max_srqs = 1 << (field & 0x1f); + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_CQ_SZ_OFFSET); + dev_cap->max_cq_sz = 1 << field; + MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_CQ_OFFSET); + dev_cap->reserved_cqs = 1 << (field & 0xf); + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_CQ_OFFSET); + dev_cap->max_cqs = 1 << (field & 0x1f); + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_MPT_OFFSET); + dev_cap->max_mpts = 1 << (field & 0x3f); + MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_EQ_OFFSET); + dev_cap->reserved_eqs = 1 << (field & 0xf); + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_EQ_OFFSET); + dev_cap->max_eqs = 1 << (field & 0xf); + MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_MTT_OFFSET); + dev_cap->reserved_mtts = 1 << (field >> 4); + MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_MRW_OFFSET); + dev_cap->reserved_mrws = 1 << (field & 0xf); + MLX4_GET(size, outbox, QUERY_DEV_CAP_NUM_SYS_EQ_OFFSET); + dev_cap->num_sys_eqs = size & 0xfff; + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_REQ_QP_OFFSET); + dev_cap->max_requester_per_qp = 1 << (field & 0x3f); + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_RES_QP_OFFSET); + dev_cap->max_responder_per_qp = 1 << (field & 0x3f); + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_GSO_OFFSET); + field &= 0x1f; + if (!field) + dev_cap->max_gso_sz = 0; + else + dev_cap->max_gso_sz = 1 << field; + + MLX4_GET(field, outbox, QUERY_DEV_CAP_RSS_OFFSET); + if (field & 0x20) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_RSS_XOR; + if (field & 0x10) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_RSS_TOP; + field &= 0xf; + if (field) { + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_RSS; + dev_cap->max_rss_tbl_sz = 1 << field; + } else + dev_cap->max_rss_tbl_sz = 0; + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_RDMA_OFFSET); + dev_cap->max_rdma_global = 1 << (field & 0x3f); + MLX4_GET(field, outbox, QUERY_DEV_CAP_ACK_DELAY_OFFSET); + dev_cap->local_ca_ack_delay = field & 0x1f; + MLX4_GET(field, outbox, QUERY_DEV_CAP_VL_PORT_OFFSET); + dev_cap->num_ports = field & 0xf; + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_MSG_SZ_OFFSET); + dev_cap->max_msg_sz = 1 << (field & 0x1f); + MLX4_GET(field, outbox, QUERY_DEV_CAP_PORT_FLOWSTATS_COUNTERS_OFFSET); + if (field & 0x10) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN; + MLX4_GET(field, outbox, QUERY_DEV_CAP_FLOW_STEERING_RANGE_EN_OFFSET); + if (field & 0x80) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_FS_EN; + dev_cap->fs_log_max_ucast_qp_range_size = field & 0x1f; + if (field & 0x20) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_DMFS_UC_MC_SNIFFER; + MLX4_GET(field, outbox, QUERY_DEV_CAP_PORT_BEACON_OFFSET); + if (field & 0x80) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_PORT_BEACON; + MLX4_GET(field, outbox, QUERY_DEV_CAP_FLOW_STEERING_IPOIB_OFFSET); + if (field & 0x80) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_DMFS_IPOIB; + MLX4_GET(field, outbox, QUERY_DEV_CAP_FLOW_STEERING_MAX_QP_OFFSET); + dev_cap->fs_max_num_qp_per_entry = field; + MLX4_GET(field, outbox, QUERY_DEV_CAP_SL2VL_EVENT_OFFSET); + if (field & (1 << 5)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_SL_TO_VL_CHANGE_EVENT; + MLX4_GET(field, outbox, QUERY_DEV_CAP_ECN_QCN_VER_OFFSET); + if (field & 0x1) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_QCN; + MLX4_GET(stat_rate, outbox, QUERY_DEV_CAP_RATE_SUPPORT_OFFSET); + dev_cap->stat_rate_support = stat_rate; + MLX4_GET(field, outbox, QUERY_DEV_CAP_CQ_TS_SUPPORT_OFFSET); + if (field & 0x80) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_TS; + MLX4_GET(ext_flags, outbox, QUERY_DEV_CAP_EXT_FLAGS_OFFSET); + MLX4_GET(flags, outbox, QUERY_DEV_CAP_FLAGS_OFFSET); + dev_cap->flags = flags | (u64)ext_flags << 32; + MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_UAR_OFFSET); + dev_cap->reserved_uars = field >> 4; + MLX4_GET(field, outbox, QUERY_DEV_CAP_UAR_SZ_OFFSET); + dev_cap->uar_size = 1 << ((field & 0x3f) + 20); + MLX4_GET(field, outbox, QUERY_DEV_CAP_PAGE_SZ_OFFSET); + dev_cap->min_page_sz = 1 << field; + + MLX4_GET(field, outbox, QUERY_DEV_CAP_BF_OFFSET); + if (field & 0x80) { + MLX4_GET(field, outbox, QUERY_DEV_CAP_LOG_BF_REG_SZ_OFFSET); + dev_cap->bf_reg_size = 1 << (field & 0x1f); + MLX4_GET(field, outbox, QUERY_DEV_CAP_LOG_MAX_BF_REGS_PER_PAGE_OFFSET); + if ((1 << (field & 0x3f)) > (PAGE_SIZE / dev_cap->bf_reg_size)) + field = 3; + dev_cap->bf_regs_per_page = 1 << (field & 0x3f); + } else { + dev_cap->bf_reg_size = 0; + } + + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_SG_SQ_OFFSET); + dev_cap->max_sq_sg = field; + MLX4_GET(size, outbox, QUERY_DEV_CAP_MAX_DESC_SZ_SQ_OFFSET); + dev_cap->max_sq_desc_sz = size; + + MLX4_GET(field, outbox, QUERY_DEV_CAP_SVLAN_BY_QP_OFFSET); + if (field & 0x1) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_SVLAN_BY_QP; + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_QP_MCG_OFFSET); + dev_cap->max_qp_per_mcg = 1 << field; + MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_MCG_OFFSET); + dev_cap->reserved_mgms = field & 0xf; + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_MCG_OFFSET); + dev_cap->max_mcgs = 1 << field; + MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_PD_OFFSET); + dev_cap->reserved_pds = field >> 4; + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_PD_OFFSET); + dev_cap->max_pds = 1 << (field & 0x3f); + MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_XRC_OFFSET); + dev_cap->reserved_xrcds = field >> 4; + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_XRC_OFFSET); + dev_cap->max_xrcds = 1 << (field & 0x1f); + + MLX4_GET(size, outbox, QUERY_DEV_CAP_RDMARC_ENTRY_SZ_OFFSET); + dev_cap->rdmarc_entry_sz = size; + MLX4_GET(size, outbox, QUERY_DEV_CAP_QPC_ENTRY_SZ_OFFSET); + dev_cap->qpc_entry_sz = size; + MLX4_GET(size, outbox, QUERY_DEV_CAP_AUX_ENTRY_SZ_OFFSET); + dev_cap->aux_entry_sz = size; + MLX4_GET(size, outbox, QUERY_DEV_CAP_ALTC_ENTRY_SZ_OFFSET); + dev_cap->altc_entry_sz = size; + MLX4_GET(size, outbox, QUERY_DEV_CAP_EQC_ENTRY_SZ_OFFSET); + dev_cap->eqc_entry_sz = size; + MLX4_GET(size, outbox, QUERY_DEV_CAP_CQC_ENTRY_SZ_OFFSET); + dev_cap->cqc_entry_sz = size; + MLX4_GET(size, outbox, QUERY_DEV_CAP_SRQ_ENTRY_SZ_OFFSET); + dev_cap->srq_entry_sz = size; + MLX4_GET(size, outbox, QUERY_DEV_CAP_C_MPT_ENTRY_SZ_OFFSET); + dev_cap->cmpt_entry_sz = size; + MLX4_GET(size, outbox, QUERY_DEV_CAP_MTT_ENTRY_SZ_OFFSET); + dev_cap->mtt_entry_sz = size; + MLX4_GET(size, outbox, QUERY_DEV_CAP_D_MPT_ENTRY_SZ_OFFSET); + dev_cap->dmpt_entry_sz = size; + + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_SRQ_SZ_OFFSET); + dev_cap->max_srq_sz = 1 << field; + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_QP_SZ_OFFSET); + dev_cap->max_qp_sz = 1 << field; + MLX4_GET(field, outbox, QUERY_DEV_CAP_RSZ_SRQ_OFFSET); + dev_cap->resize_srq = field & 1; + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_SG_RQ_OFFSET); + dev_cap->max_rq_sg = field; + MLX4_GET(size, outbox, QUERY_DEV_CAP_MAX_DESC_SZ_RQ_OFFSET); + dev_cap->max_rq_desc_sz = size; + MLX4_GET(field, outbox, QUERY_DEV_CAP_CQ_EQ_CACHE_LINE_STRIDE); + if (field & (1 << 4)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_QOS_VPP; + if (field & (1 << 5)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_ETH_PROT_CTRL; + if (field & (1 << 6)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_CQE_STRIDE; + if (field & (1 << 7)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_EQE_STRIDE; + MLX4_GET(dev_cap->bmme_flags, outbox, + QUERY_DEV_CAP_BMME_FLAGS_OFFSET); + if (dev_cap->bmme_flags & MLX4_FLAG_ROCE_V1_V2) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_ROCE_V1_V2; + if (dev_cap->bmme_flags & MLX4_FLAG_PORT_REMAP) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_PORT_REMAP; + MLX4_GET(field, outbox, QUERY_DEV_CAP_CONFIG_DEV_OFFSET); + if (field & 0x20) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_CONFIG_DEV; + if (field & (1 << 2)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_IGNORE_FCS; + MLX4_GET(field, outbox, QUERY_DEV_CAP_PHV_EN_OFFSET); + if (field & 0x80) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_PHV_EN; + if (field & 0x40) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN; + + MLX4_GET(dev_cap->reserved_lkey, outbox, + QUERY_DEV_CAP_RSVD_LKEY_OFFSET); + MLX4_GET(field32, outbox, QUERY_DEV_CAP_ETH_BACKPL_OFFSET); + if (field32 & (1 << 0)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_ETH_BACKPL_AN_REP; + if (field32 & (1 << 7)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_RECOVERABLE_ERROR_EVENT; + MLX4_GET(field32, outbox, QUERY_DEV_CAP_DIAG_RPRT_PER_PORT); + if (field32 & (1 << 17)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_DIAG_PER_PORT; + MLX4_GET(field, outbox, QUERY_DEV_CAP_FW_REASSIGN_MAC); + if (field & (1 << 6)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_REASSIGN_MAC_EN; + MLX4_GET(field, outbox, QUERY_DEV_CAP_VXLAN); + if (field & (1 << 3)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_VXLAN_OFFLOADS; + if (field & (1 << 5)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_ETS_CFG; + MLX4_GET(dev_cap->max_icm_sz, outbox, + QUERY_DEV_CAP_MAX_ICM_SZ_OFFSET); + if (dev_cap->flags & MLX4_DEV_CAP_FLAG_COUNTERS) + MLX4_GET(dev_cap->max_counters, outbox, + QUERY_DEV_CAP_MAX_COUNTERS_OFFSET); + + MLX4_GET(field32, outbox, + QUERY_DEV_CAP_MAD_DEMUX_OFFSET); + if (field32 & (1 << 0)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_MAD_DEMUX; + + MLX4_GET(dev_cap->dmfs_high_rate_qpn_base, outbox, + QUERY_DEV_CAP_DMFS_HIGH_RATE_QPN_BASE_OFFSET); + dev_cap->dmfs_high_rate_qpn_base &= MGM_QPN_MASK; + MLX4_GET(dev_cap->dmfs_high_rate_qpn_range, outbox, + QUERY_DEV_CAP_DMFS_HIGH_RATE_QPN_RANGE_OFFSET); + dev_cap->dmfs_high_rate_qpn_range &= MGM_QPN_MASK; + + MLX4_GET(size, outbox, QUERY_DEV_CAP_QP_RATE_LIMIT_NUM_OFFSET); + dev_cap->rl_caps.num_rates = size; + if (dev_cap->rl_caps.num_rates) { + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_QP_RATE_LIMIT; + MLX4_GET(size, outbox, QUERY_DEV_CAP_QP_RATE_LIMIT_MAX_OFFSET); + dev_cap->rl_caps.max_val = size & 0xfff; + dev_cap->rl_caps.max_unit = size >> 14; + MLX4_GET(size, outbox, QUERY_DEV_CAP_QP_RATE_LIMIT_MIN_OFFSET); + dev_cap->rl_caps.min_val = size & 0xfff; + dev_cap->rl_caps.min_unit = size >> 14; + } + + MLX4_GET(field32, outbox, QUERY_DEV_CAP_EXT_2_FLAGS_OFFSET); + if (field32 & (1 << 16)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_UPDATE_QP; + if (field32 & (1 << 18)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_UPDATE_QP_SRC_CHECK_LB; + if (field32 & (1 << 19)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_LB_SRC_CHK; + if (field32 & (1 << 26)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_VLAN_CONTROL; + if (field32 & (1 << 20)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_FSM; + if (field32 & (1 << 21)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_80_VFS; + + for (i = 1; i <= dev_cap->num_ports; i++) { + err = mlx4_QUERY_PORT(dev, i, dev_cap->port_cap + i); + if (err) + goto out; + } + + /* + * Each UAR has 4 EQ doorbells; so if a UAR is reserved, then + * we can't use any EQs whose doorbell falls on that page, + * even if the EQ itself isn't reserved. + */ + if (dev_cap->num_sys_eqs == 0) + dev_cap->reserved_eqs = max(dev_cap->reserved_uars * 4, + dev_cap->reserved_eqs); + else + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_SYS_EQS; + +out: + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +void mlx4_dev_cap_dump(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) +{ + if (dev_cap->bf_reg_size > 0) + mlx4_dbg(dev, "BlueFlame available (reg size %d, regs/page %d)\n", + dev_cap->bf_reg_size, dev_cap->bf_regs_per_page); + else + mlx4_dbg(dev, "BlueFlame not available\n"); + + mlx4_dbg(dev, "Base MM extensions: flags %08x, rsvd L_Key %08x\n", + dev_cap->bmme_flags, dev_cap->reserved_lkey); + mlx4_dbg(dev, "Max ICM size %lld MB\n", + (unsigned long long) dev_cap->max_icm_sz >> 20); + mlx4_dbg(dev, "Max QPs: %d, reserved QPs: %d, entry size: %d\n", + dev_cap->max_qps, dev_cap->reserved_qps, dev_cap->qpc_entry_sz); + mlx4_dbg(dev, "Max SRQs: %d, reserved SRQs: %d, entry size: %d\n", + dev_cap->max_srqs, dev_cap->reserved_srqs, dev_cap->srq_entry_sz); + mlx4_dbg(dev, "Max CQs: %d, reserved CQs: %d, entry size: %d\n", + dev_cap->max_cqs, dev_cap->reserved_cqs, dev_cap->cqc_entry_sz); + mlx4_dbg(dev, "Num sys EQs: %d, max EQs: %d, reserved EQs: %d, entry size: %d\n", + dev_cap->num_sys_eqs, dev_cap->max_eqs, dev_cap->reserved_eqs, + dev_cap->eqc_entry_sz); + mlx4_dbg(dev, "reserved MPTs: %d, reserved MTTs: %d\n", + dev_cap->reserved_mrws, dev_cap->reserved_mtts); + mlx4_dbg(dev, "Max PDs: %d, reserved PDs: %d, reserved UARs: %d\n", + dev_cap->max_pds, dev_cap->reserved_pds, dev_cap->reserved_uars); + mlx4_dbg(dev, "Max QP/MCG: %d, reserved MGMs: %d\n", + dev_cap->max_pds, dev_cap->reserved_mgms); + mlx4_dbg(dev, "Max CQEs: %d, max WQEs: %d, max SRQ WQEs: %d\n", + dev_cap->max_cq_sz, dev_cap->max_qp_sz, dev_cap->max_srq_sz); + mlx4_dbg(dev, "Local CA ACK delay: %d, max MTU: %d, port width cap: %d\n", + dev_cap->local_ca_ack_delay, 128 << dev_cap->port_cap[1].ib_mtu, + dev_cap->port_cap[1].max_port_width); + mlx4_dbg(dev, "Max SQ desc size: %d, max SQ S/G: %d\n", + dev_cap->max_sq_desc_sz, dev_cap->max_sq_sg); + mlx4_dbg(dev, "Max RQ desc size: %d, max RQ S/G: %d\n", + dev_cap->max_rq_desc_sz, dev_cap->max_rq_sg); + mlx4_dbg(dev, "Max GSO size: %d\n", dev_cap->max_gso_sz); + mlx4_dbg(dev, "Max counters: %d\n", dev_cap->max_counters); + mlx4_dbg(dev, "Max RSS Table size: %d\n", dev_cap->max_rss_tbl_sz); + mlx4_dbg(dev, "DMFS high rate steer QPn base: %d\n", + dev_cap->dmfs_high_rate_qpn_base); + mlx4_dbg(dev, "DMFS high rate steer QPn range: %d\n", + dev_cap->dmfs_high_rate_qpn_range); + + if (dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_QP_RATE_LIMIT) { + struct mlx4_rate_limit_caps *rl_caps = &dev_cap->rl_caps; + + mlx4_dbg(dev, "QP Rate-Limit: #rates %d, unit/val max %d/%d, min %d/%d\n", + rl_caps->num_rates, rl_caps->max_unit, rl_caps->max_val, + rl_caps->min_unit, rl_caps->min_val); + } + + dump_dev_cap_flags(dev, dev_cap->flags); + dump_dev_cap_flags2(dev, dev_cap->flags2); +} + +int mlx4_QUERY_PORT(struct mlx4_dev *dev, int port, struct mlx4_port_cap *port_cap) +{ + struct mlx4_cmd_mailbox *mailbox; + u32 *outbox; + u8 field; + u32 field32; + int err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + outbox = mailbox->buf; + + if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { + err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, MLX4_CMD_QUERY_DEV_CAP, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + + if (err) + goto out; + + MLX4_GET(field, outbox, QUERY_DEV_CAP_VL_PORT_OFFSET); + port_cap->max_vl = field >> 4; + MLX4_GET(field, outbox, QUERY_DEV_CAP_MTU_WIDTH_OFFSET); + port_cap->ib_mtu = field >> 4; + port_cap->max_port_width = field & 0xf; + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_GID_OFFSET); + port_cap->max_gids = 1 << (field & 0xf); + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_PKEY_OFFSET); + port_cap->max_pkeys = 1 << (field & 0xf); + } else { +#define QUERY_PORT_SUPPORTED_TYPE_OFFSET 0x00 +#define QUERY_PORT_MTU_OFFSET 0x01 +#define QUERY_PORT_ETH_MTU_OFFSET 0x02 +#define QUERY_PORT_WIDTH_OFFSET 0x06 +#define QUERY_PORT_MAX_GID_PKEY_OFFSET 0x07 +#define QUERY_PORT_MAX_MACVLAN_OFFSET 0x0a +#define QUERY_PORT_MAX_VL_OFFSET 0x0b +#define QUERY_PORT_MAC_OFFSET 0x10 +#define QUERY_PORT_TRANS_VENDOR_OFFSET 0x18 +#define QUERY_PORT_WAVELENGTH_OFFSET 0x1c +#define QUERY_PORT_TRANS_CODE_OFFSET 0x20 + + err = mlx4_cmd_box(dev, 0, mailbox->dma, port, 0, MLX4_CMD_QUERY_PORT, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); + if (err) + goto out; + + MLX4_GET(field, outbox, QUERY_PORT_SUPPORTED_TYPE_OFFSET); + port_cap->link_state = (field & 0x80) >> 7; + port_cap->supported_port_types = field & 3; + port_cap->suggested_type = (field >> 3) & 1; + port_cap->default_sense = (field >> 4) & 1; + port_cap->dmfs_optimized_state = (field >> 5) & 1; + MLX4_GET(field, outbox, QUERY_PORT_MTU_OFFSET); + port_cap->ib_mtu = field & 0xf; + MLX4_GET(field, outbox, QUERY_PORT_WIDTH_OFFSET); + port_cap->max_port_width = field & 0xf; + MLX4_GET(field, outbox, QUERY_PORT_MAX_GID_PKEY_OFFSET); + port_cap->max_gids = 1 << (field >> 4); + port_cap->max_pkeys = 1 << (field & 0xf); + MLX4_GET(field, outbox, QUERY_PORT_MAX_VL_OFFSET); + port_cap->max_vl = field & 0xf; + port_cap->max_tc_eth = field >> 4; + MLX4_GET(field, outbox, QUERY_PORT_MAX_MACVLAN_OFFSET); + port_cap->log_max_macs = field & 0xf; + port_cap->log_max_vlans = field >> 4; + MLX4_GET(port_cap->eth_mtu, outbox, QUERY_PORT_ETH_MTU_OFFSET); + MLX4_GET(port_cap->def_mac, outbox, QUERY_PORT_MAC_OFFSET); + MLX4_GET(field32, outbox, QUERY_PORT_TRANS_VENDOR_OFFSET); + port_cap->trans_type = field32 >> 24; + port_cap->vendor_oui = field32 & 0xffffff; + MLX4_GET(port_cap->wavelength, outbox, QUERY_PORT_WAVELENGTH_OFFSET); + MLX4_GET(port_cap->trans_code, outbox, QUERY_PORT_TRANS_CODE_OFFSET); + } + +out: + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +#define DEV_CAP_EXT_2_FLAG_PFC_COUNTERS (1 << 28) +#define DEV_CAP_EXT_2_FLAG_VLAN_CONTROL (1 << 26) +#define DEV_CAP_EXT_2_FLAG_80_VFS (1 << 21) +#define DEV_CAP_EXT_2_FLAG_FSM (1 << 20) + +int mlx4_QUERY_DEV_CAP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + u64 flags; + int err = 0; + u8 field; + u16 field16; + u32 bmme_flags, field32; + int real_port; + int slave_port; + int first_port; + struct mlx4_active_ports actv_ports; + + err = mlx4_cmd_box(dev, 0, outbox->dma, 0, 0, MLX4_CMD_QUERY_DEV_CAP, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + if (err) + return err; + + disable_unsupported_roce_caps(outbox->buf); + /* add port mng change event capability and disable mw type 1 + * unconditionally to slaves + */ + MLX4_GET(flags, outbox->buf, QUERY_DEV_CAP_EXT_FLAGS_OFFSET); + flags |= MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV; + flags &= ~MLX4_DEV_CAP_FLAG_MEM_WINDOW; + actv_ports = mlx4_get_active_ports(dev, slave); + first_port = find_first_bit(actv_ports.ports, dev->caps.num_ports); + for (slave_port = 0, real_port = first_port; + real_port < first_port + + bitmap_weight(actv_ports.ports, dev->caps.num_ports); + ++real_port, ++slave_port) { + if (flags & (MLX4_DEV_CAP_FLAG_WOL_PORT1 << real_port)) + flags |= MLX4_DEV_CAP_FLAG_WOL_PORT1 << slave_port; + else + flags &= ~(MLX4_DEV_CAP_FLAG_WOL_PORT1 << slave_port); + } + for (; slave_port < dev->caps.num_ports; ++slave_port) + flags &= ~(MLX4_DEV_CAP_FLAG_WOL_PORT1 << slave_port); + + /* Not exposing RSS IP fragments to guests */ + flags &= ~MLX4_DEV_CAP_FLAG_RSS_IP_FRAG; + MLX4_PUT(outbox->buf, flags, QUERY_DEV_CAP_EXT_FLAGS_OFFSET); + + MLX4_GET(field, outbox->buf, QUERY_DEV_CAP_VL_PORT_OFFSET); + field &= ~0x0F; + field |= bitmap_weight(actv_ports.ports, dev->caps.num_ports) & 0x0F; + MLX4_PUT(outbox->buf, field, QUERY_DEV_CAP_VL_PORT_OFFSET); + + /* For guests, disable timestamp */ + MLX4_GET(field, outbox->buf, QUERY_DEV_CAP_CQ_TS_SUPPORT_OFFSET); + field &= 0x7f; + MLX4_PUT(outbox->buf, field, QUERY_DEV_CAP_CQ_TS_SUPPORT_OFFSET); + + /* For guests, disable vxlan tunneling and QoS support */ + MLX4_GET(field, outbox->buf, QUERY_DEV_CAP_VXLAN); + field &= 0xd7; + MLX4_PUT(outbox->buf, field, QUERY_DEV_CAP_VXLAN); + + /* For guests, disable port BEACON */ + MLX4_GET(field, outbox->buf, QUERY_DEV_CAP_PORT_BEACON_OFFSET); + field &= 0x7f; + MLX4_PUT(outbox->buf, field, QUERY_DEV_CAP_PORT_BEACON_OFFSET); + + /* For guests, report Blueflame disabled */ + MLX4_GET(field, outbox->buf, QUERY_DEV_CAP_BF_OFFSET); + field &= 0x7f; + MLX4_PUT(outbox->buf, field, QUERY_DEV_CAP_BF_OFFSET); + + /* For guests, disable mw type 2 and port remap*/ + MLX4_GET(bmme_flags, outbox->buf, QUERY_DEV_CAP_BMME_FLAGS_OFFSET); + bmme_flags &= ~MLX4_BMME_FLAG_TYPE_2_WIN; + bmme_flags &= ~MLX4_FLAG_PORT_REMAP; + MLX4_PUT(outbox->buf, bmme_flags, QUERY_DEV_CAP_BMME_FLAGS_OFFSET); + + /* turn off device-managed steering capability if not enabled */ + if (dev->caps.steering_mode != MLX4_STEERING_MODE_DEVICE_MANAGED) { + MLX4_GET(field, outbox->buf, + QUERY_DEV_CAP_FLOW_STEERING_RANGE_EN_OFFSET); + field &= 0x7f; + MLX4_PUT(outbox->buf, field, + QUERY_DEV_CAP_FLOW_STEERING_RANGE_EN_OFFSET); + } + + /* turn off ipoib managed steering for guests */ + MLX4_GET(field, outbox->buf, QUERY_DEV_CAP_FLOW_STEERING_IPOIB_OFFSET); + field &= ~0x80; + MLX4_PUT(outbox->buf, field, QUERY_DEV_CAP_FLOW_STEERING_IPOIB_OFFSET); + + /* turn off host side virt features (VST, FSM, etc) for guests */ + MLX4_GET(field32, outbox->buf, QUERY_DEV_CAP_EXT_2_FLAGS_OFFSET); + field32 &= ~(DEV_CAP_EXT_2_FLAG_VLAN_CONTROL | DEV_CAP_EXT_2_FLAG_80_VFS | + DEV_CAP_EXT_2_FLAG_FSM | DEV_CAP_EXT_2_FLAG_PFC_COUNTERS); + MLX4_PUT(outbox->buf, field32, QUERY_DEV_CAP_EXT_2_FLAGS_OFFSET); + + /* turn off QCN for guests */ + MLX4_GET(field, outbox->buf, QUERY_DEV_CAP_ECN_QCN_VER_OFFSET); + field &= 0xfe; + MLX4_PUT(outbox->buf, field, QUERY_DEV_CAP_ECN_QCN_VER_OFFSET); + + /* turn off QP max-rate limiting for guests */ + field16 = 0; + MLX4_PUT(outbox->buf, field16, QUERY_DEV_CAP_QP_RATE_LIMIT_NUM_OFFSET); + + /* turn off QoS per VF support for guests */ + MLX4_GET(field, outbox->buf, QUERY_DEV_CAP_CQ_EQ_CACHE_LINE_STRIDE); + field &= 0xef; + MLX4_PUT(outbox->buf, field, QUERY_DEV_CAP_CQ_EQ_CACHE_LINE_STRIDE); + + /* turn off ignore FCS feature for guests */ + MLX4_GET(field, outbox->buf, QUERY_DEV_CAP_CONFIG_DEV_OFFSET); + field &= 0xfb; + MLX4_PUT(outbox->buf, field, QUERY_DEV_CAP_CONFIG_DEV_OFFSET); + + return 0; +} + +static void disable_unsupported_roce_caps(void *buf) +{ + u32 flags; + + MLX4_GET(flags, buf, QUERY_DEV_CAP_EXT_FLAGS_OFFSET); + flags &= ~(1UL << 31); + MLX4_PUT(buf, flags, QUERY_DEV_CAP_EXT_FLAGS_OFFSET); + MLX4_GET(flags, buf, QUERY_DEV_CAP_EXT_2_FLAGS_OFFSET); + flags &= ~(1UL << 24); + MLX4_PUT(buf, flags, QUERY_DEV_CAP_EXT_2_FLAGS_OFFSET); + MLX4_GET(flags, buf, QUERY_DEV_CAP_BMME_FLAGS_OFFSET); + flags &= ~(MLX4_FLAG_ROCE_V1_V2); + MLX4_PUT(buf, flags, QUERY_DEV_CAP_BMME_FLAGS_OFFSET); +} + +int mlx4_QUERY_PORT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + u64 def_mac; + u8 port_type; + u16 short_field; + int err; + int port = mlx4_slave_convert_port(dev, slave, + vhcr->in_modifier & 0xFF); + +#define MLX4_VF_PORT_NO_LINK_SENSE_MASK 0xE0 +#define MLX4_PORT_LINK_UP_MASK 0x80 +#define QUERY_PORT_CUR_MAX_PKEY_OFFSET 0x0c +#define QUERY_PORT_CUR_MAX_GID_OFFSET 0x0e + + if (port < 0) + return -EINVAL; + + /* Protect against untrusted guests: enforce that this is the + * QUERY_PORT general query. + */ + if (vhcr->op_modifier || vhcr->in_modifier & ~0xFF) + return -EINVAL; + + vhcr->in_modifier = port; + + err = mlx4_cmd_box(dev, 0, outbox->dma, vhcr->in_modifier, 0, + MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); + + if (!err && dev->caps.function != slave) { + def_mac = priv->mfunc.master.vf_oper[slave].vport[vhcr->in_modifier].state.mac; + MLX4_PUT(outbox->buf, def_mac, QUERY_PORT_MAC_OFFSET); + + /* get port type - currently only eth is enabled */ + MLX4_GET(port_type, outbox->buf, + QUERY_PORT_SUPPORTED_TYPE_OFFSET); + + /* No link sensing allowed */ + port_type &= MLX4_VF_PORT_NO_LINK_SENSE_MASK; + /* set port type to currently operating port type */ + port_type |= (dev->caps.port_type[vhcr->in_modifier] & 0x3); + + if (0 /* IFLA_VF_LINK_STATE_ENABLE == admin_link_state */) + port_type |= MLX4_PORT_LINK_UP_MASK; + else if (1 /* IFLA_VF_LINK_STATE_DISABLE == admin_link_state */) + port_type &= ~MLX4_PORT_LINK_UP_MASK; + else if (0 /* IFLA_VF_LINK_STATE_AUTO == admin_link_state && mlx4_is_bonded(dev) */) { + int other_port = (port == 1) ? 2 : 1; + struct mlx4_port_cap port_cap; + + err = mlx4_QUERY_PORT(dev, other_port, &port_cap); + if (err) + goto out; + port_type |= (port_cap.link_state << 7); + } + + MLX4_PUT(outbox->buf, port_type, + QUERY_PORT_SUPPORTED_TYPE_OFFSET); + + if (dev->caps.port_type[vhcr->in_modifier] == MLX4_PORT_TYPE_ETH) + short_field = mlx4_get_slave_num_gids(dev, slave, port); + else + short_field = 1; /* slave max gids */ + MLX4_PUT(outbox->buf, short_field, + QUERY_PORT_CUR_MAX_GID_OFFSET); + + short_field = dev->caps.pkey_table_len[vhcr->in_modifier]; + MLX4_PUT(outbox->buf, short_field, + QUERY_PORT_CUR_MAX_PKEY_OFFSET); + } +out: + return err; +} + +int mlx4_get_slave_pkey_gid_tbl_len(struct mlx4_dev *dev, u8 port, + int *gid_tbl_len, int *pkey_tbl_len) +{ + struct mlx4_cmd_mailbox *mailbox; + u32 *outbox; + u16 field; + int err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + err = mlx4_cmd_box(dev, 0, mailbox->dma, port, 0, + MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); + if (err) + goto out; + + outbox = mailbox->buf; + + MLX4_GET(field, outbox, QUERY_PORT_CUR_MAX_GID_OFFSET); + *gid_tbl_len = field; + + MLX4_GET(field, outbox, QUERY_PORT_CUR_MAX_PKEY_OFFSET); + *pkey_tbl_len = field; + +out: + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL(mlx4_get_slave_pkey_gid_tbl_len); + +int mlx4_map_cmd(struct mlx4_dev *dev, u16 op, struct mlx4_icm *icm, u64 virt) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_icm_iter iter; + __be64 *pages; + int lg; + int nent = 0; + int i; + int err = 0; + int ts = 0, tc = 0; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + pages = mailbox->buf; + + for (mlx4_icm_first(icm, &iter); + !mlx4_icm_last(&iter); + mlx4_icm_next(&iter)) { + /* + * We have to pass pages that are aligned to their + * size, so find the least significant 1 in the + * address or size and use that as our log2 size. + */ + lg = ffs(mlx4_icm_addr(&iter) | mlx4_icm_size(&iter)) - 1; + if (lg < MLX4_ICM_PAGE_SHIFT) { + mlx4_warn(dev, "Got FW area not aligned to %d (%llx/%lx)\n", + MLX4_ICM_PAGE_SIZE, + (unsigned long long) mlx4_icm_addr(&iter), + mlx4_icm_size(&iter)); + err = -EINVAL; + goto out; + } + + for (i = 0; i < mlx4_icm_size(&iter) >> lg; ++i) { + if (virt != -1) { + pages[nent * 2] = cpu_to_be64(virt); + virt += 1 << lg; + } + + pages[nent * 2 + 1] = + cpu_to_be64((mlx4_icm_addr(&iter) + (i << lg)) | + (lg - MLX4_ICM_PAGE_SHIFT)); + ts += 1 << (lg - 10); + ++tc; + + if (++nent == MLX4_MAILBOX_SIZE / 16) { + err = mlx4_cmd(dev, mailbox->dma, nent, 0, op, + MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); + if (err) + goto out; + nent = 0; + } + } + } + + if (nent) + err = mlx4_cmd(dev, mailbox->dma, nent, 0, op, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); + if (err) + goto out; + + switch (op) { + case MLX4_CMD_MAP_FA: + mlx4_dbg(dev, "Mapped %d chunks/%d KB for FW\n", tc, ts); + break; + case MLX4_CMD_MAP_ICM_AUX: + mlx4_dbg(dev, "Mapped %d chunks/%d KB for ICM aux\n", tc, ts); + break; + case MLX4_CMD_MAP_ICM: + mlx4_dbg(dev, "Mapped %d chunks/%d KB at %llx for ICM\n", + tc, ts, (unsigned long long) virt - (ts << 10)); + break; + } + +out: + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +int mlx4_MAP_FA(struct mlx4_dev *dev, struct mlx4_icm *icm) +{ + return mlx4_map_cmd(dev, MLX4_CMD_MAP_FA, icm, -1); +} + +int mlx4_UNMAP_FA(struct mlx4_dev *dev) +{ + return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_UNMAP_FA, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); +} + + +int mlx4_RUN_FW(struct mlx4_dev *dev) +{ + return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_RUN_FW, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); +} + +int mlx4_QUERY_FW(struct mlx4_dev *dev) +{ + struct mlx4_fw *fw = &mlx4_priv(dev)->fw; + struct mlx4_cmd *cmd = &mlx4_priv(dev)->cmd; + struct mlx4_cmd_mailbox *mailbox; + u32 *outbox; + int err = 0; + u64 fw_ver; + u16 cmd_if_rev; + u8 lg; + +#define QUERY_FW_OUT_SIZE 0x100 +#define QUERY_FW_VER_OFFSET 0x00 +#define QUERY_FW_PPF_ID 0x09 +#define QUERY_FW_CMD_IF_REV_OFFSET 0x0a +#define QUERY_FW_MAX_CMD_OFFSET 0x0f +#define QUERY_FW_ERR_START_OFFSET 0x30 +#define QUERY_FW_ERR_SIZE_OFFSET 0x38 +#define QUERY_FW_ERR_BAR_OFFSET 0x3c + +#define QUERY_FW_SIZE_OFFSET 0x00 +#define QUERY_FW_CLR_INT_BASE_OFFSET 0x20 +#define QUERY_FW_CLR_INT_BAR_OFFSET 0x28 + +#define QUERY_FW_COMM_BASE_OFFSET 0x40 +#define QUERY_FW_COMM_BAR_OFFSET 0x48 + +#define QUERY_FW_CLOCK_OFFSET 0x50 +#define QUERY_FW_CLOCK_BAR 0x58 + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + outbox = mailbox->buf; + + err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, MLX4_CMD_QUERY_FW, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + if (err) + goto out; + + MLX4_GET(fw_ver, outbox, QUERY_FW_VER_OFFSET); + /* + * FW subminor version is at more significant bits than minor + * version, so swap here. + */ + dev->caps.fw_ver = (fw_ver & 0xffff00000000ull) | + ((fw_ver & 0xffff0000ull) >> 16) | + ((fw_ver & 0x0000ffffull) << 16); + + MLX4_GET(lg, outbox, QUERY_FW_PPF_ID); + dev->caps.function = lg; + + if (mlx4_is_slave(dev)) + goto out; + + + MLX4_GET(cmd_if_rev, outbox, QUERY_FW_CMD_IF_REV_OFFSET); + if (cmd_if_rev < MLX4_COMMAND_INTERFACE_MIN_REV || + cmd_if_rev > MLX4_COMMAND_INTERFACE_MAX_REV) { + mlx4_err(dev, "Installed FW has unsupported command interface revision %d\n", + cmd_if_rev); + mlx4_err(dev, "(Installed FW version is %d.%d.%03d)\n", + (int) (dev->caps.fw_ver >> 32), + (int) (dev->caps.fw_ver >> 16) & 0xffff, + (int) dev->caps.fw_ver & 0xffff); + mlx4_err(dev, "This driver version supports only revisions %d to %d\n", + MLX4_COMMAND_INTERFACE_MIN_REV, MLX4_COMMAND_INTERFACE_MAX_REV); + err = -ENODEV; + goto out; + } + + if (cmd_if_rev < MLX4_COMMAND_INTERFACE_NEW_PORT_CMDS) + dev->flags |= MLX4_FLAG_OLD_PORT_CMDS; + + MLX4_GET(lg, outbox, QUERY_FW_MAX_CMD_OFFSET); + cmd->max_cmds = 1 << lg; + + mlx4_dbg(dev, "FW version %d.%d.%03d (cmd intf rev %d), max commands %d\n", + (int) (dev->caps.fw_ver >> 32), + (int) (dev->caps.fw_ver >> 16) & 0xffff, + (int) dev->caps.fw_ver & 0xffff, + cmd_if_rev, cmd->max_cmds); + + MLX4_GET(fw->catas_offset, outbox, QUERY_FW_ERR_START_OFFSET); + MLX4_GET(fw->catas_size, outbox, QUERY_FW_ERR_SIZE_OFFSET); + MLX4_GET(fw->catas_bar, outbox, QUERY_FW_ERR_BAR_OFFSET); + fw->catas_bar = (fw->catas_bar >> 6) * 2; + + mlx4_dbg(dev, "Catastrophic error buffer at 0x%llx, size 0x%x, BAR %d\n", + (unsigned long long) fw->catas_offset, fw->catas_size, fw->catas_bar); + + MLX4_GET(fw->fw_pages, outbox, QUERY_FW_SIZE_OFFSET); + MLX4_GET(fw->clr_int_base, outbox, QUERY_FW_CLR_INT_BASE_OFFSET); + MLX4_GET(fw->clr_int_bar, outbox, QUERY_FW_CLR_INT_BAR_OFFSET); + fw->clr_int_bar = (fw->clr_int_bar >> 6) * 2; + + MLX4_GET(fw->comm_base, outbox, QUERY_FW_COMM_BASE_OFFSET); + MLX4_GET(fw->comm_bar, outbox, QUERY_FW_COMM_BAR_OFFSET); + fw->comm_bar = (fw->comm_bar >> 6) * 2; + mlx4_dbg(dev, "Communication vector bar:%d offset:0x%llx\n", + fw->comm_bar, (unsigned long long)fw->comm_base); + mlx4_dbg(dev, "FW size %d KB\n", fw->fw_pages >> 2); + + MLX4_GET(fw->clock_offset, outbox, QUERY_FW_CLOCK_OFFSET); + MLX4_GET(fw->clock_bar, outbox, QUERY_FW_CLOCK_BAR); + fw->clock_bar = (fw->clock_bar >> 6) * 2; + mlx4_dbg(dev, "Internal clock bar:%d offset:0x%llx\n", + fw->clock_bar, (unsigned long long)fw->clock_offset); + + /* + * Round up number of system pages needed in case + * MLX4_ICM_PAGE_SIZE < PAGE_SIZE. + */ + fw->fw_pages = + ALIGN(fw->fw_pages, PAGE_SIZE / MLX4_ICM_PAGE_SIZE) >> + (PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT); + + mlx4_dbg(dev, "Clear int @ %llx, BAR %d\n", + (unsigned long long) fw->clr_int_base, fw->clr_int_bar); + +out: + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +int mlx4_QUERY_FW_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + u8 *outbuf; + int err; + + outbuf = outbox->buf; + err = mlx4_cmd_box(dev, 0, outbox->dma, 0, 0, MLX4_CMD_QUERY_FW, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + if (err) + return err; + + /* for slaves, set pci PPF ID to invalid and zero out everything + * else except FW version */ + outbuf[0] = outbuf[1] = 0; + memset(&outbuf[8], 0, QUERY_FW_OUT_SIZE - 8); + outbuf[QUERY_FW_PPF_ID] = MLX4_INVALID_SLAVE_ID; + + return 0; +} + +static void get_board_id(void *vsd, char *board_id) +{ + int i; + +#define VSD_OFFSET_SIG1 0x00 +#define VSD_OFFSET_SIG2 0xde +#define VSD_OFFSET_MLX_BOARD_ID 0xd0 +#define VSD_OFFSET_TS_BOARD_ID 0x20 + +#define VSD_SIGNATURE_TOPSPIN 0x5ad + + memset(board_id, 0, MLX4_BOARD_ID_LEN); + + if (be16_to_cpup(vsd + VSD_OFFSET_SIG1) == VSD_SIGNATURE_TOPSPIN && + be16_to_cpup(vsd + VSD_OFFSET_SIG2) == VSD_SIGNATURE_TOPSPIN) { + strlcpy(board_id, vsd + VSD_OFFSET_TS_BOARD_ID, MLX4_BOARD_ID_LEN); + } else { + /* + * The board ID is a string but the firmware byte + * swaps each 4-byte word before passing it back to + * us. Therefore we need to swab it before printing. + */ + u32 *bid_u32 = (u32 *)board_id; + + for (i = 0; i < 4; ++i) { + typedef struct { u32 value; } __packed u64_p_t; + + u32 *addr; + u32 val; + + addr = (u32 *) (vsd + VSD_OFFSET_MLX_BOARD_ID + i * 4); + val = ((u64_p_t *)addr)->value; + val = swab32(val); + ((u64_p_t *)&bid_u32[i])->value = val; + } + } +} + +int mlx4_QUERY_ADAPTER(struct mlx4_dev *dev, struct mlx4_adapter *adapter) +{ + struct mlx4_cmd_mailbox *mailbox; + u32 *outbox; + int err; + +#define QUERY_ADAPTER_OUT_SIZE 0x100 +#define QUERY_ADAPTER_INTA_PIN_OFFSET 0x10 +#define QUERY_ADAPTER_VSD_OFFSET 0x20 + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + outbox = mailbox->buf; + + err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, MLX4_CMD_QUERY_ADAPTER, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + if (err) + goto out; + + MLX4_GET(adapter->inta_pin, outbox, QUERY_ADAPTER_INTA_PIN_OFFSET); + + get_board_id(outbox + QUERY_ADAPTER_VSD_OFFSET / 4, + adapter->board_id); + +out: + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +int mlx4_INIT_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param) +{ + struct mlx4_cmd_mailbox *mailbox; + __be32 *inbox; + int err; + static const u8 a0_dmfs_hw_steering[] = { + [MLX4_STEERING_DMFS_A0_DEFAULT] = 0, + [MLX4_STEERING_DMFS_A0_DYNAMIC] = 1, + [MLX4_STEERING_DMFS_A0_STATIC] = 2, + [MLX4_STEERING_DMFS_A0_DISABLE] = 3 + }; + +#define INIT_HCA_IN_SIZE 0x200 +#define INIT_HCA_VERSION_OFFSET 0x000 +#define INIT_HCA_VERSION 2 +#define INIT_HCA_VXLAN_OFFSET 0x0c +#define INIT_HCA_CACHELINE_SZ_OFFSET 0x0e +#define INIT_HCA_FLAGS_OFFSET 0x014 +#define INIT_HCA_RECOVERABLE_ERROR_EVENT_OFFSET 0x018 +#define INIT_HCA_QPC_OFFSET 0x020 +#define INIT_HCA_QPC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x10) +#define INIT_HCA_LOG_QP_OFFSET (INIT_HCA_QPC_OFFSET + 0x17) +#define INIT_HCA_SRQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x28) +#define INIT_HCA_LOG_SRQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x2f) +#define INIT_HCA_CQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x30) +#define INIT_HCA_LOG_CQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x37) +#define INIT_HCA_EQE_CQE_OFFSETS (INIT_HCA_QPC_OFFSET + 0x38) +#define INIT_HCA_EQE_CQE_STRIDE_OFFSET (INIT_HCA_QPC_OFFSET + 0x3b) +#define INIT_HCA_ALTC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x40) +#define INIT_HCA_AUXC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x50) +#define INIT_HCA_EQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x60) +#define INIT_HCA_LOG_EQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x67) +#define INIT_HCA_NUM_SYS_EQS_OFFSET (INIT_HCA_QPC_OFFSET + 0x6a) +#define INIT_HCA_RDMARC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x70) +#define INIT_HCA_LOG_RD_OFFSET (INIT_HCA_QPC_OFFSET + 0x77) +#define INIT_HCA_MCAST_OFFSET 0x0c0 +#define INIT_HCA_MC_BASE_OFFSET (INIT_HCA_MCAST_OFFSET + 0x00) +#define INIT_HCA_LOG_MC_ENTRY_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x12) +#define INIT_HCA_LOG_MC_HASH_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x16) +#define INIT_HCA_UC_STEERING_OFFSET (INIT_HCA_MCAST_OFFSET + 0x18) +#define INIT_HCA_LOG_MC_TABLE_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x1b) +#define INIT_HCA_DEVICE_MANAGED_FLOW_STEERING_EN 0x6 +#define INIT_HCA_FS_PARAM_OFFSET 0x1d0 +#define INIT_HCA_FS_BASE_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x00) +#define INIT_HCA_FS_LOG_ENTRY_SZ_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x12) +#define INIT_HCA_FS_A0_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x18) +#define INIT_HCA_FS_LOG_TABLE_SZ_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x1b) +#define INIT_HCA_FS_ETH_BITS_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x21) +#define INIT_HCA_FS_ETH_NUM_ADDRS_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x22) +#define INIT_HCA_FS_IB_BITS_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x25) +#define INIT_HCA_FS_IB_NUM_ADDRS_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x26) +#define INIT_HCA_TPT_OFFSET 0x0f0 +#define INIT_HCA_DMPT_BASE_OFFSET (INIT_HCA_TPT_OFFSET + 0x00) +#define INIT_HCA_TPT_MW_OFFSET (INIT_HCA_TPT_OFFSET + 0x08) +#define INIT_HCA_LOG_MPT_SZ_OFFSET (INIT_HCA_TPT_OFFSET + 0x0b) +#define INIT_HCA_MTT_BASE_OFFSET (INIT_HCA_TPT_OFFSET + 0x10) +#define INIT_HCA_CMPT_BASE_OFFSET (INIT_HCA_TPT_OFFSET + 0x18) +#define INIT_HCA_UAR_OFFSET 0x120 +#define INIT_HCA_LOG_UAR_SZ_OFFSET (INIT_HCA_UAR_OFFSET + 0x0a) +#define INIT_HCA_UAR_PAGE_SZ_OFFSET (INIT_HCA_UAR_OFFSET + 0x0b) + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + inbox = mailbox->buf; + + *((u8 *) mailbox->buf + INIT_HCA_VERSION_OFFSET) = INIT_HCA_VERSION; + + *((u8 *) mailbox->buf + INIT_HCA_CACHELINE_SZ_OFFSET) = + (ilog2(cache_line_size()) - 4) << 5; + +#if defined(__LITTLE_ENDIAN) + *(inbox + INIT_HCA_FLAGS_OFFSET / 4) &= ~cpu_to_be32(1 << 1); +#elif defined(__BIG_ENDIAN) + *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 1); +#else +#error Host endianness not defined +#endif + /* Check port for UD address vector: */ + *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1); + + /* Enable IPoIB checksumming if we can: */ + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_IPOIB_CSUM) + *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 3); + + /* Enable QoS support if module parameter set */ + if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETS_CFG && enable_qos) + *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 2); + + /* enable counters */ + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS) + *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 4); + + /* Enable RSS spread to fragmented IP packets when supported */ + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_RSS_IP_FRAG) + *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 13); + + /* CX3 is capable of extending CQEs/EQEs from 32 to 64 bytes */ + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_64B_EQE) { + *(inbox + INIT_HCA_EQE_CQE_OFFSETS / 4) |= cpu_to_be32(1 << 29); + dev->caps.eqe_size = 64; + dev->caps.eqe_factor = 1; + } else { + dev->caps.eqe_size = 32; + dev->caps.eqe_factor = 0; + } + + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_64B_CQE) { + *(inbox + INIT_HCA_EQE_CQE_OFFSETS / 4) |= cpu_to_be32(1 << 30); + dev->caps.cqe_size = 64; + dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_LARGE_CQE; + } else { + dev->caps.cqe_size = 32; + } + +#if 0 + /* XXX not currently supported by the FreeBSD's mlxen */ + /* CX3 is capable of extending CQEs\EQEs to strides larger than 64B */ + if ((dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_EQE_STRIDE) && + (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_CQE_STRIDE)) { + dev->caps.eqe_size = cache_line_size(); + dev->caps.cqe_size = cache_line_size(); + dev->caps.eqe_factor = 0; + MLX4_PUT(inbox, (u8)((ilog2(dev->caps.eqe_size) - 5) << 4 | + (ilog2(dev->caps.eqe_size) - 5)), + INIT_HCA_EQE_CQE_STRIDE_OFFSET); + + /* User still need to know to support CQE > 32B */ + dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_LARGE_CQE; + } +#endif + + if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RECOVERABLE_ERROR_EVENT) + *(inbox + INIT_HCA_RECOVERABLE_ERROR_EVENT_OFFSET / 4) |= cpu_to_be32(1 << 31); + + /* QPC/EEC/CQC/EQC/RDMARC attributes */ + + MLX4_PUT(inbox, param->qpc_base, INIT_HCA_QPC_BASE_OFFSET); + MLX4_PUT(inbox, param->log_num_qps, INIT_HCA_LOG_QP_OFFSET); + MLX4_PUT(inbox, param->srqc_base, INIT_HCA_SRQC_BASE_OFFSET); + MLX4_PUT(inbox, param->log_num_srqs, INIT_HCA_LOG_SRQ_OFFSET); + MLX4_PUT(inbox, param->cqc_base, INIT_HCA_CQC_BASE_OFFSET); + MLX4_PUT(inbox, param->log_num_cqs, INIT_HCA_LOG_CQ_OFFSET); + MLX4_PUT(inbox, param->altc_base, INIT_HCA_ALTC_BASE_OFFSET); + MLX4_PUT(inbox, param->auxc_base, INIT_HCA_AUXC_BASE_OFFSET); + MLX4_PUT(inbox, param->eqc_base, INIT_HCA_EQC_BASE_OFFSET); + MLX4_PUT(inbox, param->log_num_eqs, INIT_HCA_LOG_EQ_OFFSET); + MLX4_PUT(inbox, param->num_sys_eqs, INIT_HCA_NUM_SYS_EQS_OFFSET); + MLX4_PUT(inbox, param->rdmarc_base, INIT_HCA_RDMARC_BASE_OFFSET); + MLX4_PUT(inbox, param->log_rd_per_qp, INIT_HCA_LOG_RD_OFFSET); + + /* steering attributes */ + if (dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) { + *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= + cpu_to_be32(1 << + INIT_HCA_DEVICE_MANAGED_FLOW_STEERING_EN); + + MLX4_PUT(inbox, param->mc_base, INIT_HCA_FS_BASE_OFFSET); + MLX4_PUT(inbox, param->log_mc_entry_sz, + INIT_HCA_FS_LOG_ENTRY_SZ_OFFSET); + MLX4_PUT(inbox, param->log_mc_table_sz, + INIT_HCA_FS_LOG_TABLE_SZ_OFFSET); + /* Enable Ethernet flow steering + * with udp unicast and tcp unicast + */ + if (dev->caps.dmfs_high_steer_mode != + MLX4_STEERING_DMFS_A0_STATIC) + MLX4_PUT(inbox, + (u8)(MLX4_FS_UDP_UC_EN | MLX4_FS_TCP_UC_EN), + INIT_HCA_FS_ETH_BITS_OFFSET); + MLX4_PUT(inbox, (u16) MLX4_FS_NUM_OF_L2_ADDR, + INIT_HCA_FS_ETH_NUM_ADDRS_OFFSET); + /* Enable IPoIB flow steering + * with udp unicast and tcp unicast + */ + MLX4_PUT(inbox, (u8) (MLX4_FS_UDP_UC_EN | MLX4_FS_TCP_UC_EN), + INIT_HCA_FS_IB_BITS_OFFSET); + MLX4_PUT(inbox, (u16) MLX4_FS_NUM_OF_L2_ADDR, + INIT_HCA_FS_IB_NUM_ADDRS_OFFSET); + + if (dev->caps.dmfs_high_steer_mode != + MLX4_STEERING_DMFS_A0_NOT_SUPPORTED) + MLX4_PUT(inbox, + ((u8)(a0_dmfs_hw_steering[dev->caps.dmfs_high_steer_mode] + << 6)), + INIT_HCA_FS_A0_OFFSET); + } else { + MLX4_PUT(inbox, param->mc_base, INIT_HCA_MC_BASE_OFFSET); + MLX4_PUT(inbox, param->log_mc_entry_sz, + INIT_HCA_LOG_MC_ENTRY_SZ_OFFSET); + MLX4_PUT(inbox, param->log_mc_hash_sz, + INIT_HCA_LOG_MC_HASH_SZ_OFFSET); + MLX4_PUT(inbox, param->log_mc_table_sz, + INIT_HCA_LOG_MC_TABLE_SZ_OFFSET); + if (dev->caps.steering_mode == MLX4_STEERING_MODE_B0) + MLX4_PUT(inbox, (u8) (1 << 3), + INIT_HCA_UC_STEERING_OFFSET); + } + + /* TPT attributes */ + + MLX4_PUT(inbox, param->dmpt_base, INIT_HCA_DMPT_BASE_OFFSET); + MLX4_PUT(inbox, param->mw_enabled, INIT_HCA_TPT_MW_OFFSET); + MLX4_PUT(inbox, param->log_mpt_sz, INIT_HCA_LOG_MPT_SZ_OFFSET); + MLX4_PUT(inbox, param->mtt_base, INIT_HCA_MTT_BASE_OFFSET); + MLX4_PUT(inbox, param->cmpt_base, INIT_HCA_CMPT_BASE_OFFSET); + + /* UAR attributes */ + + MLX4_PUT(inbox, param->uar_page_sz, INIT_HCA_UAR_PAGE_SZ_OFFSET); + MLX4_PUT(inbox, param->log_uar_sz, INIT_HCA_LOG_UAR_SZ_OFFSET); + + /* set parser VXLAN attributes */ + if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_VXLAN_OFFLOADS) { + u8 parser_params = 0; + MLX4_PUT(inbox, parser_params, INIT_HCA_VXLAN_OFFSET); + } + + err = mlx4_cmd(dev, mailbox->dma, 0, 0, MLX4_CMD_INIT_HCA, + MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); + + if (err) + mlx4_err(dev, "INIT_HCA returns %d\n", err); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +int mlx4_QUERY_HCA(struct mlx4_dev *dev, + struct mlx4_init_hca_param *param) +{ + struct mlx4_cmd_mailbox *mailbox; + __be32 *outbox; + u32 dword_field; + int err; + u8 byte_field; + static const u8 a0_dmfs_query_hw_steering[] = { + [0] = MLX4_STEERING_DMFS_A0_DEFAULT, + [1] = MLX4_STEERING_DMFS_A0_DYNAMIC, + [2] = MLX4_STEERING_DMFS_A0_STATIC, + [3] = MLX4_STEERING_DMFS_A0_DISABLE + }; + +#define QUERY_HCA_GLOBAL_CAPS_OFFSET 0x04 +#define QUERY_HCA_CORE_CLOCK_OFFSET 0x0c + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + outbox = mailbox->buf; + + err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, + MLX4_CMD_QUERY_HCA, + MLX4_CMD_TIME_CLASS_B, + !mlx4_is_slave(dev)); + if (err) + goto out; + + MLX4_GET(param->global_caps, outbox, QUERY_HCA_GLOBAL_CAPS_OFFSET); + MLX4_GET(param->hca_core_clock, outbox, QUERY_HCA_CORE_CLOCK_OFFSET); + + /* QPC/EEC/CQC/EQC/RDMARC attributes */ + + MLX4_GET(param->qpc_base, outbox, INIT_HCA_QPC_BASE_OFFSET); + MLX4_GET(param->log_num_qps, outbox, INIT_HCA_LOG_QP_OFFSET); + MLX4_GET(param->srqc_base, outbox, INIT_HCA_SRQC_BASE_OFFSET); + MLX4_GET(param->log_num_srqs, outbox, INIT_HCA_LOG_SRQ_OFFSET); + MLX4_GET(param->cqc_base, outbox, INIT_HCA_CQC_BASE_OFFSET); + MLX4_GET(param->log_num_cqs, outbox, INIT_HCA_LOG_CQ_OFFSET); + MLX4_GET(param->altc_base, outbox, INIT_HCA_ALTC_BASE_OFFSET); + MLX4_GET(param->auxc_base, outbox, INIT_HCA_AUXC_BASE_OFFSET); + MLX4_GET(param->eqc_base, outbox, INIT_HCA_EQC_BASE_OFFSET); + MLX4_GET(param->log_num_eqs, outbox, INIT_HCA_LOG_EQ_OFFSET); + MLX4_GET(param->num_sys_eqs, outbox, INIT_HCA_NUM_SYS_EQS_OFFSET); + MLX4_GET(param->rdmarc_base, outbox, INIT_HCA_RDMARC_BASE_OFFSET); + MLX4_GET(param->log_rd_per_qp, outbox, INIT_HCA_LOG_RD_OFFSET); + + MLX4_GET(dword_field, outbox, INIT_HCA_FLAGS_OFFSET); + if (dword_field & (1 << INIT_HCA_DEVICE_MANAGED_FLOW_STEERING_EN)) { + param->steering_mode = MLX4_STEERING_MODE_DEVICE_MANAGED; + } else { + MLX4_GET(byte_field, outbox, INIT_HCA_UC_STEERING_OFFSET); + if (byte_field & 0x8) + param->steering_mode = MLX4_STEERING_MODE_B0; + else + param->steering_mode = MLX4_STEERING_MODE_A0; + } + + if (dword_field & (1 << 13)) + param->rss_ip_frags = 1; + + /* steering attributes */ + if (param->steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) { + MLX4_GET(param->mc_base, outbox, INIT_HCA_FS_BASE_OFFSET); + MLX4_GET(param->log_mc_entry_sz, outbox, + INIT_HCA_FS_LOG_ENTRY_SZ_OFFSET); + MLX4_GET(param->log_mc_table_sz, outbox, + INIT_HCA_FS_LOG_TABLE_SZ_OFFSET); + MLX4_GET(byte_field, outbox, + INIT_HCA_FS_A0_OFFSET); + param->dmfs_high_steer_mode = + a0_dmfs_query_hw_steering[(byte_field >> 6) & 3]; + } else { + MLX4_GET(param->mc_base, outbox, INIT_HCA_MC_BASE_OFFSET); + MLX4_GET(param->log_mc_entry_sz, outbox, + INIT_HCA_LOG_MC_ENTRY_SZ_OFFSET); + MLX4_GET(param->log_mc_hash_sz, outbox, + INIT_HCA_LOG_MC_HASH_SZ_OFFSET); + MLX4_GET(param->log_mc_table_sz, outbox, + INIT_HCA_LOG_MC_TABLE_SZ_OFFSET); + } + + /* CX3 is capable of extending CQEs/EQEs from 32 to 64 bytes */ + MLX4_GET(byte_field, outbox, INIT_HCA_EQE_CQE_OFFSETS); + if (byte_field & 0x20) /* 64-bytes eqe enabled */ + param->dev_cap_enabled |= MLX4_DEV_CAP_64B_EQE_ENABLED; + if (byte_field & 0x40) /* 64-bytes cqe enabled */ + param->dev_cap_enabled |= MLX4_DEV_CAP_64B_CQE_ENABLED; + + /* CX3 is capable of extending CQEs\EQEs to strides larger than 64B */ + MLX4_GET(byte_field, outbox, INIT_HCA_EQE_CQE_STRIDE_OFFSET); + if (byte_field) { + param->dev_cap_enabled |= MLX4_DEV_CAP_EQE_STRIDE_ENABLED; + param->dev_cap_enabled |= MLX4_DEV_CAP_CQE_STRIDE_ENABLED; + param->cqe_size = 1 << ((byte_field & + MLX4_CQE_SIZE_MASK_STRIDE) + 5); + param->eqe_size = 1 << (((byte_field & + MLX4_EQE_SIZE_MASK_STRIDE) >> 4) + 5); + } + + /* TPT attributes */ + + MLX4_GET(param->dmpt_base, outbox, INIT_HCA_DMPT_BASE_OFFSET); + MLX4_GET(param->mw_enabled, outbox, INIT_HCA_TPT_MW_OFFSET); + MLX4_GET(param->log_mpt_sz, outbox, INIT_HCA_LOG_MPT_SZ_OFFSET); + MLX4_GET(param->mtt_base, outbox, INIT_HCA_MTT_BASE_OFFSET); + MLX4_GET(param->cmpt_base, outbox, INIT_HCA_CMPT_BASE_OFFSET); + + /* UAR attributes */ + + MLX4_GET(param->uar_page_sz, outbox, INIT_HCA_UAR_PAGE_SZ_OFFSET); + MLX4_GET(param->log_uar_sz, outbox, INIT_HCA_LOG_UAR_SZ_OFFSET); + + /* phv_check enable */ + MLX4_GET(byte_field, outbox, INIT_HCA_CACHELINE_SZ_OFFSET); + if (byte_field & 0x2) + param->phv_check_en = 1; +out: + mlx4_free_cmd_mailbox(dev, mailbox); + + return err; +} + +static int mlx4_hca_core_clock_update(struct mlx4_dev *dev) +{ + struct mlx4_cmd_mailbox *mailbox; + __be32 *outbox; + int err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + mlx4_warn(dev, "hca_core_clock mailbox allocation failed\n"); + return PTR_ERR(mailbox); + } + outbox = mailbox->buf; + + err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, + MLX4_CMD_QUERY_HCA, + MLX4_CMD_TIME_CLASS_B, + !mlx4_is_slave(dev)); + if (err) { + mlx4_warn(dev, "hca_core_clock update failed\n"); + goto out; + } + + MLX4_GET(dev->caps.hca_core_clock, outbox, QUERY_HCA_CORE_CLOCK_OFFSET); + +out: + mlx4_free_cmd_mailbox(dev, mailbox); + + return err; +} + +/* for IB-type ports only in SRIOV mode. Checks that both proxy QP0 + * and real QP0 are active, so that the paravirtualized QP0 is ready + * to operate */ +static int check_qp0_state(struct mlx4_dev *dev, int function, int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + /* irrelevant if not infiniband */ + if (priv->mfunc.master.qp0_state[port].proxy_qp0_active && + priv->mfunc.master.qp0_state[port].qp0_active) + return 1; + return 0; +} + +int mlx4_INIT_PORT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int port = mlx4_slave_convert_port(dev, slave, vhcr->in_modifier); + int err; + + if (port < 0) + return -EINVAL; + + if (priv->mfunc.master.slave_state[slave].init_port_mask & (1 << port)) + return 0; + + if (dev->caps.port_mask[port] != MLX4_PORT_TYPE_IB) { + /* Enable port only if it was previously disabled */ + if (!priv->mfunc.master.init_port_ref[port]) { + err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_INIT_PORT, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + if (err) + return err; + } + priv->mfunc.master.slave_state[slave].init_port_mask |= (1 << port); + } else { + if (slave == mlx4_master_func_num(dev)) { + if (check_qp0_state(dev, slave, port) && + !priv->mfunc.master.qp0_state[port].port_active) { + err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_INIT_PORT, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + if (err) + return err; + priv->mfunc.master.qp0_state[port].port_active = 1; + priv->mfunc.master.slave_state[slave].init_port_mask |= (1 << port); + } + } else + priv->mfunc.master.slave_state[slave].init_port_mask |= (1 << port); + } + ++priv->mfunc.master.init_port_ref[port]; + return 0; +} + +int mlx4_INIT_PORT(struct mlx4_dev *dev, int port) +{ + struct mlx4_cmd_mailbox *mailbox; + u32 *inbox; + int err; + u32 flags; + u16 field; + + if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { +#define INIT_PORT_IN_SIZE 256 +#define INIT_PORT_FLAGS_OFFSET 0x00 +#define INIT_PORT_FLAG_SIG (1 << 18) +#define INIT_PORT_FLAG_NG (1 << 17) +#define INIT_PORT_FLAG_G0 (1 << 16) +#define INIT_PORT_VL_SHIFT 4 +#define INIT_PORT_PORT_WIDTH_SHIFT 8 +#define INIT_PORT_MTU_OFFSET 0x04 +#define INIT_PORT_MAX_GID_OFFSET 0x06 +#define INIT_PORT_MAX_PKEY_OFFSET 0x0a +#define INIT_PORT_GUID0_OFFSET 0x10 +#define INIT_PORT_NODE_GUID_OFFSET 0x18 +#define INIT_PORT_SI_GUID_OFFSET 0x20 + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + inbox = mailbox->buf; + + flags = 0; + flags |= (dev->caps.vl_cap[port] & 0xf) << INIT_PORT_VL_SHIFT; + flags |= (dev->caps.port_width_cap[port] & 0xf) << INIT_PORT_PORT_WIDTH_SHIFT; + MLX4_PUT(inbox, flags, INIT_PORT_FLAGS_OFFSET); + + field = 128 << dev->caps.ib_mtu_cap[port]; + MLX4_PUT(inbox, field, INIT_PORT_MTU_OFFSET); + field = dev->caps.gid_table_len[port]; + MLX4_PUT(inbox, field, INIT_PORT_MAX_GID_OFFSET); + field = dev->caps.pkey_table_len[port]; + MLX4_PUT(inbox, field, INIT_PORT_MAX_PKEY_OFFSET); + + err = mlx4_cmd(dev, mailbox->dma, port, 0, MLX4_CMD_INIT_PORT, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, mailbox); + } else + err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_INIT_PORT, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + + if (!err) + mlx4_hca_core_clock_update(dev); + + return err; +} +EXPORT_SYMBOL_GPL(mlx4_INIT_PORT); + +int mlx4_CLOSE_PORT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int port = mlx4_slave_convert_port(dev, slave, vhcr->in_modifier); + int err; + + if (port < 0) + return -EINVAL; + + if (!(priv->mfunc.master.slave_state[slave].init_port_mask & + (1 << port))) + return 0; + + if (dev->caps.port_mask[port] != MLX4_PORT_TYPE_IB) { + if (priv->mfunc.master.init_port_ref[port] == 1) { + err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_CLOSE_PORT, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + if (err) + return err; + } + priv->mfunc.master.slave_state[slave].init_port_mask &= ~(1 << port); + } else { + /* infiniband port */ + if (slave == mlx4_master_func_num(dev)) { + if (!priv->mfunc.master.qp0_state[port].qp0_active && + priv->mfunc.master.qp0_state[port].port_active) { + err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_CLOSE_PORT, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + if (err) + return err; + priv->mfunc.master.slave_state[slave].init_port_mask &= ~(1 << port); + priv->mfunc.master.qp0_state[port].port_active = 0; + } + } else + priv->mfunc.master.slave_state[slave].init_port_mask &= ~(1 << port); + } + --priv->mfunc.master.init_port_ref[port]; + return 0; +} + +int mlx4_CLOSE_PORT(struct mlx4_dev *dev, int port) +{ + return mlx4_cmd(dev, 0, port, 0, MLX4_CMD_CLOSE_PORT, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); +} +EXPORT_SYMBOL_GPL(mlx4_CLOSE_PORT); + +int mlx4_CLOSE_HCA(struct mlx4_dev *dev, int panic) +{ + return mlx4_cmd(dev, 0, 0, panic, MLX4_CMD_CLOSE_HCA, + MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); +} + +struct mlx4_config_dev { + __be32 update_flags; + __be32 rsvd1[3]; + __be16 vxlan_udp_dport; + __be16 rsvd2; + __be16 roce_v2_entropy; + __be16 roce_v2_udp_dport; + __be32 roce_flags; + __be32 rsvd4[25]; + __be16 rsvd5; + u8 rsvd6; + u8 rx_checksum_val; +}; + +#define MLX4_VXLAN_UDP_DPORT (1 << 0) +#define MLX4_ROCE_V2_UDP_DPORT BIT(3) +#define MLX4_DISABLE_RX_PORT BIT(18) + +static int mlx4_CONFIG_DEV_set(struct mlx4_dev *dev, struct mlx4_config_dev *config_dev) +{ + int err; + struct mlx4_cmd_mailbox *mailbox; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + memcpy(mailbox->buf, config_dev, sizeof(*config_dev)); + + err = mlx4_cmd(dev, mailbox->dma, 0, 0, MLX4_CMD_CONFIG_DEV, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +static int mlx4_CONFIG_DEV_get(struct mlx4_dev *dev, struct mlx4_config_dev *config_dev) +{ + int err; + struct mlx4_cmd_mailbox *mailbox; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 1, MLX4_CMD_CONFIG_DEV, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + + if (!err) + memcpy(config_dev, mailbox->buf, sizeof(*config_dev)); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +/* Conversion between the HW values and the actual functionality. + * The value represented by the array index, + * and the functionality determined by the flags. + */ +static const u8 config_dev_csum_flags[] = { + [0] = 0, + [1] = MLX4_RX_CSUM_MODE_VAL_NON_TCP_UDP, + [2] = MLX4_RX_CSUM_MODE_VAL_NON_TCP_UDP | + MLX4_RX_CSUM_MODE_L4, + [3] = MLX4_RX_CSUM_MODE_L4 | + MLX4_RX_CSUM_MODE_IP_OK_IP_NON_TCP_UDP | + MLX4_RX_CSUM_MODE_MULTI_VLAN +}; + +int mlx4_config_dev_retrieval(struct mlx4_dev *dev, + struct mlx4_config_dev_params *params) +{ + struct mlx4_config_dev config_dev = {0}; + int err; + u8 csum_mask; + +#define CONFIG_DEV_RX_CSUM_MODE_MASK 0x7 +#define CONFIG_DEV_RX_CSUM_MODE_PORT1_BIT_OFFSET 0 +#define CONFIG_DEV_RX_CSUM_MODE_PORT2_BIT_OFFSET 4 + + if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_CONFIG_DEV)) + return -ENOTSUPP; + + err = mlx4_CONFIG_DEV_get(dev, &config_dev); + if (err) + return err; + + csum_mask = (config_dev.rx_checksum_val >> CONFIG_DEV_RX_CSUM_MODE_PORT1_BIT_OFFSET) & + CONFIG_DEV_RX_CSUM_MODE_MASK; + + if (csum_mask >= sizeof(config_dev_csum_flags)/sizeof(config_dev_csum_flags[0])) + return -EINVAL; + params->rx_csum_flags_port_1 = config_dev_csum_flags[csum_mask]; + + csum_mask = (config_dev.rx_checksum_val >> CONFIG_DEV_RX_CSUM_MODE_PORT2_BIT_OFFSET) & + CONFIG_DEV_RX_CSUM_MODE_MASK; + + if (csum_mask >= sizeof(config_dev_csum_flags)/sizeof(config_dev_csum_flags[0])) + return -EINVAL; + params->rx_csum_flags_port_2 = config_dev_csum_flags[csum_mask]; + + params->vxlan_udp_dport = be16_to_cpu(config_dev.vxlan_udp_dport); + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_config_dev_retrieval); + +int mlx4_config_vxlan_port(struct mlx4_dev *dev, __be16 udp_port) +{ + struct mlx4_config_dev config_dev; + + memset(&config_dev, 0, sizeof(config_dev)); + config_dev.update_flags = cpu_to_be32(MLX4_VXLAN_UDP_DPORT); + config_dev.vxlan_udp_dport = udp_port; + + return mlx4_CONFIG_DEV_set(dev, &config_dev); +} +EXPORT_SYMBOL_GPL(mlx4_config_vxlan_port); + +#define CONFIG_DISABLE_RX_PORT BIT(15) +int mlx4_disable_rx_port_check(struct mlx4_dev *dev, bool dis) +{ + struct mlx4_config_dev config_dev; + + memset(&config_dev, 0, sizeof(config_dev)); + config_dev.update_flags = cpu_to_be32(MLX4_DISABLE_RX_PORT); + if (dis) + config_dev.roce_flags = + cpu_to_be32(CONFIG_DISABLE_RX_PORT); + + return mlx4_CONFIG_DEV_set(dev, &config_dev); +} + +int mlx4_config_roce_v2_port(struct mlx4_dev *dev, u16 udp_port) +{ + struct mlx4_config_dev config_dev; + + memset(&config_dev, 0, sizeof(config_dev)); + config_dev.update_flags = cpu_to_be32(MLX4_ROCE_V2_UDP_DPORT); + config_dev.roce_v2_udp_dport = cpu_to_be16(udp_port); + + return mlx4_CONFIG_DEV_set(dev, &config_dev); +} +EXPORT_SYMBOL_GPL(mlx4_config_roce_v2_port); + +int mlx4_virt2phy_port_map(struct mlx4_dev *dev, u32 port1, u32 port2) +{ + struct mlx4_cmd_mailbox *mailbox; + struct { + __be32 v_port1; + __be32 v_port2; + } *v2p; + int err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return -ENOMEM; + + v2p = mailbox->buf; + v2p->v_port1 = cpu_to_be32(port1); + v2p->v_port2 = cpu_to_be32(port2); + + err = mlx4_cmd(dev, mailbox->dma, 0, + MLX4_SET_PORT_VIRT2PHY, MLX4_CMD_VIRT_PORT_MAP, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + + +int mlx4_SET_ICM_SIZE(struct mlx4_dev *dev, u64 icm_size, u64 *aux_pages) +{ + int ret = mlx4_cmd_imm(dev, icm_size, aux_pages, 0, 0, + MLX4_CMD_SET_ICM_SIZE, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + if (ret) + return ret; + + /* + * Round up number of system pages needed in case + * MLX4_ICM_PAGE_SIZE < PAGE_SIZE. + */ + *aux_pages = ALIGN(*aux_pages, PAGE_SIZE / MLX4_ICM_PAGE_SIZE) >> + (PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT); + + return 0; +} + +int mlx4_NOP(struct mlx4_dev *dev) +{ + /* Input modifier of 0x1f means "finish as soon as possible." */ + return mlx4_cmd(dev, 0, 0x1f, 0, MLX4_CMD_NOP, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); +} + +int mlx4_query_diag_counters(struct mlx4_dev *dev, u8 op_modifier, + const u32 offset[], + u32 value[], size_t array_len, u8 port) +{ + struct mlx4_cmd_mailbox *mailbox; + u32 *outbox; + size_t i; + int ret; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + outbox = mailbox->buf; + + ret = mlx4_cmd_box(dev, 0, mailbox->dma, port, op_modifier, + MLX4_CMD_DIAG_RPRT, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (ret) + goto out; + + for (i = 0; i < array_len; i++) { + if (offset[i] > MLX4_MAILBOX_SIZE) { + ret = -EINVAL; + goto out; + } + + MLX4_GET(value[i], outbox, offset[i]); + } + +out: + mlx4_free_cmd_mailbox(dev, mailbox); + return ret; +} +EXPORT_SYMBOL(mlx4_query_diag_counters); + +int mlx4_get_phys_port_id(struct mlx4_dev *dev) +{ + u8 port; + u32 *outbox; + struct mlx4_cmd_mailbox *mailbox; + u32 in_mod; + u32 guid_hi, guid_lo; + int err, ret = 0; +#define MOD_STAT_CFG_PORT_OFFSET 8 +#define MOD_STAT_CFG_GUID_H 0X14 +#define MOD_STAT_CFG_GUID_L 0X1c + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + outbox = mailbox->buf; + + for (port = 1; port <= dev->caps.num_ports; port++) { + in_mod = port << MOD_STAT_CFG_PORT_OFFSET; + err = mlx4_cmd_box(dev, 0, mailbox->dma, in_mod, 0x2, + MLX4_CMD_MOD_STAT_CFG, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) { + mlx4_err(dev, "Fail to get port %d uplink guid\n", + port); + ret = err; + } else { + MLX4_GET(guid_hi, outbox, MOD_STAT_CFG_GUID_H); + MLX4_GET(guid_lo, outbox, MOD_STAT_CFG_GUID_L); + dev->caps.phys_port_id[port] = (u64)guid_lo | + (u64)guid_hi << 32; + } + } + mlx4_free_cmd_mailbox(dev, mailbox); + return ret; +} + +#define MLX4_WOL_SETUP_MODE (5 << 28) +int mlx4_wol_read(struct mlx4_dev *dev, u64 *config, int port) +{ + u32 in_mod = MLX4_WOL_SETUP_MODE | port << 8; + + return mlx4_cmd_imm(dev, 0, config, in_mod, 0x3, + MLX4_CMD_MOD_STAT_CFG, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); +} +EXPORT_SYMBOL_GPL(mlx4_wol_read); + +int mlx4_wol_write(struct mlx4_dev *dev, u64 config, int port) +{ + u32 in_mod = MLX4_WOL_SETUP_MODE | port << 8; + + return mlx4_cmd(dev, config, in_mod, 0x1, MLX4_CMD_MOD_STAT_CFG, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); +} +EXPORT_SYMBOL_GPL(mlx4_wol_write); + +enum { + ADD_TO_MCG = 0x26, +}; + + +void mlx4_opreq_action(struct work_struct *work) +{ + struct mlx4_priv *priv = container_of(work, struct mlx4_priv, + opreq_task); + struct mlx4_dev *dev = &priv->dev; + int num_tasks = atomic_read(&priv->opreq_count); + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_mgm *mgm; + u32 *outbox; + u32 modifier; + u16 token; + u16 type; + int err; + u32 num_qps; + struct mlx4_qp qp; + int i; + u8 rem_mcg; + u8 prot; + +#define GET_OP_REQ_MODIFIER_OFFSET 0x08 +#define GET_OP_REQ_TOKEN_OFFSET 0x14 +#define GET_OP_REQ_TYPE_OFFSET 0x1a +#define GET_OP_REQ_DATA_OFFSET 0x20 + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + mlx4_err(dev, "Failed to allocate mailbox for GET_OP_REQ\n"); + return; + } + outbox = mailbox->buf; + + while (num_tasks) { + err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, + MLX4_CMD_GET_OP_REQ, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) { + mlx4_err(dev, "Failed to retrieve required operation: %d\n", + err); + return; + } + MLX4_GET(modifier, outbox, GET_OP_REQ_MODIFIER_OFFSET); + MLX4_GET(token, outbox, GET_OP_REQ_TOKEN_OFFSET); + MLX4_GET(type, outbox, GET_OP_REQ_TYPE_OFFSET); + type &= 0xfff; + + switch (type) { + case ADD_TO_MCG: + if (dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) { + mlx4_warn(dev, "ADD MCG operation is not supported in DEVICE_MANAGED steering mode\n"); + err = EPERM; + break; + } + mgm = (struct mlx4_mgm *)((u8 *)(outbox) + + GET_OP_REQ_DATA_OFFSET); + num_qps = be32_to_cpu(mgm->members_count) & + MGM_QPN_MASK; + rem_mcg = ((u8 *)(&mgm->members_count))[0] & 1; + prot = ((u8 *)(&mgm->members_count))[0] >> 6; + + for (i = 0; i < num_qps; i++) { + qp.qpn = be32_to_cpu(mgm->qp[i]); + if (rem_mcg) + err = mlx4_multicast_detach(dev, &qp, + mgm->gid, + prot, 0); + else + err = mlx4_multicast_attach(dev, &qp, + mgm->gid, + mgm->gid[5] + , 0, prot, + NULL); + if (err) + break; + } + break; + default: + mlx4_warn(dev, "Bad type for required operation\n"); + err = EINVAL; + break; + } + err = mlx4_cmd(dev, 0, ((u32) err | + (__force u32)cpu_to_be32(token) << 16), + 1, MLX4_CMD_GET_OP_REQ, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) { + mlx4_err(dev, "Failed to acknowledge required request: %d\n", + err); + goto out; + } + memset(outbox, 0, 0xffc); + num_tasks = atomic_dec_return(&priv->opreq_count); + } + +out: + mlx4_free_cmd_mailbox(dev, mailbox); +} + +static int mlx4_check_smp_firewall_active(struct mlx4_dev *dev, + struct mlx4_cmd_mailbox *mailbox) +{ +#define MLX4_CMD_MAD_DEMUX_SET_ATTR_OFFSET 0x10 +#define MLX4_CMD_MAD_DEMUX_GETRESP_ATTR_OFFSET 0x20 +#define MLX4_CMD_MAD_DEMUX_TRAP_ATTR_OFFSET 0x40 +#define MLX4_CMD_MAD_DEMUX_TRAP_REPRESS_ATTR_OFFSET 0x70 + + u32 set_attr_mask, getresp_attr_mask; + u32 trap_attr_mask, traprepress_attr_mask; + + MLX4_GET(set_attr_mask, mailbox->buf, + MLX4_CMD_MAD_DEMUX_SET_ATTR_OFFSET); + mlx4_dbg(dev, "SMP firewall set_attribute_mask = 0x%x\n", + set_attr_mask); + + MLX4_GET(getresp_attr_mask, mailbox->buf, + MLX4_CMD_MAD_DEMUX_GETRESP_ATTR_OFFSET); + mlx4_dbg(dev, "SMP firewall getresp_attribute_mask = 0x%x\n", + getresp_attr_mask); + + MLX4_GET(trap_attr_mask, mailbox->buf, + MLX4_CMD_MAD_DEMUX_TRAP_ATTR_OFFSET); + mlx4_dbg(dev, "SMP firewall trap_attribute_mask = 0x%x\n", + trap_attr_mask); + + MLX4_GET(traprepress_attr_mask, mailbox->buf, + MLX4_CMD_MAD_DEMUX_TRAP_REPRESS_ATTR_OFFSET); + mlx4_dbg(dev, "SMP firewall traprepress_attribute_mask = 0x%x\n", + traprepress_attr_mask); + + if (set_attr_mask && getresp_attr_mask && trap_attr_mask && + traprepress_attr_mask) + return 1; + + return 0; +} + +int mlx4_config_mad_demux(struct mlx4_dev *dev) +{ + struct mlx4_cmd_mailbox *mailbox; + int err; + + /* Check if mad_demux is supported */ + if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_MAD_DEMUX)) + return 0; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + mlx4_warn(dev, "Failed to allocate mailbox for cmd MAD_DEMUX"); + return -ENOMEM; + } + + /* Query mad_demux to find out which MADs are handled by internal sma */ + err = mlx4_cmd_box(dev, 0, mailbox->dma, 0x01 /* subn mgmt class */, + MLX4_CMD_MAD_DEMUX_QUERY_RESTR, MLX4_CMD_MAD_DEMUX, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); + if (err) { + mlx4_warn(dev, "MLX4_CMD_MAD_DEMUX: query restrictions failed (%d)\n", + err); + goto out; + } + + if (mlx4_check_smp_firewall_active(dev, mailbox)) + dev->flags |= MLX4_FLAG_SECURE_HOST; + + /* Config mad_demux to handle all MADs returned by the query above */ + err = mlx4_cmd(dev, mailbox->dma, 0x01 /* subn mgmt class */, + MLX4_CMD_MAD_DEMUX_CONFIG, MLX4_CMD_MAD_DEMUX, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); + if (err) { + mlx4_warn(dev, "MLX4_CMD_MAD_DEMUX: configure failed (%d)\n", err); + goto out; + } + + if (dev->flags & MLX4_FLAG_SECURE_HOST) + mlx4_warn(dev, "HCA operating in secure-host mode. SMP firewall activated.\n"); +out: + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +/* Access Reg commands */ +enum mlx4_access_reg_masks { + MLX4_ACCESS_REG_STATUS_MASK = 0x7f, + MLX4_ACCESS_REG_METHOD_MASK = 0x7f, + MLX4_ACCESS_REG_LEN_MASK = 0x7ff +}; + +struct mlx4_access_reg { + __be16 constant1; + u8 status; + u8 resrvd1; + __be16 reg_id; + u8 method; + u8 constant2; + __be32 resrvd2[2]; + __be16 len_const; + __be16 resrvd3; +#define MLX4_ACCESS_REG_HEADER_SIZE (20) + u8 reg_data[MLX4_MAILBOX_SIZE-MLX4_ACCESS_REG_HEADER_SIZE]; +} __attribute__((__packed__)); + +/** + * mlx4_ACCESS_REG - Generic access reg command. + * @dev: mlx4_dev. + * @reg_id: register ID to access. + * @method: Access method Read/Write. + * @reg_len: register length to Read/Write in bytes. + * @reg_data: reg_data pointer to Read/Write From/To. + * + * Access ConnectX registers FW command. + * Returns 0 on success and copies outbox mlx4_access_reg data + * field into reg_data or a negative error code. + */ +static int mlx4_ACCESS_REG(struct mlx4_dev *dev, u16 reg_id, + enum mlx4_access_reg_method method, + u16 reg_len, void *reg_data) +{ + struct mlx4_cmd_mailbox *inbox, *outbox; + struct mlx4_access_reg *inbuf, *outbuf; + int err; + + inbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(inbox)) + return PTR_ERR(inbox); + + outbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(outbox)) { + mlx4_free_cmd_mailbox(dev, inbox); + return PTR_ERR(outbox); + } + + inbuf = inbox->buf; + outbuf = outbox->buf; + + inbuf->constant1 = cpu_to_be16(0x1<<11 | 0x4); + inbuf->constant2 = 0x1; + inbuf->reg_id = cpu_to_be16(reg_id); + inbuf->method = method & MLX4_ACCESS_REG_METHOD_MASK; + + reg_len = min(reg_len, (u16)(sizeof(inbuf->reg_data))); + inbuf->len_const = + cpu_to_be16(((reg_len/4 + 1) & MLX4_ACCESS_REG_LEN_MASK) | + ((0x3) << 12)); + + memcpy(inbuf->reg_data, reg_data, reg_len); + err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, 0, 0, + MLX4_CMD_ACCESS_REG, MLX4_CMD_TIME_CLASS_C, + MLX4_CMD_WRAPPED); + if (err) + goto out; + + if (outbuf->status & MLX4_ACCESS_REG_STATUS_MASK) { + err = outbuf->status & MLX4_ACCESS_REG_STATUS_MASK; + mlx4_err(dev, + "MLX4_CMD_ACCESS_REG(%x) returned REG status (%x)\n", + reg_id, err); + goto out; + } + + memcpy(reg_data, outbuf->reg_data, reg_len); +out: + mlx4_free_cmd_mailbox(dev, inbox); + mlx4_free_cmd_mailbox(dev, outbox); + return err; +} + +/* ConnectX registers IDs */ +enum mlx4_reg_id { + MLX4_REG_ID_PTYS = 0x5004, +}; + +/** + * mlx4_ACCESS_PTYS_REG - Access PTYs (Port Type and Speed) + * register + * @dev: mlx4_dev. + * @method: Access method Read/Write. + * @ptys_reg: PTYS register data pointer. + * + * Access ConnectX PTYS register, to Read/Write Port Type/Speed + * configuration + * Returns 0 on success or a negative error code. + */ +int mlx4_ACCESS_PTYS_REG(struct mlx4_dev *dev, + enum mlx4_access_reg_method method, + struct mlx4_ptys_reg *ptys_reg) +{ + return mlx4_ACCESS_REG(dev, MLX4_REG_ID_PTYS, + method, sizeof(*ptys_reg), ptys_reg); +} +EXPORT_SYMBOL_GPL(mlx4_ACCESS_PTYS_REG); + +int mlx4_ACCESS_REG_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_access_reg *inbuf = inbox->buf; + u8 method = inbuf->method & MLX4_ACCESS_REG_METHOD_MASK; + u16 reg_id = be16_to_cpu(inbuf->reg_id); + + if (slave != mlx4_master_func_num(dev) && + method == MLX4_ACCESS_REG_WRITE) + return -EPERM; + + if (reg_id == MLX4_REG_ID_PTYS) { + struct mlx4_ptys_reg *ptys_reg = + (struct mlx4_ptys_reg *)inbuf->reg_data; + + ptys_reg->local_port = + mlx4_slave_convert_port(dev, slave, + ptys_reg->local_port); + } + + return mlx4_cmd_box(dev, inbox->dma, outbox->dma, vhcr->in_modifier, + 0, MLX4_CMD_ACCESS_REG, MLX4_CMD_TIME_CLASS_C, + MLX4_CMD_NATIVE); +} + +static int mlx4_SET_PORT_phv_bit(struct mlx4_dev *dev, u8 port, u8 phv_bit) +{ +#define SET_PORT_GEN_PHV_VALID 0x10 +#define SET_PORT_GEN_PHV_EN 0x80 + + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_set_port_general_context *context; + u32 in_mod; + int err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + context = mailbox->buf; + + context->v_ignore_fcs |= SET_PORT_GEN_PHV_VALID; + if (phv_bit) + context->phv_en |= SET_PORT_GEN_PHV_EN; + + in_mod = MLX4_SET_PORT_GENERAL << 8 | port; + err = mlx4_cmd(dev, mailbox->dma, in_mod, MLX4_SET_PORT_ETH_OPCODE, + MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +int get_phv_bit(struct mlx4_dev *dev, u8 port, int *phv) +{ + int err; + struct mlx4_func_cap func_cap; + + memset(&func_cap, 0, sizeof(func_cap)); + err = mlx4_QUERY_FUNC_CAP(dev, port, &func_cap); + if (!err) + *phv = func_cap.flags0 & QUERY_FUNC_CAP_PHV_BIT; + return err; +} +EXPORT_SYMBOL(get_phv_bit); + +int set_phv_bit(struct mlx4_dev *dev, u8 port, int new_val) +{ + int ret; + + if (mlx4_is_slave(dev)) + return -EPERM; + + if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PHV_EN && + !(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN)) { + ret = mlx4_SET_PORT_phv_bit(dev, port, new_val); + if (!ret) + dev->caps.phv_bit[port] = new_val; + return ret; + } + + return -EOPNOTSUPP; +} +EXPORT_SYMBOL(set_phv_bit); + +void mlx4_replace_zero_macs(struct mlx4_dev *dev) +{ + int i; + u8 mac_addr[ETH_ALEN]; + + dev->port_random_macs = 0; + for (i = 1; i <= dev->caps.num_ports; ++i) + if (!dev->caps.def_mac[i] && + dev->caps.port_type[i] == MLX4_PORT_TYPE_ETH) { + random_ether_addr(mac_addr); + dev->port_random_macs |= 1 << i; + dev->caps.def_mac[i] = mlx4_mac_to_u64(mac_addr); + } +} +EXPORT_SYMBOL_GPL(mlx4_replace_zero_macs); Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_fw.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_fw_qos.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_fw_qos.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_fw_qos.c (revision 329159) @@ -0,0 +1,289 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. + * All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include "fw_qos.h" +#include "fw.h" + +enum { + /* allocate vpp opcode modifiers */ + MLX4_ALLOCATE_VPP_ALLOCATE = 0x0, + MLX4_ALLOCATE_VPP_QUERY = 0x1 +}; + +enum { + /* set vport qos opcode modifiers */ + MLX4_SET_VPORT_QOS_SET = 0x0, + MLX4_SET_VPORT_QOS_QUERY = 0x1 +}; + +struct mlx4_set_port_prio2tc_context { + u8 prio2tc[4]; +}; + +struct mlx4_port_scheduler_tc_cfg_be { + __be16 pg; + __be16 bw_precentage; + __be16 max_bw_units; /* 3-100Mbps, 4-1Gbps, other values - reserved */ + __be16 max_bw_value; +}; + +struct mlx4_set_port_scheduler_context { + struct mlx4_port_scheduler_tc_cfg_be tc[MLX4_NUM_TC]; +}; + +/* Granular Qos (per VF) section */ +struct mlx4_alloc_vpp_param { + __be32 availible_vpp; + __be32 vpp_p_up[MLX4_NUM_UP]; +}; + +struct mlx4_prio_qos_param { + __be32 bw_share; + __be32 max_avg_bw; + __be32 reserved; + __be32 enable; + __be32 reserved1[4]; +}; + +struct mlx4_set_vport_context { + __be32 reserved[8]; + struct mlx4_prio_qos_param qos_p_up[MLX4_NUM_UP]; +}; + +int mlx4_SET_PORT_PRIO2TC(struct mlx4_dev *dev, u8 port, u8 *prio2tc) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_set_port_prio2tc_context *context; + int err; + u32 in_mod; + int i; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + context = mailbox->buf; + + for (i = 0; i < MLX4_NUM_UP; i += 2) + context->prio2tc[i >> 1] = prio2tc[i] << 4 | prio2tc[i + 1]; + + in_mod = MLX4_SET_PORT_PRIO2TC << 8 | port; + err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL(mlx4_SET_PORT_PRIO2TC); + +int mlx4_SET_PORT_SCHEDULER(struct mlx4_dev *dev, u8 port, u8 *tc_tx_bw, + u8 *pg, u16 *ratelimit) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_set_port_scheduler_context *context; + int err; + u32 in_mod; + int i; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + context = mailbox->buf; + + for (i = 0; i < MLX4_NUM_TC; i++) { + struct mlx4_port_scheduler_tc_cfg_be *tc = &context->tc[i]; + u16 r; + + if (ratelimit && ratelimit[i]) { + if (ratelimit[i] <= MLX4_MAX_100M_UNITS_VAL) { + r = ratelimit[i]; + tc->max_bw_units = + htons(MLX4_RATELIMIT_100M_UNITS); + } else { + r = ratelimit[i] / 10; + tc->max_bw_units = + htons(MLX4_RATELIMIT_1G_UNITS); + } + tc->max_bw_value = htons(r); + } else { + tc->max_bw_value = htons(MLX4_RATELIMIT_DEFAULT); + tc->max_bw_units = htons(MLX4_RATELIMIT_1G_UNITS); + } + + tc->pg = htons(pg[i]); + tc->bw_precentage = htons(tc_tx_bw[i]); + } + + in_mod = MLX4_SET_PORT_SCHEDULER << 8 | port; + err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL(mlx4_SET_PORT_SCHEDULER); + +int mlx4_ALLOCATE_VPP_get(struct mlx4_dev *dev, u8 port, + u16 *availible_vpp, u8 *vpp_p_up) +{ + int i; + int err; + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_alloc_vpp_param *out_param; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + out_param = mailbox->buf; + + err = mlx4_cmd_box(dev, 0, mailbox->dma, port, + MLX4_ALLOCATE_VPP_QUERY, + MLX4_CMD_ALLOCATE_VPP, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + goto out; + + /* Total number of supported VPPs */ + *availible_vpp = (u16)be32_to_cpu(out_param->availible_vpp); + + for (i = 0; i < MLX4_NUM_UP; i++) + vpp_p_up[i] = (u8)be32_to_cpu(out_param->vpp_p_up[i]); + +out: + mlx4_free_cmd_mailbox(dev, mailbox); + + return err; +} +EXPORT_SYMBOL(mlx4_ALLOCATE_VPP_get); + +int mlx4_ALLOCATE_VPP_set(struct mlx4_dev *dev, u8 port, u8 *vpp_p_up) +{ + int i; + int err; + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_alloc_vpp_param *in_param; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + in_param = mailbox->buf; + + for (i = 0; i < MLX4_NUM_UP; i++) + in_param->vpp_p_up[i] = cpu_to_be32(vpp_p_up[i]); + + err = mlx4_cmd(dev, mailbox->dma, port, + MLX4_ALLOCATE_VPP_ALLOCATE, + MLX4_CMD_ALLOCATE_VPP, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL(mlx4_ALLOCATE_VPP_set); + +int mlx4_SET_VPORT_QOS_get(struct mlx4_dev *dev, u8 port, u8 vport, + struct mlx4_vport_qos_param *out_param) +{ + int i; + int err; + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_set_vport_context *ctx; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + ctx = mailbox->buf; + + err = mlx4_cmd_box(dev, 0, mailbox->dma, (vport << 8) | port, + MLX4_SET_VPORT_QOS_QUERY, + MLX4_CMD_SET_VPORT_QOS, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + goto out; + + for (i = 0; i < MLX4_NUM_UP; i++) { + out_param[i].bw_share = be32_to_cpu(ctx->qos_p_up[i].bw_share); + out_param[i].max_avg_bw = + be32_to_cpu(ctx->qos_p_up[i].max_avg_bw); + out_param[i].enable = + !!(be32_to_cpu(ctx->qos_p_up[i].enable) & 31); + } + +out: + mlx4_free_cmd_mailbox(dev, mailbox); + + return err; +} +EXPORT_SYMBOL(mlx4_SET_VPORT_QOS_get); + +int mlx4_SET_VPORT_QOS_set(struct mlx4_dev *dev, u8 port, u8 vport, + struct mlx4_vport_qos_param *in_param) +{ + int i; + int err; + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_set_vport_context *ctx; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + ctx = mailbox->buf; + + for (i = 0; i < MLX4_NUM_UP; i++) { + ctx->qos_p_up[i].bw_share = cpu_to_be32(in_param[i].bw_share); + ctx->qos_p_up[i].max_avg_bw = + cpu_to_be32(in_param[i].max_avg_bw); + ctx->qos_p_up[i].enable = + cpu_to_be32(in_param[i].enable << 31); + } + + err = mlx4_cmd(dev, mailbox->dma, (vport << 8) | port, + MLX4_SET_VPORT_QOS_SET, + MLX4_CMD_SET_VPORT_QOS, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL(mlx4_SET_VPORT_QOS_set); Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_fw_qos.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_icm.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_icm.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_icm.c (revision 329159) @@ -0,0 +1,463 @@ +/* + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include + +#include + +#include "mlx4.h" +#include "icm.h" +#include "fw.h" + +/* + * We allocate in as big chunks as we can, up to a maximum of 256 KB + * per chunk. + */ +enum { + MLX4_ICM_ALLOC_SIZE = 1 << 18, + MLX4_TABLE_CHUNK_SIZE = 1 << 18 +}; + +static void mlx4_free_icm_pages(struct mlx4_dev *dev, struct mlx4_icm_chunk *chunk) +{ + int i; + + if (chunk->nsg > 0) + pci_unmap_sg(dev->persist->pdev, chunk->mem, chunk->npages, + PCI_DMA_BIDIRECTIONAL); + + for (i = 0; i < chunk->npages; ++i) + __free_pages(sg_page(&chunk->mem[i]), + get_order(chunk->mem[i].length)); +} + +static void mlx4_free_icm_coherent(struct mlx4_dev *dev, struct mlx4_icm_chunk *chunk) +{ + int i; + + for (i = 0; i < chunk->npages; ++i) + dma_free_coherent(&dev->persist->pdev->dev, + chunk->mem[i].length, + lowmem_page_address(sg_page(&chunk->mem[i])), + sg_dma_address(&chunk->mem[i])); +} + +void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm, int coherent) +{ + struct mlx4_icm_chunk *chunk, *tmp; + + if (!icm) + return; + + list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) { + if (coherent) + mlx4_free_icm_coherent(dev, chunk); + else + mlx4_free_icm_pages(dev, chunk); + + kfree(chunk); + } + + kfree(icm); +} + +static int mlx4_alloc_icm_pages(struct scatterlist *mem, int order, + gfp_t gfp_mask, int node) +{ + struct page *page; + + page = alloc_pages_node(node, gfp_mask, order); + if (!page) { + page = alloc_pages(gfp_mask, order); + if (!page) + return -ENOMEM; + } + + sg_set_page(mem, page, PAGE_SIZE << order, 0); + return 0; +} + +static int mlx4_alloc_icm_coherent(struct device *dev, struct scatterlist *mem, + int order, gfp_t gfp_mask) +{ + void *buf = dma_alloc_coherent(dev, PAGE_SIZE << order, + &sg_dma_address(mem), gfp_mask); + if (!buf) + return -ENOMEM; + + sg_set_buf(mem, buf, PAGE_SIZE << order); + BUG_ON(mem->offset); + sg_dma_len(mem) = PAGE_SIZE << order; + return 0; +} + +struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages, + gfp_t gfp_mask, int coherent) +{ + struct mlx4_icm *icm; + struct mlx4_icm_chunk *chunk = NULL; + int cur_order; + int ret; + + /* We use sg_set_buf for coherent allocs, which assumes low memory */ + BUG_ON(coherent && (gfp_mask & __GFP_HIGHMEM)); + + icm = kmalloc_node(sizeof(*icm), + gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN), + dev->numa_node); + if (!icm) { + icm = kmalloc(sizeof(*icm), + gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); + if (!icm) + return NULL; + } + + icm->refcount = 0; + INIT_LIST_HEAD(&icm->chunk_list); + + cur_order = get_order(MLX4_ICM_ALLOC_SIZE); + + while (npages > 0) { + if (!chunk) { + chunk = kmalloc_node(sizeof(*chunk), + gfp_mask & ~(__GFP_HIGHMEM | + __GFP_NOWARN), + dev->numa_node); + if (!chunk) { + chunk = kmalloc(sizeof(*chunk), + gfp_mask & ~(__GFP_HIGHMEM | + __GFP_NOWARN)); + if (!chunk) + goto fail; + } + + sg_init_table(chunk->mem, MLX4_ICM_CHUNK_LEN); + chunk->npages = 0; + chunk->nsg = 0; + list_add_tail(&chunk->list, &icm->chunk_list); + } + + while (1 << cur_order > npages) + --cur_order; + + if (coherent) + ret = mlx4_alloc_icm_coherent(&dev->persist->pdev->dev, + &chunk->mem[chunk->npages], + cur_order, gfp_mask); + else + ret = mlx4_alloc_icm_pages(&chunk->mem[chunk->npages], + cur_order, gfp_mask, + dev->numa_node); + + if (ret) { + if (--cur_order < 0) + goto fail; + else + continue; + } + + ++chunk->npages; + + if (coherent) + ++chunk->nsg; + else if (chunk->npages == MLX4_ICM_CHUNK_LEN) { + chunk->nsg = pci_map_sg(dev->persist->pdev, chunk->mem, + chunk->npages, + PCI_DMA_BIDIRECTIONAL); + + if (chunk->nsg <= 0) + goto fail; + } + + if (chunk->npages == MLX4_ICM_CHUNK_LEN) + chunk = NULL; + + npages -= 1 << cur_order; + } + + if (!coherent && chunk) { + chunk->nsg = pci_map_sg(dev->persist->pdev, chunk->mem, + chunk->npages, + PCI_DMA_BIDIRECTIONAL); + + if (chunk->nsg <= 0) + goto fail; + } + + return icm; + +fail: + mlx4_free_icm(dev, icm, coherent); + return NULL; +} + +static int mlx4_MAP_ICM(struct mlx4_dev *dev, struct mlx4_icm *icm, u64 virt) +{ + return mlx4_map_cmd(dev, MLX4_CMD_MAP_ICM, icm, virt); +} + +static int mlx4_UNMAP_ICM(struct mlx4_dev *dev, u64 virt, u32 page_count) +{ + return mlx4_cmd(dev, virt, page_count, 0, MLX4_CMD_UNMAP_ICM, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); +} + +int mlx4_MAP_ICM_AUX(struct mlx4_dev *dev, struct mlx4_icm *icm) +{ + return mlx4_map_cmd(dev, MLX4_CMD_MAP_ICM_AUX, icm, -1); +} + +int mlx4_UNMAP_ICM_AUX(struct mlx4_dev *dev) +{ + return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_UNMAP_ICM_AUX, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); +} + +int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj, + gfp_t gfp) +{ + u32 i = (obj & (table->num_obj - 1)) / + (MLX4_TABLE_CHUNK_SIZE / table->obj_size); + int ret = 0; + + mutex_lock(&table->mutex); + + if (table->icm[i]) { + ++table->icm[i]->refcount; + goto out; + } + + table->icm[i] = mlx4_alloc_icm(dev, MLX4_TABLE_CHUNK_SIZE >> PAGE_SHIFT, + (table->lowmem ? gfp : GFP_HIGHUSER) | + __GFP_NOWARN, table->coherent); + if (!table->icm[i]) { + ret = -ENOMEM; + goto out; + } + + if (mlx4_MAP_ICM(dev, table->icm[i], table->virt + + (u64) i * MLX4_TABLE_CHUNK_SIZE)) { + mlx4_free_icm(dev, table->icm[i], table->coherent); + table->icm[i] = NULL; + ret = -ENOMEM; + goto out; + } + + ++table->icm[i]->refcount; + +out: + mutex_unlock(&table->mutex); + return ret; +} + +void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj) +{ + u32 i; + u64 offset; + + i = (obj & (table->num_obj - 1)) / (MLX4_TABLE_CHUNK_SIZE / table->obj_size); + + mutex_lock(&table->mutex); + + if (--table->icm[i]->refcount == 0) { + offset = (u64) i * MLX4_TABLE_CHUNK_SIZE; + mlx4_UNMAP_ICM(dev, table->virt + offset, + MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE); + mlx4_free_icm(dev, table->icm[i], table->coherent); + table->icm[i] = NULL; + } + + mutex_unlock(&table->mutex); +} + +void *mlx4_table_find(struct mlx4_icm_table *table, u32 obj, + dma_addr_t *dma_handle) +{ + int offset, dma_offset, i; + u64 idx; + struct mlx4_icm_chunk *chunk; + struct mlx4_icm *icm; + struct page *page = NULL; + + if (!table->lowmem) + return NULL; + + mutex_lock(&table->mutex); + + idx = (u64) (obj & (table->num_obj - 1)) * table->obj_size; + icm = table->icm[idx / MLX4_TABLE_CHUNK_SIZE]; + dma_offset = offset = idx % MLX4_TABLE_CHUNK_SIZE; + + if (!icm) + goto out; + + list_for_each_entry(chunk, &icm->chunk_list, list) { + for (i = 0; i < chunk->npages; ++i) { + if (dma_handle && dma_offset >= 0) { + if (sg_dma_len(&chunk->mem[i]) > dma_offset) + *dma_handle = sg_dma_address(&chunk->mem[i]) + + dma_offset; + dma_offset -= sg_dma_len(&chunk->mem[i]); + } + /* + * DMA mapping can merge pages but not split them, + * so if we found the page, dma_handle has already + * been assigned to. + */ + if (chunk->mem[i].length > offset) { + page = sg_page(&chunk->mem[i]); + goto out; + } + offset -= chunk->mem[i].length; + } + } + +out: + mutex_unlock(&table->mutex); + return page ? lowmem_page_address(page) + offset : NULL; +} + +int mlx4_table_get_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, + u32 start, u32 end) +{ + int inc = MLX4_TABLE_CHUNK_SIZE / table->obj_size; + int err; + u32 i; + + for (i = start; i <= end; i += inc) { + err = mlx4_table_get(dev, table, i, GFP_KERNEL); + if (err) + goto fail; + } + + return 0; + +fail: + while (i > start) { + i -= inc; + mlx4_table_put(dev, table, i); + } + + return err; +} + +void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, + u32 start, u32 end) +{ + u32 i; + + for (i = start; i <= end; i += MLX4_TABLE_CHUNK_SIZE / table->obj_size) + mlx4_table_put(dev, table, i); +} + +int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table, + u64 virt, int obj_size, u32 nobj, int reserved, + int use_lowmem, int use_coherent) +{ + int obj_per_chunk; + int num_icm; + unsigned chunk_size; + int i; + u64 size; + + obj_per_chunk = MLX4_TABLE_CHUNK_SIZE / obj_size; + num_icm = (nobj + obj_per_chunk - 1) / obj_per_chunk; + + table->icm = kcalloc(num_icm, sizeof *table->icm, GFP_KERNEL); + if (!table->icm) + return -ENOMEM; + table->virt = virt; + table->num_icm = num_icm; + table->num_obj = nobj; + table->obj_size = obj_size; + table->lowmem = use_lowmem; + table->coherent = use_coherent; + mutex_init(&table->mutex); + + size = (u64) nobj * obj_size; + for (i = 0; i * MLX4_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) { + chunk_size = MLX4_TABLE_CHUNK_SIZE; + if ((i + 1) * MLX4_TABLE_CHUNK_SIZE > size) + chunk_size = PAGE_ALIGN(size - + i * MLX4_TABLE_CHUNK_SIZE); + + table->icm[i] = mlx4_alloc_icm(dev, chunk_size >> PAGE_SHIFT, + (use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) | + __GFP_NOWARN, use_coherent); + if (!table->icm[i]) + goto err; + if (mlx4_MAP_ICM(dev, table->icm[i], virt + i * MLX4_TABLE_CHUNK_SIZE)) { + mlx4_free_icm(dev, table->icm[i], use_coherent); + table->icm[i] = NULL; + goto err; + } + + /* + * Add a reference to this ICM chunk so that it never + * gets freed (since it contains reserved firmware objects). + */ + ++table->icm[i]->refcount; + } + + return 0; + +err: + for (i = 0; i < num_icm; ++i) + if (table->icm[i]) { + mlx4_UNMAP_ICM(dev, virt + i * MLX4_TABLE_CHUNK_SIZE, + MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE); + mlx4_free_icm(dev, table->icm[i], use_coherent); + } + + kfree(table->icm); + + return -ENOMEM; +} + +void mlx4_cleanup_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table) +{ + int i; + + for (i = 0; i < table->num_icm; ++i) + if (table->icm[i]) { + mlx4_UNMAP_ICM(dev, table->virt + i * MLX4_TABLE_CHUNK_SIZE, + MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE); + mlx4_free_icm(dev, table->icm[i], table->coherent); + } + + kfree(table->icm); +} Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_icm.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_intf.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_intf.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_intf.c (revision 329159) @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include + +#include "mlx4.h" + +struct mlx4_device_context { + struct list_head list; + struct list_head bond_list; + struct mlx4_interface *intf; + void *context; +}; + +static LIST_HEAD(intf_list); +static LIST_HEAD(dev_list); +static DEFINE_MUTEX(intf_mutex); + +static void mlx4_add_device(struct mlx4_interface *intf, struct mlx4_priv *priv) +{ + struct mlx4_device_context *dev_ctx; + + dev_ctx = kmalloc(sizeof *dev_ctx, GFP_KERNEL); + if (!dev_ctx) + return; + + dev_ctx->intf = intf; + dev_ctx->context = intf->add(&priv->dev); + + if (dev_ctx->context) { + spin_lock_irq(&priv->ctx_lock); + list_add_tail(&dev_ctx->list, &priv->ctx_list); + spin_unlock_irq(&priv->ctx_lock); + if (intf->activate) + intf->activate(&priv->dev, dev_ctx->context); + } else + kfree(dev_ctx); +} + +static void mlx4_remove_device(struct mlx4_interface *intf, struct mlx4_priv *priv) +{ + struct mlx4_device_context *dev_ctx; + + list_for_each_entry(dev_ctx, &priv->ctx_list, list) + if (dev_ctx->intf == intf) { + spin_lock_irq(&priv->ctx_lock); + list_del(&dev_ctx->list); + spin_unlock_irq(&priv->ctx_lock); + + intf->remove(&priv->dev, dev_ctx->context); + kfree(dev_ctx); + return; + } +} + +int mlx4_register_interface(struct mlx4_interface *intf) +{ + struct mlx4_priv *priv; + + if (!intf->add || !intf->remove) + return -EINVAL; + + mutex_lock(&intf_mutex); + + list_add_tail(&intf->list, &intf_list); + list_for_each_entry(priv, &dev_list, dev_list) { + if (mlx4_is_mfunc(&priv->dev) && (intf->flags & MLX4_INTFF_BONDING)) { + mlx4_dbg(&priv->dev, + "SRIOV, disabling HA mode for intf proto %d\n", intf->protocol); + intf->flags &= ~MLX4_INTFF_BONDING; + } + mlx4_add_device(intf, priv); + } + + mutex_unlock(&intf_mutex); + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_register_interface); + +void mlx4_unregister_interface(struct mlx4_interface *intf) +{ + struct mlx4_priv *priv; + + mutex_lock(&intf_mutex); + + list_for_each_entry(priv, &dev_list, dev_list) + mlx4_remove_device(intf, priv); + + list_del(&intf->list); + + mutex_unlock(&intf_mutex); +} +EXPORT_SYMBOL_GPL(mlx4_unregister_interface); + +int mlx4_do_bond(struct mlx4_dev *dev, bool enable) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_device_context *dev_ctx = NULL, *temp_dev_ctx; + unsigned long flags; + int ret; + LIST_HEAD(bond_list); + + if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PORT_REMAP)) + return -ENOTSUPP; + + ret = mlx4_disable_rx_port_check(dev, enable); + if (ret) { + mlx4_err(dev, "Fail to %s rx port check\n", + enable ? "enable" : "disable"); + return ret; + } + if (enable) { + dev->flags |= MLX4_FLAG_BONDED; + } else { + ret = mlx4_virt2phy_port_map(dev, 1, 2); + if (ret) { + mlx4_err(dev, "Fail to reset port map\n"); + return ret; + } + dev->flags &= ~MLX4_FLAG_BONDED; + } + + spin_lock_irqsave(&priv->ctx_lock, flags); + list_for_each_entry_safe(dev_ctx, temp_dev_ctx, &priv->ctx_list, list) { + if (dev_ctx->intf->flags & MLX4_INTFF_BONDING) { + list_add_tail(&dev_ctx->bond_list, &bond_list); + list_del(&dev_ctx->list); + } + } + spin_unlock_irqrestore(&priv->ctx_lock, flags); + + list_for_each_entry(dev_ctx, &bond_list, bond_list) { + dev_ctx->intf->remove(dev, dev_ctx->context); + dev_ctx->context = dev_ctx->intf->add(dev); + + spin_lock_irqsave(&priv->ctx_lock, flags); + list_add_tail(&dev_ctx->list, &priv->ctx_list); + spin_unlock_irqrestore(&priv->ctx_lock, flags); + + mlx4_dbg(dev, "Inrerface for protocol %d restarted with when bonded mode is %s\n", + dev_ctx->intf->protocol, enable ? + "enabled" : "disabled"); + } + return 0; +} + +void mlx4_dispatch_event(struct mlx4_dev *dev, enum mlx4_dev_event type, + unsigned long param) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_device_context *dev_ctx; + unsigned long flags; + + spin_lock_irqsave(&priv->ctx_lock, flags); + + list_for_each_entry(dev_ctx, &priv->ctx_list, list) + if (dev_ctx->intf->event) + dev_ctx->intf->event(dev, dev_ctx->context, type, param); + + spin_unlock_irqrestore(&priv->ctx_lock, flags); +} + +int mlx4_register_device(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_interface *intf; + + mutex_lock(&intf_mutex); + + dev->persist->interface_state |= MLX4_INTERFACE_STATE_UP; + list_add_tail(&priv->dev_list, &dev_list); + list_for_each_entry(intf, &intf_list, list) + mlx4_add_device(intf, priv); + + mutex_unlock(&intf_mutex); + mlx4_start_catas_poll(dev); + + return 0; +} + +void mlx4_unregister_device(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_interface *intf; + + if (!(dev->persist->interface_state & MLX4_INTERFACE_STATE_UP)) + return; + + mlx4_stop_catas_poll(dev); + mutex_lock(&intf_mutex); + + list_for_each_entry(intf, &intf_list, list) + mlx4_remove_device(intf, priv); + + list_del(&priv->dev_list); + dev->persist->interface_state &= ~MLX4_INTERFACE_STATE_UP; + + mutex_unlock(&intf_mutex); +} + +void *mlx4_get_protocol_dev(struct mlx4_dev *dev, enum mlx4_protocol proto, int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_device_context *dev_ctx; + unsigned long flags; + void *result = NULL; + + spin_lock_irqsave(&priv->ctx_lock, flags); + + list_for_each_entry(dev_ctx, &priv->ctx_list, list) + if (dev_ctx->intf->protocol == proto && dev_ctx->intf->get_dev) { + result = dev_ctx->intf->get_dev(dev, dev_ctx->context, port); + break; + } + + spin_unlock_irqrestore(&priv->ctx_lock, flags); + + return result; +} +EXPORT_SYMBOL_GPL(mlx4_get_protocol_dev); + Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_intf.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_main.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_main.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_main.c (revision 329159) @@ -0,0 +1,4203 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#define LINUXKPI_PARAM_PREFIX mlx4_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "mlx4.h" +#include "fw.h" +#include "icm.h" +#include + +MODULE_AUTHOR("Roland Dreier"); +MODULE_DESCRIPTION("Mellanox ConnectX HCA low-level driver"); +MODULE_LICENSE("Dual BSD/GPL"); + +struct workqueue_struct *mlx4_wq; + +#ifdef CONFIG_MLX4_DEBUG + +int mlx4_debug_level = 0; +module_param_named(debug_level, mlx4_debug_level, int, 0644); +MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0"); + +#endif /* CONFIG_MLX4_DEBUG */ + +#ifdef CONFIG_PCI_MSI + +static int msi_x = 1; +module_param(msi_x, int, 0444); +MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero"); + +#else /* CONFIG_PCI_MSI */ + +#define msi_x (0) + +#endif /* CONFIG_PCI_MSI */ + +static uint8_t num_vfs[3] = {0, 0, 0}; +static int num_vfs_argc; +module_param_array(num_vfs, byte , &num_vfs_argc, 0444); +MODULE_PARM_DESC(num_vfs, "enable #num_vfs functions if num_vfs > 0\n" + "num_vfs=port1,port2,port1+2"); + +static uint8_t probe_vf[3] = {0, 0, 0}; +static int probe_vfs_argc; +module_param_array(probe_vf, byte, &probe_vfs_argc, 0444); +MODULE_PARM_DESC(probe_vf, "number of vfs to probe by pf driver (num_vfs > 0)\n" + "probe_vf=port1,port2,port1+2"); + +int mlx4_log_num_mgm_entry_size = MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE; +module_param_named(log_num_mgm_entry_size, + mlx4_log_num_mgm_entry_size, int, 0444); +MODULE_PARM_DESC(log_num_mgm_entry_size, "log mgm size, that defines the num" + " of qp per mcg, for example:" + " 10 gives 248.range: 7 <=" + " log_num_mgm_entry_size <= 12." + " To activate device managed" + " flow steering when available, set to -1"); + +static bool enable_64b_cqe_eqe = true; +module_param(enable_64b_cqe_eqe, bool, 0444); +MODULE_PARM_DESC(enable_64b_cqe_eqe, + "Enable 64 byte CQEs/EQEs when the FW supports this (default: True)"); + +static bool enable_4k_uar; +module_param(enable_4k_uar, bool, 0444); +MODULE_PARM_DESC(enable_4k_uar, + "Enable using 4K UAR. Should not be enabled if have VFs which do not support 4K UARs (default: false)"); + +#define PF_CONTEXT_BEHAVIOUR_MASK (MLX4_FUNC_CAP_64B_EQE_CQE | \ + MLX4_FUNC_CAP_EQE_CQE_STRIDE | \ + MLX4_FUNC_CAP_DMFS_A0_STATIC) + +#define RESET_PERSIST_MASK_FLAGS (MLX4_FLAG_SRIOV) + +static char mlx4_version[] = + DRV_NAME ": Mellanox ConnectX core driver v" + DRV_VERSION " (" DRV_RELDATE ")\n"; + +static struct mlx4_profile default_profile = { + .num_qp = 1 << 18, + .num_srq = 1 << 16, + .rdmarc_per_qp = 1 << 4, + .num_cq = 1 << 16, + .num_mcg = 1 << 13, + .num_mpt = 1 << 19, + .num_mtt = 1 << 20, /* It is really num mtt segements */ +}; + +static struct mlx4_profile low_mem_profile = { + .num_qp = 1 << 17, + .num_srq = 1 << 6, + .rdmarc_per_qp = 1 << 4, + .num_cq = 1 << 8, + .num_mcg = 1 << 8, + .num_mpt = 1 << 9, + .num_mtt = 1 << 7, +}; + +static int log_num_mac = 7; +module_param_named(log_num_mac, log_num_mac, int, 0444); +MODULE_PARM_DESC(log_num_mac, "Log2 max number of MACs per ETH port (1-7)"); + +static int log_num_vlan; +module_param_named(log_num_vlan, log_num_vlan, int, 0444); +MODULE_PARM_DESC(log_num_vlan, "Log2 max number of VLANs per ETH port (0-7)"); +/* Log2 max number of VLANs per ETH port (0-7) */ +#define MLX4_LOG_NUM_VLANS 7 +#define MLX4_MIN_LOG_NUM_VLANS 0 +#define MLX4_MIN_LOG_NUM_MAC 1 + +static bool use_prio; +module_param_named(use_prio, use_prio, bool, 0444); +MODULE_PARM_DESC(use_prio, "Enable steering by VLAN priority on ETH ports (deprecated)"); + +int log_mtts_per_seg = ilog2(MLX4_MTT_ENTRY_PER_SEG); +module_param_named(log_mtts_per_seg, log_mtts_per_seg, int, 0444); +MODULE_PARM_DESC(log_mtts_per_seg, "Log2 number of MTT entries per segment (1-7)"); + +static int port_type_array[2] = {MLX4_PORT_TYPE_NONE, MLX4_PORT_TYPE_NONE}; + +struct mlx4_port_config { + struct list_head list; + enum mlx4_port_type port_type[MLX4_MAX_PORTS + 1]; + struct pci_dev *pdev; +}; + +static atomic_t pf_loading = ATOMIC_INIT(0); + +static inline void mlx4_set_num_reserved_uars(struct mlx4_dev *dev, + struct mlx4_dev_cap *dev_cap) +{ + /* The reserved_uars is calculated by system page size unit. + * Therefore, adjustment is added when the uar page size is less + * than the system page size + */ + dev->caps.reserved_uars = + max_t(int, + mlx4_get_num_reserved_uar(dev), + dev_cap->reserved_uars / + (1 << (PAGE_SHIFT - dev->uar_page_shift))); +} + +int mlx4_check_port_params(struct mlx4_dev *dev, + enum mlx4_port_type *port_type) +{ + int i; + + if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) { + for (i = 0; i < dev->caps.num_ports - 1; i++) { + if (port_type[i] != port_type[i + 1]) { + mlx4_err(dev, "Only same port types supported on this HCA, aborting\n"); + return -EINVAL; + } + } + } + + for (i = 0; i < dev->caps.num_ports; i++) { + if (!(port_type[i] & dev->caps.supported_type[i+1])) { + mlx4_err(dev, "Requested port type for port %d is not supported on this HCA\n", + i + 1); + return -EINVAL; + } + } + return 0; +} + +static void mlx4_set_port_mask(struct mlx4_dev *dev) +{ + int i; + + for (i = 1; i <= dev->caps.num_ports; ++i) + dev->caps.port_mask[i] = dev->caps.port_type[i]; +} + +enum { + MLX4_QUERY_FUNC_NUM_SYS_EQS = 1 << 0, +}; + +static int mlx4_query_func(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) +{ + int err = 0; + struct mlx4_func func; + + if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) { + err = mlx4_QUERY_FUNC(dev, &func, 0); + if (err) { + mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n"); + return err; + } + dev_cap->max_eqs = func.max_eq; + dev_cap->reserved_eqs = func.rsvd_eqs; + dev_cap->reserved_uars = func.rsvd_uars; + err |= MLX4_QUERY_FUNC_NUM_SYS_EQS; + } + return err; +} + +static void mlx4_enable_cqe_eqe_stride(struct mlx4_dev *dev) +{ + struct mlx4_caps *dev_cap = &dev->caps; + + /* FW not supporting or cancelled by user */ + if (!(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_EQE_STRIDE) || + !(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_CQE_STRIDE)) + return; + + /* Must have 64B CQE_EQE enabled by FW to use bigger stride + * When FW has NCSI it may decide not to report 64B CQE/EQEs + */ + if (!(dev_cap->flags & MLX4_DEV_CAP_FLAG_64B_EQE) || + !(dev_cap->flags & MLX4_DEV_CAP_FLAG_64B_CQE)) { + dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_CQE_STRIDE; + dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_EQE_STRIDE; + return; + } + + if (cache_line_size() == 128 || cache_line_size() == 256) { + mlx4_dbg(dev, "Enabling CQE stride cacheLine supported\n"); + /* Changing the real data inside CQE size to 32B */ + dev_cap->flags &= ~MLX4_DEV_CAP_FLAG_64B_CQE; + dev_cap->flags &= ~MLX4_DEV_CAP_FLAG_64B_EQE; + + if (mlx4_is_master(dev)) + dev_cap->function_caps |= MLX4_FUNC_CAP_EQE_CQE_STRIDE; + } else { + if (cache_line_size() != 32 && cache_line_size() != 64) + mlx4_dbg(dev, "Disabling CQE stride, cacheLine size unsupported\n"); + dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_CQE_STRIDE; + dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_EQE_STRIDE; + } +} + +static int _mlx4_dev_port(struct mlx4_dev *dev, int port, + struct mlx4_port_cap *port_cap) +{ + dev->caps.vl_cap[port] = port_cap->max_vl; + dev->caps.ib_mtu_cap[port] = port_cap->ib_mtu; + dev->phys_caps.gid_phys_table_len[port] = port_cap->max_gids; + dev->phys_caps.pkey_phys_table_len[port] = port_cap->max_pkeys; + /* set gid and pkey table operating lengths by default + * to non-sriov values + */ + dev->caps.gid_table_len[port] = port_cap->max_gids; + dev->caps.pkey_table_len[port] = port_cap->max_pkeys; + dev->caps.port_width_cap[port] = port_cap->max_port_width; + dev->caps.eth_mtu_cap[port] = port_cap->eth_mtu; + dev->caps.max_tc_eth = port_cap->max_tc_eth; + dev->caps.def_mac[port] = port_cap->def_mac; + dev->caps.supported_type[port] = port_cap->supported_port_types; + dev->caps.suggested_type[port] = port_cap->suggested_type; + dev->caps.default_sense[port] = port_cap->default_sense; + dev->caps.trans_type[port] = port_cap->trans_type; + dev->caps.vendor_oui[port] = port_cap->vendor_oui; + dev->caps.wavelength[port] = port_cap->wavelength; + dev->caps.trans_code[port] = port_cap->trans_code; + + return 0; +} + +static int mlx4_dev_port(struct mlx4_dev *dev, int port, + struct mlx4_port_cap *port_cap) +{ + int err = 0; + + err = mlx4_QUERY_PORT(dev, port, port_cap); + + if (err) + mlx4_err(dev, "QUERY_PORT command failed.\n"); + + return err; +} + +static inline void mlx4_enable_ignore_fcs(struct mlx4_dev *dev) +{ + if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_IGNORE_FCS)) + return; + + if (mlx4_is_mfunc(dev)) { + mlx4_dbg(dev, "SRIOV mode - Disabling Ignore FCS"); + dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_IGNORE_FCS; + return; + } + + if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP)) { + mlx4_dbg(dev, + "Keep FCS is not supported - Disabling Ignore FCS"); + dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_IGNORE_FCS; + return; + } +} + +#define MLX4_A0_STEERING_TABLE_SIZE 256 +static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) +{ + int err; + int i; + + err = mlx4_QUERY_DEV_CAP(dev, dev_cap); + if (err) { + mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n"); + return err; + } + mlx4_dev_cap_dump(dev, dev_cap); + + if (dev_cap->min_page_sz > PAGE_SIZE) { + mlx4_err(dev, "HCA minimum page size of %d bigger than kernel PAGE_SIZE of %ld, aborting\n", + dev_cap->min_page_sz, (long)PAGE_SIZE); + return -ENODEV; + } + if (dev_cap->num_ports > MLX4_MAX_PORTS) { + mlx4_err(dev, "HCA has %d ports, but we only support %d, aborting\n", + dev_cap->num_ports, MLX4_MAX_PORTS); + return -ENODEV; + } + + if (dev_cap->uar_size > pci_resource_len(dev->persist->pdev, 2)) { + mlx4_err(dev, "HCA reported UAR size of 0x%x bigger than PCI resource 2 size of 0x%llx, aborting\n", + dev_cap->uar_size, + (unsigned long long) + pci_resource_len(dev->persist->pdev, 2)); + return -ENODEV; + } + + dev->caps.num_ports = dev_cap->num_ports; + dev->caps.num_sys_eqs = dev_cap->num_sys_eqs; + dev->phys_caps.num_phys_eqs = dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS ? + dev->caps.num_sys_eqs : + MLX4_MAX_EQ_NUM; + for (i = 1; i <= dev->caps.num_ports; ++i) { + err = _mlx4_dev_port(dev, i, dev_cap->port_cap + i); + if (err) { + mlx4_err(dev, "QUERY_PORT command failed, aborting\n"); + return err; + } + } + + dev->caps.uar_page_size = PAGE_SIZE; + dev->caps.num_uars = dev_cap->uar_size / PAGE_SIZE; + dev->caps.local_ca_ack_delay = dev_cap->local_ca_ack_delay; + dev->caps.bf_reg_size = dev_cap->bf_reg_size; + dev->caps.bf_regs_per_page = dev_cap->bf_regs_per_page; + dev->caps.max_sq_sg = dev_cap->max_sq_sg; + dev->caps.max_rq_sg = dev_cap->max_rq_sg; + dev->caps.max_wqes = dev_cap->max_qp_sz; + dev->caps.max_qp_init_rdma = dev_cap->max_requester_per_qp; + dev->caps.max_srq_wqes = dev_cap->max_srq_sz; + dev->caps.max_srq_sge = dev_cap->max_rq_sg - 1; + dev->caps.reserved_srqs = dev_cap->reserved_srqs; + dev->caps.max_sq_desc_sz = dev_cap->max_sq_desc_sz; + dev->caps.max_rq_desc_sz = dev_cap->max_rq_desc_sz; + /* + * Subtract 1 from the limit because we need to allocate a + * spare CQE so the HCA HW can tell the difference between an + * empty CQ and a full CQ. + */ + dev->caps.max_cqes = dev_cap->max_cq_sz - 1; + dev->caps.reserved_cqs = dev_cap->reserved_cqs; + dev->caps.reserved_eqs = dev_cap->reserved_eqs; + dev->caps.reserved_mtts = dev_cap->reserved_mtts; + dev->caps.reserved_mrws = dev_cap->reserved_mrws; + + dev->caps.reserved_pds = dev_cap->reserved_pds; + dev->caps.reserved_xrcds = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ? + dev_cap->reserved_xrcds : 0; + dev->caps.max_xrcds = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ? + dev_cap->max_xrcds : 0; + dev->caps.mtt_entry_sz = dev_cap->mtt_entry_sz; + + dev->caps.max_msg_sz = dev_cap->max_msg_sz; + dev->caps.page_size_cap = ~(u32) (dev_cap->min_page_sz - 1); + dev->caps.flags = dev_cap->flags; + dev->caps.flags2 = dev_cap->flags2; + dev->caps.bmme_flags = dev_cap->bmme_flags; + dev->caps.reserved_lkey = dev_cap->reserved_lkey; + dev->caps.stat_rate_support = dev_cap->stat_rate_support; + dev->caps.max_gso_sz = dev_cap->max_gso_sz; + dev->caps.max_rss_tbl_sz = dev_cap->max_rss_tbl_sz; + + /* Save uar page shift */ + if (!mlx4_is_slave(dev)) { + /* Virtual PCI function needs to determine UAR page size from + * firmware. Only master PCI function can set the uar page size + */ + if (enable_4k_uar) + dev->uar_page_shift = DEFAULT_UAR_PAGE_SHIFT; + else + dev->uar_page_shift = PAGE_SHIFT; + + mlx4_set_num_reserved_uars(dev, dev_cap); + } + + if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PHV_EN) { + struct mlx4_init_hca_param hca_param; + + memset(&hca_param, 0, sizeof(hca_param)); + err = mlx4_QUERY_HCA(dev, &hca_param); + /* Turn off PHV_EN flag in case phv_check_en is set. + * phv_check_en is a HW check that parse the packet and verify + * phv bit was reported correctly in the wqe. To allow QinQ + * PHV_EN flag should be set and phv_check_en must be cleared + * otherwise QinQ packets will be drop by the HW. + */ + if (err || hca_param.phv_check_en) + dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_PHV_EN; + } + + /* Sense port always allowed on supported devices for ConnectX-1 and -2 */ + if (mlx4_priv(dev)->pci_dev_data & MLX4_PCI_DEV_FORCE_SENSE_PORT) + dev->caps.flags |= MLX4_DEV_CAP_FLAG_SENSE_SUPPORT; + /* Don't do sense port on multifunction devices (for now at least) */ + if (mlx4_is_mfunc(dev)) + dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_SENSE_SUPPORT; + + if (mlx4_low_memory_profile()) { + dev->caps.log_num_macs = MLX4_MIN_LOG_NUM_MAC; + dev->caps.log_num_vlans = MLX4_MIN_LOG_NUM_VLANS; + } else { + dev->caps.log_num_macs = log_num_mac; + dev->caps.log_num_vlans = MLX4_LOG_NUM_VLANS; + } + + for (i = 1; i <= dev->caps.num_ports; ++i) { + dev->caps.port_type[i] = MLX4_PORT_TYPE_NONE; + if (dev->caps.supported_type[i]) { + /* if only ETH is supported - assign ETH */ + if (dev->caps.supported_type[i] == MLX4_PORT_TYPE_ETH) + dev->caps.port_type[i] = MLX4_PORT_TYPE_ETH; + /* if only IB is supported, assign IB */ + else if (dev->caps.supported_type[i] == + MLX4_PORT_TYPE_IB) + dev->caps.port_type[i] = MLX4_PORT_TYPE_IB; + else { + /* if IB and ETH are supported, we set the port + * type according to user selection of port type; + * if user selected none, take the FW hint */ + if (port_type_array[i - 1] == MLX4_PORT_TYPE_NONE) + dev->caps.port_type[i] = dev->caps.suggested_type[i] ? + MLX4_PORT_TYPE_ETH : MLX4_PORT_TYPE_IB; + else + dev->caps.port_type[i] = port_type_array[i - 1]; + } + } + /* + * Link sensing is allowed on the port if 3 conditions are true: + * 1. Both protocols are supported on the port. + * 2. Different types are supported on the port + * 3. FW declared that it supports link sensing + */ + mlx4_priv(dev)->sense.sense_allowed[i] = + ((dev->caps.supported_type[i] == MLX4_PORT_TYPE_AUTO) && + (dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP) && + (dev->caps.flags & MLX4_DEV_CAP_FLAG_SENSE_SUPPORT)); + + /* + * If "default_sense" bit is set, we move the port to "AUTO" mode + * and perform sense_port FW command to try and set the correct + * port type from beginning + */ + if (mlx4_priv(dev)->sense.sense_allowed[i] && dev->caps.default_sense[i]) { + enum mlx4_port_type sensed_port = MLX4_PORT_TYPE_NONE; + dev->caps.possible_type[i] = MLX4_PORT_TYPE_AUTO; + mlx4_SENSE_PORT(dev, i, &sensed_port); + if (sensed_port != MLX4_PORT_TYPE_NONE) + dev->caps.port_type[i] = sensed_port; + } else { + dev->caps.possible_type[i] = dev->caps.port_type[i]; + } + + if (dev->caps.log_num_macs > dev_cap->port_cap[i].log_max_macs) { + dev->caps.log_num_macs = dev_cap->port_cap[i].log_max_macs; + mlx4_warn(dev, "Requested number of MACs is too much for port %d, reducing to %d\n", + i, 1 << dev->caps.log_num_macs); + } + if (dev->caps.log_num_vlans > dev_cap->port_cap[i].log_max_vlans) { + dev->caps.log_num_vlans = dev_cap->port_cap[i].log_max_vlans; + mlx4_warn(dev, "Requested number of VLANs is too much for port %d, reducing to %d\n", + i, 1 << dev->caps.log_num_vlans); + } + } + + if (mlx4_is_master(dev) && (dev->caps.num_ports == 2) && + (port_type_array[0] == MLX4_PORT_TYPE_IB) && + (port_type_array[1] == MLX4_PORT_TYPE_ETH)) { + mlx4_warn(dev, + "Granular QoS per VF not supported with IB/Eth configuration\n"); + dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_QOS_VPP; + } + + dev->caps.max_counters = dev_cap->max_counters; + + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] = dev_cap->reserved_qps; + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] = + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] = + (1 << dev->caps.log_num_macs) * + (1 << dev->caps.log_num_vlans) * + dev->caps.num_ports; + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH] = MLX4_NUM_FEXCH; + + if (dev_cap->dmfs_high_rate_qpn_base > 0 && + dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FS_EN) + dev->caps.dmfs_high_rate_qpn_base = dev_cap->dmfs_high_rate_qpn_base; + else + dev->caps.dmfs_high_rate_qpn_base = + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW]; + + if (dev_cap->dmfs_high_rate_qpn_range > 0 && + dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FS_EN) { + dev->caps.dmfs_high_rate_qpn_range = dev_cap->dmfs_high_rate_qpn_range; + dev->caps.dmfs_high_steer_mode = MLX4_STEERING_DMFS_A0_DEFAULT; + dev->caps.flags2 |= MLX4_DEV_CAP_FLAG2_FS_A0; + } else { + dev->caps.dmfs_high_steer_mode = MLX4_STEERING_DMFS_A0_NOT_SUPPORTED; + dev->caps.dmfs_high_rate_qpn_base = + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW]; + dev->caps.dmfs_high_rate_qpn_range = MLX4_A0_STEERING_TABLE_SIZE; + } + + dev->caps.rl_caps = dev_cap->rl_caps; + + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_RSS_RAW_ETH] = + dev->caps.dmfs_high_rate_qpn_range; + + dev->caps.reserved_qps = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] + + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] + + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] + + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH]; + + dev->caps.sqp_demux = (mlx4_is_master(dev)) ? MLX4_MAX_NUM_SLAVES : 0; + + if (!enable_64b_cqe_eqe && !mlx4_is_slave(dev)) { + if (dev_cap->flags & + (MLX4_DEV_CAP_FLAG_64B_CQE | MLX4_DEV_CAP_FLAG_64B_EQE)) { + mlx4_warn(dev, "64B EQEs/CQEs supported by the device but not enabled\n"); + dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_CQE; + dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_EQE; + } + + if (dev_cap->flags2 & + (MLX4_DEV_CAP_FLAG2_CQE_STRIDE | + MLX4_DEV_CAP_FLAG2_EQE_STRIDE)) { + mlx4_warn(dev, "Disabling EQE/CQE stride per user request\n"); + dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_CQE_STRIDE; + dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_EQE_STRIDE; + } + } + + if ((dev->caps.flags & + (MLX4_DEV_CAP_FLAG_64B_CQE | MLX4_DEV_CAP_FLAG_64B_EQE)) && + mlx4_is_master(dev)) + dev->caps.function_caps |= MLX4_FUNC_CAP_64B_EQE_CQE; + + if (!mlx4_is_slave(dev)) { + mlx4_enable_cqe_eqe_stride(dev); + dev->caps.alloc_res_qp_mask = + (dev->caps.bf_reg_size ? MLX4_RESERVE_ETH_BF_QP : 0) | + MLX4_RESERVE_A0_QP; + + if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETS_CFG) && + dev->caps.flags & MLX4_DEV_CAP_FLAG_SET_ETH_SCHED) { + mlx4_warn(dev, "Old device ETS support detected\n"); + mlx4_warn(dev, "Consider upgrading device FW.\n"); + dev->caps.flags2 |= MLX4_DEV_CAP_FLAG2_ETS_CFG; + } + + } else { + dev->caps.alloc_res_qp_mask = 0; + } + + mlx4_enable_ignore_fcs(dev); + + return 0; +} + +static int mlx4_get_pcie_dev_link_caps(struct mlx4_dev *dev, + enum pci_bus_speed *speed, + enum pcie_link_width *width) +{ + u32 lnkcap1, lnkcap2; + int err1, err2; + +#define PCIE_MLW_CAP_SHIFT 4 /* start of MLW mask in link capabilities */ + + *speed = PCI_SPEED_UNKNOWN; + *width = PCIE_LNK_WIDTH_UNKNOWN; + + err1 = pcie_capability_read_dword(dev->persist->pdev, PCI_EXP_LNKCAP, + &lnkcap1); + err2 = pcie_capability_read_dword(dev->persist->pdev, PCI_EXP_LNKCAP2, + &lnkcap2); + if (!err2 && lnkcap2) { /* PCIe r3.0-compliant */ + if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_8_0GB) + *speed = PCIE_SPEED_8_0GT; + else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_5_0GB) + *speed = PCIE_SPEED_5_0GT; + else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_2_5GB) + *speed = PCIE_SPEED_2_5GT; + } + if (!err1) { + *width = (lnkcap1 & PCI_EXP_LNKCAP_MLW) >> PCIE_MLW_CAP_SHIFT; + if (!lnkcap2) { /* pre-r3.0 */ + if (lnkcap1 & PCI_EXP_LNKCAP_SLS_5_0GB) + *speed = PCIE_SPEED_5_0GT; + else if (lnkcap1 & PCI_EXP_LNKCAP_SLS_2_5GB) + *speed = PCIE_SPEED_2_5GT; + } + } + + if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN) { + return err1 ? err1 : + err2 ? err2 : -EINVAL; + } + return 0; +} + +static void mlx4_check_pcie_caps(struct mlx4_dev *dev) +{ + enum pcie_link_width width, width_cap; + enum pci_bus_speed speed, speed_cap; + int err; + +#define PCIE_SPEED_STR(speed) \ + (speed == PCIE_SPEED_8_0GT ? "8.0GT/s" : \ + speed == PCIE_SPEED_5_0GT ? "5.0GT/s" : \ + speed == PCIE_SPEED_2_5GT ? "2.5GT/s" : \ + "Unknown") + + err = mlx4_get_pcie_dev_link_caps(dev, &speed_cap, &width_cap); + if (err) { + mlx4_warn(dev, + "Unable to determine PCIe device BW capabilities\n"); + return; + } + + err = pcie_get_minimum_link(dev->persist->pdev, &speed, &width); + if (err || speed == PCI_SPEED_UNKNOWN || + width == PCIE_LNK_WIDTH_UNKNOWN) { + mlx4_warn(dev, + "Unable to determine PCI device chain minimum BW\n"); + return; + } + + if (width != width_cap || speed != speed_cap) + mlx4_warn(dev, + "PCIe BW is different than device's capability\n"); + + mlx4_info(dev, "PCIe link speed is %s, device supports %s\n", + PCIE_SPEED_STR(speed), PCIE_SPEED_STR(speed_cap)); + mlx4_info(dev, "PCIe link width is x%d, device supports x%d\n", + width, width_cap); + return; +} + +/*The function checks if there are live vf, return the num of them*/ +static int mlx4_how_many_lives_vf(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *s_state; + int i; + int ret = 0; + + for (i = 1/*the ppf is 0*/; i < dev->num_slaves; ++i) { + s_state = &priv->mfunc.master.slave_state[i]; + if (s_state->active && s_state->last_cmd != + MLX4_COMM_CMD_RESET) { + mlx4_warn(dev, "%s: slave: %d is still active\n", + __func__, i); + ret++; + } + } + return ret; +} + +int mlx4_get_parav_qkey(struct mlx4_dev *dev, u32 qpn, u32 *qkey) +{ + u32 qk = MLX4_RESERVED_QKEY_BASE; + + if (qpn >= dev->phys_caps.base_tunnel_sqpn + 8 * MLX4_MFUNC_MAX || + qpn < dev->phys_caps.base_proxy_sqpn) + return -EINVAL; + + if (qpn >= dev->phys_caps.base_tunnel_sqpn) + /* tunnel qp */ + qk += qpn - dev->phys_caps.base_tunnel_sqpn; + else + qk += qpn - dev->phys_caps.base_proxy_sqpn; + *qkey = qk; + return 0; +} +EXPORT_SYMBOL(mlx4_get_parav_qkey); + +void mlx4_sync_pkey_table(struct mlx4_dev *dev, int slave, int port, int i, int val) +{ + struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev); + + if (!mlx4_is_master(dev)) + return; + + priv->virt2phys_pkey[slave][port - 1][i] = val; +} +EXPORT_SYMBOL(mlx4_sync_pkey_table); + +void mlx4_put_slave_node_guid(struct mlx4_dev *dev, int slave, __be64 guid) +{ + struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev); + + if (!mlx4_is_master(dev)) + return; + + priv->slave_node_guids[slave] = guid; +} +EXPORT_SYMBOL(mlx4_put_slave_node_guid); + +__be64 mlx4_get_slave_node_guid(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev); + + if (!mlx4_is_master(dev)) + return 0; + + return priv->slave_node_guids[slave]; +} +EXPORT_SYMBOL(mlx4_get_slave_node_guid); + +int mlx4_is_slave_active(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *s_slave; + + if (!mlx4_is_master(dev)) + return 0; + + s_slave = &priv->mfunc.master.slave_state[slave]; + return !!s_slave->active; +} +EXPORT_SYMBOL(mlx4_is_slave_active); + +static void slave_adjust_steering_mode(struct mlx4_dev *dev, + struct mlx4_dev_cap *dev_cap, + struct mlx4_init_hca_param *hca_param) +{ + dev->caps.steering_mode = hca_param->steering_mode; + if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) { + dev->caps.num_qp_per_mgm = dev_cap->fs_max_num_qp_per_entry; + dev->caps.fs_log_max_ucast_qp_range_size = + dev_cap->fs_log_max_ucast_qp_range_size; + } else + dev->caps.num_qp_per_mgm = + 4 * ((1 << hca_param->log_mc_entry_sz)/16 - 2); + + mlx4_dbg(dev, "Steering mode is: %s\n", + mlx4_steering_mode_str(dev->caps.steering_mode)); +} + +static int mlx4_slave_cap(struct mlx4_dev *dev) +{ + int err; + u32 page_size; + struct mlx4_dev_cap dev_cap; + struct mlx4_func_cap func_cap; + struct mlx4_init_hca_param hca_param; + u8 i; + + memset(&hca_param, 0, sizeof(hca_param)); + err = mlx4_QUERY_HCA(dev, &hca_param); + if (err) { + mlx4_err(dev, "QUERY_HCA command failed, aborting\n"); + return err; + } + + /* fail if the hca has an unknown global capability + * at this time global_caps should be always zeroed + */ + if (hca_param.global_caps) { + mlx4_err(dev, "Unknown hca global capabilities\n"); + return -ENOSYS; + } + + mlx4_log_num_mgm_entry_size = hca_param.log_mc_entry_sz; + + dev->caps.hca_core_clock = hca_param.hca_core_clock; + + memset(&dev_cap, 0, sizeof(dev_cap)); + dev->caps.max_qp_dest_rdma = 1 << hca_param.log_rd_per_qp; + err = mlx4_dev_cap(dev, &dev_cap); + if (err) { + mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n"); + return err; + } + + err = mlx4_QUERY_FW(dev); + if (err) + mlx4_err(dev, "QUERY_FW command failed: could not get FW version\n"); + + page_size = ~dev->caps.page_size_cap + 1; + mlx4_warn(dev, "HCA minimum page size:%d\n", page_size); + if (page_size > PAGE_SIZE) { + mlx4_err(dev, "HCA minimum page size of %d bigger than kernel PAGE_SIZE of %ld, aborting\n", + page_size, (long)PAGE_SIZE); + return -ENODEV; + } + + /* Set uar_page_shift for VF */ + dev->uar_page_shift = hca_param.uar_page_sz + 12; + + /* Make sure the master uar page size is valid */ + if (dev->uar_page_shift > PAGE_SHIFT) { + mlx4_err(dev, + "Invalid configuration: uar page size is larger than system page size\n"); + return -ENODEV; + } + + /* Set reserved_uars based on the uar_page_shift */ + mlx4_set_num_reserved_uars(dev, &dev_cap); + + /* Although uar page size in FW differs from system page size, + * upper software layers (mlx4_ib, mlx4_en and part of mlx4_core) + * still works with assumption that uar page size == system page size + */ + dev->caps.uar_page_size = PAGE_SIZE; + + memset(&func_cap, 0, sizeof(func_cap)); + err = mlx4_QUERY_FUNC_CAP(dev, 0, &func_cap); + if (err) { + mlx4_err(dev, "QUERY_FUNC_CAP general command failed, aborting (%d)\n", + err); + return err; + } + + if ((func_cap.pf_context_behaviour | PF_CONTEXT_BEHAVIOUR_MASK) != + PF_CONTEXT_BEHAVIOUR_MASK) { + mlx4_err(dev, "Unknown pf context behaviour %x known flags %x\n", + func_cap.pf_context_behaviour, PF_CONTEXT_BEHAVIOUR_MASK); + return -ENOSYS; + } + + dev->caps.num_ports = func_cap.num_ports; + dev->quotas.qp = func_cap.qp_quota; + dev->quotas.srq = func_cap.srq_quota; + dev->quotas.cq = func_cap.cq_quota; + dev->quotas.mpt = func_cap.mpt_quota; + dev->quotas.mtt = func_cap.mtt_quota; + dev->caps.num_qps = 1 << hca_param.log_num_qps; + dev->caps.num_srqs = 1 << hca_param.log_num_srqs; + dev->caps.num_cqs = 1 << hca_param.log_num_cqs; + dev->caps.num_mpts = 1 << hca_param.log_mpt_sz; + dev->caps.num_eqs = func_cap.max_eq; + dev->caps.reserved_eqs = func_cap.reserved_eq; + dev->caps.reserved_lkey = func_cap.reserved_lkey; + dev->caps.num_pds = MLX4_NUM_PDS; + dev->caps.num_mgms = 0; + dev->caps.num_amgms = 0; + + if (dev->caps.num_ports > MLX4_MAX_PORTS) { + mlx4_err(dev, "HCA has %d ports, but we only support %d, aborting\n", + dev->caps.num_ports, MLX4_MAX_PORTS); + return -ENODEV; + } + + mlx4_replace_zero_macs(dev); + + dev->caps.qp0_qkey = kcalloc(dev->caps.num_ports, sizeof(u32), GFP_KERNEL); + dev->caps.qp0_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); + dev->caps.qp0_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); + dev->caps.qp1_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); + dev->caps.qp1_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); + + if (!dev->caps.qp0_tunnel || !dev->caps.qp0_proxy || + !dev->caps.qp1_tunnel || !dev->caps.qp1_proxy || + !dev->caps.qp0_qkey) { + err = -ENOMEM; + goto err_mem; + } + + for (i = 1; i <= dev->caps.num_ports; ++i) { + err = mlx4_QUERY_FUNC_CAP(dev, i, &func_cap); + if (err) { + mlx4_err(dev, "QUERY_FUNC_CAP port command failed for port %d, aborting (%d)\n", + i, err); + goto err_mem; + } + dev->caps.qp0_qkey[i - 1] = func_cap.qp0_qkey; + dev->caps.qp0_tunnel[i - 1] = func_cap.qp0_tunnel_qpn; + dev->caps.qp0_proxy[i - 1] = func_cap.qp0_proxy_qpn; + dev->caps.qp1_tunnel[i - 1] = func_cap.qp1_tunnel_qpn; + dev->caps.qp1_proxy[i - 1] = func_cap.qp1_proxy_qpn; + dev->caps.port_mask[i] = dev->caps.port_type[i]; + dev->caps.phys_port_id[i] = func_cap.phys_port_id; + err = mlx4_get_slave_pkey_gid_tbl_len(dev, i, + &dev->caps.gid_table_len[i], + &dev->caps.pkey_table_len[i]); + if (err) + goto err_mem; + } + + if (dev->caps.uar_page_size * (dev->caps.num_uars - + dev->caps.reserved_uars) > + pci_resource_len(dev->persist->pdev, + 2)) { + mlx4_err(dev, "HCA reported UAR region size of 0x%x bigger than PCI resource 2 size of 0x%llx, aborting\n", + dev->caps.uar_page_size * dev->caps.num_uars, + (unsigned long long) + pci_resource_len(dev->persist->pdev, 2)); + err = -ENOMEM; + goto err_mem; + } + + if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_64B_EQE_ENABLED) { + dev->caps.eqe_size = 64; + dev->caps.eqe_factor = 1; + } else { + dev->caps.eqe_size = 32; + dev->caps.eqe_factor = 0; + } + + if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_64B_CQE_ENABLED) { + dev->caps.cqe_size = 64; + dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_LARGE_CQE; + } else { + dev->caps.cqe_size = 32; + } + + if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_EQE_STRIDE_ENABLED) { + dev->caps.eqe_size = hca_param.eqe_size; + dev->caps.eqe_factor = 0; + } + + if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_CQE_STRIDE_ENABLED) { + dev->caps.cqe_size = hca_param.cqe_size; + /* User still need to know when CQE > 32B */ + dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_LARGE_CQE; + } + + dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS; + mlx4_warn(dev, "Timestamping is not supported in slave mode\n"); + + slave_adjust_steering_mode(dev, &dev_cap, &hca_param); + mlx4_dbg(dev, "RSS support for IP fragments is %s\n", + hca_param.rss_ip_frags ? "on" : "off"); + + if (func_cap.extra_flags & MLX4_QUERY_FUNC_FLAGS_BF_RES_QP && + dev->caps.bf_reg_size) + dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_ETH_BF_QP; + + if (func_cap.extra_flags & MLX4_QUERY_FUNC_FLAGS_A0_RES_QP) + dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_A0_QP; + + return 0; + +err_mem: + kfree(dev->caps.qp0_qkey); + kfree(dev->caps.qp0_tunnel); + kfree(dev->caps.qp0_proxy); + kfree(dev->caps.qp1_tunnel); + kfree(dev->caps.qp1_proxy); + dev->caps.qp0_qkey = NULL; + dev->caps.qp0_tunnel = NULL; + dev->caps.qp0_proxy = NULL; + dev->caps.qp1_tunnel = NULL; + dev->caps.qp1_proxy = NULL; + + return err; +} + +static void mlx4_request_modules(struct mlx4_dev *dev) +{ + int port; + int has_ib_port = false; + int has_eth_port = false; +#define EN_DRV_NAME "mlx4_en" +#define IB_DRV_NAME "mlx4_ib" + + for (port = 1; port <= dev->caps.num_ports; port++) { + if (dev->caps.port_type[port] == MLX4_PORT_TYPE_IB) + has_ib_port = true; + else if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) + has_eth_port = true; + } + + if (has_eth_port) + request_module_nowait(EN_DRV_NAME); + if (has_ib_port || (dev->caps.flags & MLX4_DEV_CAP_FLAG_IBOE)) + request_module_nowait(IB_DRV_NAME); +} + +/* + * Change the port configuration of the device. + * Every user of this function must hold the port mutex. + */ +int mlx4_change_port_types(struct mlx4_dev *dev, + enum mlx4_port_type *port_types) +{ + int err = 0; + int change = 0; + int port; + + for (port = 0; port < dev->caps.num_ports; port++) { + /* Change the port type only if the new type is different + * from the current, and not set to Auto */ + if (port_types[port] != dev->caps.port_type[port + 1]) + change = 1; + } + if (change) { + mlx4_unregister_device(dev); + for (port = 1; port <= dev->caps.num_ports; port++) { + mlx4_CLOSE_PORT(dev, port); + dev->caps.port_type[port] = port_types[port - 1]; + err = mlx4_SET_PORT(dev, port, -1); + if (err) { + mlx4_err(dev, "Failed to set port %d, aborting\n", + port); + goto out; + } + } + mlx4_set_port_mask(dev); + err = mlx4_register_device(dev); + if (err) { + mlx4_err(dev, "Failed to register device\n"); + goto out; + } + mlx4_request_modules(dev); + } + +out: + return err; +} + +static ssize_t show_port_type(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info, + port_attr); + struct mlx4_dev *mdev = info->dev; + char type[8]; + + sprintf(type, "%s", + (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_IB) ? + "ib" : "eth"); + if (mdev->caps.possible_type[info->port] == MLX4_PORT_TYPE_AUTO) + sprintf(buf, "auto (%s)\n", type); + else + sprintf(buf, "%s\n", type); + + return strlen(buf); +} + +static int __set_port_type(struct mlx4_port_info *info, + enum mlx4_port_type port_type) +{ + struct mlx4_dev *mdev = info->dev; + struct mlx4_priv *priv = mlx4_priv(mdev); + enum mlx4_port_type types[MLX4_MAX_PORTS]; + enum mlx4_port_type new_types[MLX4_MAX_PORTS]; + int i; + int err = 0; + + if ((port_type & mdev->caps.supported_type[info->port]) != port_type) { + mlx4_err(mdev, + "Requested port type for port %d is not supported on this HCA\n", + info->port); + err = -EINVAL; + goto err_sup; + } + + mlx4_stop_sense(mdev); + mutex_lock(&priv->port_mutex); + info->tmp_type = port_type; + + /* Possible type is always the one that was delivered */ + mdev->caps.possible_type[info->port] = info->tmp_type; + + for (i = 0; i < mdev->caps.num_ports; i++) { + types[i] = priv->port[i+1].tmp_type ? priv->port[i+1].tmp_type : + mdev->caps.possible_type[i+1]; + if (types[i] == MLX4_PORT_TYPE_AUTO) + types[i] = mdev->caps.port_type[i+1]; + } + + if (!(mdev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP) && + !(mdev->caps.flags & MLX4_DEV_CAP_FLAG_SENSE_SUPPORT)) { + for (i = 1; i <= mdev->caps.num_ports; i++) { + if (mdev->caps.possible_type[i] == MLX4_PORT_TYPE_AUTO) { + mdev->caps.possible_type[i] = mdev->caps.port_type[i]; + err = -EINVAL; + } + } + } + if (err) { + mlx4_err(mdev, "Auto sensing is not supported on this HCA. Set only 'eth' or 'ib' for both ports (should be the same)\n"); + goto out; + } + + mlx4_do_sense_ports(mdev, new_types, types); + + err = mlx4_check_port_params(mdev, new_types); + if (err) + goto out; + + /* We are about to apply the changes after the configuration + * was verified, no need to remember the temporary types + * any more */ + for (i = 0; i < mdev->caps.num_ports; i++) + priv->port[i + 1].tmp_type = 0; + + err = mlx4_change_port_types(mdev, new_types); + +out: + mutex_unlock(&priv->port_mutex); + mlx4_start_sense(mdev); +err_sup: + return err; +} + +static ssize_t set_port_type(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info, + port_attr); + struct mlx4_dev *mdev = info->dev; + enum mlx4_port_type port_type; + static DEFINE_MUTEX(set_port_type_mutex); + int err; + + mutex_lock(&set_port_type_mutex); + + if (!strcmp(buf, "ib\n")) { + port_type = MLX4_PORT_TYPE_IB; + } else if (!strcmp(buf, "eth\n")) { + port_type = MLX4_PORT_TYPE_ETH; + } else if (!strcmp(buf, "auto\n")) { + port_type = MLX4_PORT_TYPE_AUTO; + } else { + mlx4_err(mdev, "%s is not supported port type\n", buf); + err = -EINVAL; + goto err_out; + } + + err = __set_port_type(info, port_type); + +err_out: + mutex_unlock(&set_port_type_mutex); + + return err ? err : count; +} + +enum ibta_mtu { + IB_MTU_256 = 1, + IB_MTU_512 = 2, + IB_MTU_1024 = 3, + IB_MTU_2048 = 4, + IB_MTU_4096 = 5 +}; + +static inline int int_to_ibta_mtu(int mtu) +{ + switch (mtu) { + case 256: return IB_MTU_256; + case 512: return IB_MTU_512; + case 1024: return IB_MTU_1024; + case 2048: return IB_MTU_2048; + case 4096: return IB_MTU_4096; + default: return -1; + } +} + +static inline int ibta_mtu_to_int(enum ibta_mtu mtu) +{ + switch (mtu) { + case IB_MTU_256: return 256; + case IB_MTU_512: return 512; + case IB_MTU_1024: return 1024; + case IB_MTU_2048: return 2048; + case IB_MTU_4096: return 4096; + default: return -1; + } +} + +static ssize_t show_port_ib_mtu(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info, + port_mtu_attr); + struct mlx4_dev *mdev = info->dev; + + if (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_ETH) + mlx4_warn(mdev, "port level mtu is only used for IB ports\n"); + + sprintf(buf, "%d\n", + ibta_mtu_to_int(mdev->caps.port_ib_mtu[info->port])); + return strlen(buf); +} + +static ssize_t set_port_ib_mtu(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info, + port_mtu_attr); + struct mlx4_dev *mdev = info->dev; + struct mlx4_priv *priv = mlx4_priv(mdev); + int err, port, mtu, ibta_mtu = -1; + + if (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_ETH) { + mlx4_warn(mdev, "port level mtu is only used for IB ports\n"); + return -EINVAL; + } + + err = kstrtoint(buf, 0, &mtu); + if (!err) + ibta_mtu = int_to_ibta_mtu(mtu); + + if (err || ibta_mtu < 0) { + mlx4_err(mdev, "%s is invalid IBTA mtu\n", buf); + return -EINVAL; + } + + mdev->caps.port_ib_mtu[info->port] = ibta_mtu; + + mlx4_stop_sense(mdev); + mutex_lock(&priv->port_mutex); + mlx4_unregister_device(mdev); + for (port = 1; port <= mdev->caps.num_ports; port++) { + mlx4_CLOSE_PORT(mdev, port); + err = mlx4_SET_PORT(mdev, port, -1); + if (err) { + mlx4_err(mdev, "Failed to set port %d, aborting\n", + port); + goto err_set_port; + } + } + err = mlx4_register_device(mdev); +err_set_port: + mutex_unlock(&priv->port_mutex); + mlx4_start_sense(mdev); + return err ? err : count; +} + +/* bond for multi-function device */ +#define MAX_MF_BOND_ALLOWED_SLAVES 63 +static int mlx4_mf_bond(struct mlx4_dev *dev) +{ + int err = 0; + int nvfs; + struct mlx4_slaves_pport slaves_port1; + struct mlx4_slaves_pport slaves_port2; + DECLARE_BITMAP(slaves_port_1_2, MLX4_MFUNC_MAX); + + slaves_port1 = mlx4_phys_to_slaves_pport(dev, 1); + slaves_port2 = mlx4_phys_to_slaves_pport(dev, 2); + bitmap_and(slaves_port_1_2, + slaves_port1.slaves, slaves_port2.slaves, + dev->persist->num_vfs + 1); + + /* only single port vfs are allowed */ + if (bitmap_weight(slaves_port_1_2, dev->persist->num_vfs + 1) > 1) { + mlx4_warn(dev, "HA mode unsupported for dual ported VFs\n"); + return -EINVAL; + } + + /* number of virtual functions is number of total functions minus one + * physical function for each port. + */ + nvfs = bitmap_weight(slaves_port1.slaves, dev->persist->num_vfs + 1) + + bitmap_weight(slaves_port2.slaves, dev->persist->num_vfs + 1) - 2; + + /* limit on maximum allowed VFs */ + if (nvfs > MAX_MF_BOND_ALLOWED_SLAVES) { + mlx4_warn(dev, "HA mode is not supported for %d VFs (max %d are allowed)\n", + nvfs, MAX_MF_BOND_ALLOWED_SLAVES); + return -EINVAL; + } + + if (dev->caps.steering_mode != MLX4_STEERING_MODE_DEVICE_MANAGED) { + mlx4_warn(dev, "HA mode unsupported for NON DMFS steering\n"); + return -EINVAL; + } + + err = mlx4_bond_mac_table(dev); + if (err) + return err; + err = mlx4_bond_vlan_table(dev); + if (err) + goto err1; + err = mlx4_bond_fs_rules(dev); + if (err) + goto err2; + + return 0; +err2: + (void)mlx4_unbond_vlan_table(dev); +err1: + (void)mlx4_unbond_mac_table(dev); + return err; +} + +static int mlx4_mf_unbond(struct mlx4_dev *dev) +{ + int ret, ret1; + + ret = mlx4_unbond_fs_rules(dev); + if (ret) + mlx4_warn(dev, "multifunction unbond for flow rules failedi (%d)\n", ret); + ret1 = mlx4_unbond_mac_table(dev); + if (ret1) { + mlx4_warn(dev, "multifunction unbond for MAC table failed (%d)\n", ret1); + ret = ret1; + } + ret1 = mlx4_unbond_vlan_table(dev); + if (ret1) { + mlx4_warn(dev, "multifunction unbond for VLAN table failed (%d)\n", ret1); + ret = ret1; + } + return ret; +} + +int mlx4_bond(struct mlx4_dev *dev) +{ + int ret = 0; + struct mlx4_priv *priv = mlx4_priv(dev); + + mutex_lock(&priv->bond_mutex); + + if (!mlx4_is_bonded(dev)) { + ret = mlx4_do_bond(dev, true); + if (ret) + mlx4_err(dev, "Failed to bond device: %d\n", ret); + if (!ret && mlx4_is_master(dev)) { + ret = mlx4_mf_bond(dev); + if (ret) { + mlx4_err(dev, "bond for multifunction failed\n"); + mlx4_do_bond(dev, false); + } + } + } + + mutex_unlock(&priv->bond_mutex); + if (!ret) + mlx4_dbg(dev, "Device is bonded\n"); + + return ret; +} +EXPORT_SYMBOL_GPL(mlx4_bond); + +int mlx4_unbond(struct mlx4_dev *dev) +{ + int ret = 0; + struct mlx4_priv *priv = mlx4_priv(dev); + + mutex_lock(&priv->bond_mutex); + + if (mlx4_is_bonded(dev)) { + int ret2 = 0; + + ret = mlx4_do_bond(dev, false); + if (ret) + mlx4_err(dev, "Failed to unbond device: %d\n", ret); + if (mlx4_is_master(dev)) + ret2 = mlx4_mf_unbond(dev); + if (ret2) { + mlx4_warn(dev, "Failed to unbond device for multifunction (%d)\n", ret2); + ret = ret2; + } + } + + mutex_unlock(&priv->bond_mutex); + if (!ret) + mlx4_dbg(dev, "Device is unbonded\n"); + + return ret; +} +EXPORT_SYMBOL_GPL(mlx4_unbond); + + +int mlx4_port_map_set(struct mlx4_dev *dev, struct mlx4_port_map *v2p) +{ + u8 port1 = v2p->port1; + u8 port2 = v2p->port2; + struct mlx4_priv *priv = mlx4_priv(dev); + int err; + + if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PORT_REMAP)) + return -ENOTSUPP; + + mutex_lock(&priv->bond_mutex); + + /* zero means keep current mapping for this port */ + if (port1 == 0) + port1 = priv->v2p.port1; + if (port2 == 0) + port2 = priv->v2p.port2; + + if ((port1 < 1) || (port1 > MLX4_MAX_PORTS) || + (port2 < 1) || (port2 > MLX4_MAX_PORTS) || + (port1 == 2 && port2 == 1)) { + /* besides boundary checks cross mapping makes + * no sense and therefore not allowed */ + err = -EINVAL; + } else if ((port1 == priv->v2p.port1) && + (port2 == priv->v2p.port2)) { + err = 0; + } else { + err = mlx4_virt2phy_port_map(dev, port1, port2); + if (!err) { + mlx4_dbg(dev, "port map changed: [%d][%d]\n", + port1, port2); + priv->v2p.port1 = port1; + priv->v2p.port2 = port2; + } else { + mlx4_err(dev, "Failed to change port mape: %d\n", err); + } + } + + mutex_unlock(&priv->bond_mutex); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_port_map_set); + +static int mlx4_load_fw(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int err; + + priv->fw.fw_icm = mlx4_alloc_icm(dev, priv->fw.fw_pages, + GFP_HIGHUSER | __GFP_NOWARN, 0); + if (!priv->fw.fw_icm) { + mlx4_err(dev, "Couldn't allocate FW area, aborting\n"); + return -ENOMEM; + } + + err = mlx4_MAP_FA(dev, priv->fw.fw_icm); + if (err) { + mlx4_err(dev, "MAP_FA command failed, aborting\n"); + goto err_free; + } + + err = mlx4_RUN_FW(dev); + if (err) { + mlx4_err(dev, "RUN_FW command failed, aborting\n"); + goto err_unmap_fa; + } + + return 0; + +err_unmap_fa: + mlx4_UNMAP_FA(dev); + +err_free: + mlx4_free_icm(dev, priv->fw.fw_icm, 0); + return err; +} + +static int mlx4_init_cmpt_table(struct mlx4_dev *dev, u64 cmpt_base, + int cmpt_entry_sz) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int err; + int num_eqs; + + err = mlx4_init_icm_table(dev, &priv->qp_table.cmpt_table, + cmpt_base + + ((u64) (MLX4_CMPT_TYPE_QP * + cmpt_entry_sz) << MLX4_CMPT_SHIFT), + cmpt_entry_sz, dev->caps.num_qps, + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], + 0, 0); + if (err) + goto err; + + err = mlx4_init_icm_table(dev, &priv->srq_table.cmpt_table, + cmpt_base + + ((u64) (MLX4_CMPT_TYPE_SRQ * + cmpt_entry_sz) << MLX4_CMPT_SHIFT), + cmpt_entry_sz, dev->caps.num_srqs, + dev->caps.reserved_srqs, 0, 0); + if (err) + goto err_qp; + + err = mlx4_init_icm_table(dev, &priv->cq_table.cmpt_table, + cmpt_base + + ((u64) (MLX4_CMPT_TYPE_CQ * + cmpt_entry_sz) << MLX4_CMPT_SHIFT), + cmpt_entry_sz, dev->caps.num_cqs, + dev->caps.reserved_cqs, 0, 0); + if (err) + goto err_srq; + + num_eqs = dev->phys_caps.num_phys_eqs; + err = mlx4_init_icm_table(dev, &priv->eq_table.cmpt_table, + cmpt_base + + ((u64) (MLX4_CMPT_TYPE_EQ * + cmpt_entry_sz) << MLX4_CMPT_SHIFT), + cmpt_entry_sz, num_eqs, num_eqs, 0, 0); + if (err) + goto err_cq; + + return 0; + +err_cq: + mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table); + +err_srq: + mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table); + +err_qp: + mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table); + +err: + return err; +} + +static int mlx4_init_icm(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap, + struct mlx4_init_hca_param *init_hca, u64 icm_size) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + u64 aux_pages; + int num_eqs; + int err; + + err = mlx4_SET_ICM_SIZE(dev, icm_size, &aux_pages); + if (err) { + mlx4_err(dev, "SET_ICM_SIZE command failed, aborting\n"); + return err; + } + + mlx4_dbg(dev, "%lld KB of HCA context requires %lld KB aux memory\n", + (unsigned long long) icm_size >> 10, + (unsigned long long) aux_pages << 2); + + priv->fw.aux_icm = mlx4_alloc_icm(dev, aux_pages, + GFP_HIGHUSER | __GFP_NOWARN, 0); + if (!priv->fw.aux_icm) { + mlx4_err(dev, "Couldn't allocate aux memory, aborting\n"); + return -ENOMEM; + } + + err = mlx4_MAP_ICM_AUX(dev, priv->fw.aux_icm); + if (err) { + mlx4_err(dev, "MAP_ICM_AUX command failed, aborting\n"); + goto err_free_aux; + } + + err = mlx4_init_cmpt_table(dev, init_hca->cmpt_base, dev_cap->cmpt_entry_sz); + if (err) { + mlx4_err(dev, "Failed to map cMPT context memory, aborting\n"); + goto err_unmap_aux; + } + + + num_eqs = dev->phys_caps.num_phys_eqs; + err = mlx4_init_icm_table(dev, &priv->eq_table.table, + init_hca->eqc_base, dev_cap->eqc_entry_sz, + num_eqs, num_eqs, 0, 0); + if (err) { + mlx4_err(dev, "Failed to map EQ context memory, aborting\n"); + goto err_unmap_cmpt; + } + + /* + * Reserved MTT entries must be aligned up to a cacheline + * boundary, since the FW will write to them, while the driver + * writes to all other MTT entries. (The variable + * dev->caps.mtt_entry_sz below is really the MTT segment + * size, not the raw entry size) + */ + dev->caps.reserved_mtts = + ALIGN(dev->caps.reserved_mtts * dev->caps.mtt_entry_sz, + dma_get_cache_alignment()) / dev->caps.mtt_entry_sz; + + err = mlx4_init_icm_table(dev, &priv->mr_table.mtt_table, + init_hca->mtt_base, + dev->caps.mtt_entry_sz, + dev->caps.num_mtts, + dev->caps.reserved_mtts, 1, 0); + if (err) { + mlx4_err(dev, "Failed to map MTT context memory, aborting\n"); + goto err_unmap_eq; + } + + err = mlx4_init_icm_table(dev, &priv->mr_table.dmpt_table, + init_hca->dmpt_base, + dev_cap->dmpt_entry_sz, + dev->caps.num_mpts, + dev->caps.reserved_mrws, 1, 1); + if (err) { + mlx4_err(dev, "Failed to map dMPT context memory, aborting\n"); + goto err_unmap_mtt; + } + + err = mlx4_init_icm_table(dev, &priv->qp_table.qp_table, + init_hca->qpc_base, + dev_cap->qpc_entry_sz, + dev->caps.num_qps, + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], + 0, 0); + if (err) { + mlx4_err(dev, "Failed to map QP context memory, aborting\n"); + goto err_unmap_dmpt; + } + + err = mlx4_init_icm_table(dev, &priv->qp_table.auxc_table, + init_hca->auxc_base, + dev_cap->aux_entry_sz, + dev->caps.num_qps, + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], + 0, 0); + if (err) { + mlx4_err(dev, "Failed to map AUXC context memory, aborting\n"); + goto err_unmap_qp; + } + + err = mlx4_init_icm_table(dev, &priv->qp_table.altc_table, + init_hca->altc_base, + dev_cap->altc_entry_sz, + dev->caps.num_qps, + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], + 0, 0); + if (err) { + mlx4_err(dev, "Failed to map ALTC context memory, aborting\n"); + goto err_unmap_auxc; + } + + err = mlx4_init_icm_table(dev, &priv->qp_table.rdmarc_table, + init_hca->rdmarc_base, + dev_cap->rdmarc_entry_sz << priv->qp_table.rdmarc_shift, + dev->caps.num_qps, + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], + 0, 0); + if (err) { + mlx4_err(dev, "Failed to map RDMARC context memory, aborting\n"); + goto err_unmap_altc; + } + + err = mlx4_init_icm_table(dev, &priv->cq_table.table, + init_hca->cqc_base, + dev_cap->cqc_entry_sz, + dev->caps.num_cqs, + dev->caps.reserved_cqs, 0, 0); + if (err) { + mlx4_err(dev, "Failed to map CQ context memory, aborting\n"); + goto err_unmap_rdmarc; + } + + err = mlx4_init_icm_table(dev, &priv->srq_table.table, + init_hca->srqc_base, + dev_cap->srq_entry_sz, + dev->caps.num_srqs, + dev->caps.reserved_srqs, 0, 0); + if (err) { + mlx4_err(dev, "Failed to map SRQ context memory, aborting\n"); + goto err_unmap_cq; + } + + /* + * For flow steering device managed mode it is required to use + * mlx4_init_icm_table. For B0 steering mode it's not strictly + * required, but for simplicity just map the whole multicast + * group table now. The table isn't very big and it's a lot + * easier than trying to track ref counts. + */ + err = mlx4_init_icm_table(dev, &priv->mcg_table.table, + init_hca->mc_base, + mlx4_get_mgm_entry_size(dev), + dev->caps.num_mgms + dev->caps.num_amgms, + dev->caps.num_mgms + dev->caps.num_amgms, + 0, 0); + if (err) { + mlx4_err(dev, "Failed to map MCG context memory, aborting\n"); + goto err_unmap_srq; + } + + return 0; + +err_unmap_srq: + mlx4_cleanup_icm_table(dev, &priv->srq_table.table); + +err_unmap_cq: + mlx4_cleanup_icm_table(dev, &priv->cq_table.table); + +err_unmap_rdmarc: + mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table); + +err_unmap_altc: + mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table); + +err_unmap_auxc: + mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table); + +err_unmap_qp: + mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table); + +err_unmap_dmpt: + mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table); + +err_unmap_mtt: + mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table); + +err_unmap_eq: + mlx4_cleanup_icm_table(dev, &priv->eq_table.table); + +err_unmap_cmpt: + mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table); + mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table); + mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table); + mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table); + +err_unmap_aux: + mlx4_UNMAP_ICM_AUX(dev); + +err_free_aux: + mlx4_free_icm(dev, priv->fw.aux_icm, 0); + + return err; +} + +static void mlx4_free_icms(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + mlx4_cleanup_icm_table(dev, &priv->mcg_table.table); + mlx4_cleanup_icm_table(dev, &priv->srq_table.table); + mlx4_cleanup_icm_table(dev, &priv->cq_table.table); + mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table); + mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table); + mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table); + mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table); + mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table); + mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table); + mlx4_cleanup_icm_table(dev, &priv->eq_table.table); + mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table); + mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table); + mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table); + mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table); + + mlx4_UNMAP_ICM_AUX(dev); + mlx4_free_icm(dev, priv->fw.aux_icm, 0); +} + +static void mlx4_slave_exit(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + mutex_lock(&priv->cmd.slave_cmd_mutex); + if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, MLX4_COMM_CMD_NA_OP, + MLX4_COMM_TIME)) + mlx4_warn(dev, "Failed to close slave function\n"); + mutex_unlock(&priv->cmd.slave_cmd_mutex); +} + +static int map_bf_area(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + resource_size_t bf_start; + resource_size_t bf_len; + int err = 0; + + if (!dev->caps.bf_reg_size) + return -ENXIO; + + bf_start = pci_resource_start(dev->persist->pdev, 2) + + (dev->caps.num_uars << PAGE_SHIFT); + bf_len = pci_resource_len(dev->persist->pdev, 2) - + (dev->caps.num_uars << PAGE_SHIFT); + priv->bf_mapping = io_mapping_create_wc(bf_start, bf_len); + if (!priv->bf_mapping) + err = -ENOMEM; + + return err; +} + +static void unmap_bf_area(struct mlx4_dev *dev) +{ + if (mlx4_priv(dev)->bf_mapping) + io_mapping_free(mlx4_priv(dev)->bf_mapping); +} + +s64 mlx4_read_clock(struct mlx4_dev *dev) +{ + u32 clockhi, clocklo, clockhi1; + s64 cycles; + int i; + struct mlx4_priv *priv = mlx4_priv(dev); + + if (!priv->clock_mapping) + return -ENOTSUPP; + + for (i = 0; i < 10; i++) { + clockhi = swab32(readl(priv->clock_mapping)); + clocklo = swab32(readl(priv->clock_mapping + 4)); + clockhi1 = swab32(readl(priv->clock_mapping)); + if (clockhi == clockhi1) + break; + } + + cycles = (u64) clockhi << 32 | (u64) clocklo; + + return cycles & CORE_CLOCK_MASK; +} +EXPORT_SYMBOL_GPL(mlx4_read_clock); + + +static int map_internal_clock(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + priv->clock_mapping = + ioremap(pci_resource_start(dev->persist->pdev, + priv->fw.clock_bar) + + priv->fw.clock_offset, MLX4_CLOCK_SIZE); + + if (!priv->clock_mapping) + return -ENOMEM; + + return 0; +} + +int mlx4_get_internal_clock_params(struct mlx4_dev *dev, + struct mlx4_clock_params *params) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + if (mlx4_is_slave(dev)) + return -ENOTSUPP; + + if (!params) + return -EINVAL; + + params->bar = priv->fw.clock_bar; + params->offset = priv->fw.clock_offset; + params->size = MLX4_CLOCK_SIZE; + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_get_internal_clock_params); + +static void unmap_internal_clock(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + if (priv->clock_mapping) + iounmap(priv->clock_mapping); +} + +static void mlx4_close_hca(struct mlx4_dev *dev) +{ + unmap_internal_clock(dev); + unmap_bf_area(dev); + if (mlx4_is_slave(dev)) + mlx4_slave_exit(dev); + else { + mlx4_CLOSE_HCA(dev, 0); + mlx4_free_icms(dev); + } +} + +static void mlx4_close_fw(struct mlx4_dev *dev) +{ + if (!mlx4_is_slave(dev)) { + mlx4_UNMAP_FA(dev); + mlx4_free_icm(dev, mlx4_priv(dev)->fw.fw_icm, 0); + } +} + +static int mlx4_comm_check_offline(struct mlx4_dev *dev) +{ +#define COMM_CHAN_OFFLINE_OFFSET 0x09 + + u32 comm_flags; + u32 offline_bit; + unsigned long end; + struct mlx4_priv *priv = mlx4_priv(dev); + + end = msecs_to_jiffies(MLX4_COMM_OFFLINE_TIME_OUT) + jiffies; + while (time_before(jiffies, end)) { + comm_flags = swab32(readl((__iomem char *)priv->mfunc.comm + + MLX4_COMM_CHAN_FLAGS)); + offline_bit = (comm_flags & + (u32)(1 << COMM_CHAN_OFFLINE_OFFSET)); + if (!offline_bit) + return 0; + /* There are cases as part of AER/Reset flow that PF needs + * around 100 msec to load. We therefore sleep for 100 msec + * to allow other tasks to make use of that CPU during this + * time interval. + */ + msleep(100); + } + mlx4_err(dev, "Communication channel is offline.\n"); + return -EIO; +} + +static void mlx4_reset_vf_support(struct mlx4_dev *dev) +{ +#define COMM_CHAN_RST_OFFSET 0x1e + + struct mlx4_priv *priv = mlx4_priv(dev); + u32 comm_rst; + u32 comm_caps; + + comm_caps = swab32(readl((__iomem char *)priv->mfunc.comm + + MLX4_COMM_CHAN_CAPS)); + comm_rst = (comm_caps & (u32)(1 << COMM_CHAN_RST_OFFSET)); + + if (comm_rst) + dev->caps.vf_caps |= MLX4_VF_CAP_FLAG_RESET; +} + +static int mlx4_init_slave(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + u64 dma = (u64) priv->mfunc.vhcr_dma; + int ret_from_reset = 0; + u32 slave_read; + u32 cmd_channel_ver; + + if (atomic_read(&pf_loading)) { + mlx4_warn(dev, "PF is not ready - Deferring probe\n"); + return -EAGAIN; + } + + mutex_lock(&priv->cmd.slave_cmd_mutex); + priv->cmd.max_cmds = 1; + if (mlx4_comm_check_offline(dev)) { + mlx4_err(dev, "PF is not responsive, skipping initialization\n"); + goto err_offline; + } + + mlx4_reset_vf_support(dev); + mlx4_warn(dev, "Sending reset\n"); + ret_from_reset = mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, + MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME); + /* if we are in the middle of flr the slave will try + * NUM_OF_RESET_RETRIES times before leaving.*/ + if (ret_from_reset) { + if (MLX4_DELAY_RESET_SLAVE == ret_from_reset) { + mlx4_warn(dev, "slave is currently in the middle of FLR - Deferring probe\n"); + mutex_unlock(&priv->cmd.slave_cmd_mutex); + return -EAGAIN; + } else + goto err; + } + + /* check the driver version - the slave I/F revision + * must match the master's */ + slave_read = swab32(readl(&priv->mfunc.comm->slave_read)); + cmd_channel_ver = mlx4_comm_get_version(); + + if (MLX4_COMM_GET_IF_REV(cmd_channel_ver) != + MLX4_COMM_GET_IF_REV(slave_read)) { + mlx4_err(dev, "slave driver version is not supported by the master\n"); + goto err; + } + + mlx4_warn(dev, "Sending vhcr0\n"); + if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR0, dma >> 48, + MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME)) + goto err; + if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR1, dma >> 32, + MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME)) + goto err; + if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR2, dma >> 16, + MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME)) + goto err; + if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR_EN, dma, + MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME)) + goto err; + + mutex_unlock(&priv->cmd.slave_cmd_mutex); + return 0; + +err: + mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, MLX4_COMM_CMD_NA_OP, 0); +err_offline: + mutex_unlock(&priv->cmd.slave_cmd_mutex); + return -EIO; +} + +static void mlx4_parav_master_pf_caps(struct mlx4_dev *dev) +{ + int i; + + for (i = 1; i <= dev->caps.num_ports; i++) { + if (dev->caps.port_type[i] == MLX4_PORT_TYPE_ETH) + dev->caps.gid_table_len[i] = + mlx4_get_slave_num_gids(dev, 0, i); + else + dev->caps.gid_table_len[i] = 1; + dev->caps.pkey_table_len[i] = + dev->phys_caps.pkey_phys_table_len[i] - 1; + } +} + +static int choose_log_fs_mgm_entry_size(int qp_per_entry) +{ + int i = MLX4_MIN_MGM_LOG_ENTRY_SIZE; + + for (i = MLX4_MIN_MGM_LOG_ENTRY_SIZE; i <= MLX4_MAX_MGM_LOG_ENTRY_SIZE; + i++) { + if (qp_per_entry <= 4 * ((1 << i) / 16 - 2)) + break; + } + + return (i <= MLX4_MAX_MGM_LOG_ENTRY_SIZE) ? i : -1; +} + +static const char *dmfs_high_rate_steering_mode_str(int dmfs_high_steer_mode) +{ + switch (dmfs_high_steer_mode) { + case MLX4_STEERING_DMFS_A0_DEFAULT: + return "default performance"; + + case MLX4_STEERING_DMFS_A0_DYNAMIC: + return "dynamic hybrid mode"; + + case MLX4_STEERING_DMFS_A0_STATIC: + return "performance optimized for limited rule configuration (static)"; + + case MLX4_STEERING_DMFS_A0_DISABLE: + return "disabled performance optimized steering"; + + case MLX4_STEERING_DMFS_A0_NOT_SUPPORTED: + return "performance optimized steering not supported"; + + default: + return "Unrecognized mode"; + } +} + +#define MLX4_DMFS_A0_STEERING (1UL << 2) + +static void choose_steering_mode(struct mlx4_dev *dev, + struct mlx4_dev_cap *dev_cap) +{ + if (mlx4_log_num_mgm_entry_size <= 0) { + if ((-mlx4_log_num_mgm_entry_size) & MLX4_DMFS_A0_STEERING) { + if (dev->caps.dmfs_high_steer_mode == + MLX4_STEERING_DMFS_A0_NOT_SUPPORTED) + mlx4_err(dev, "DMFS high rate mode not supported\n"); + else + dev->caps.dmfs_high_steer_mode = + MLX4_STEERING_DMFS_A0_STATIC; + } + } + + if (mlx4_log_num_mgm_entry_size <= 0 && + dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_FS_EN && + (!mlx4_is_mfunc(dev) || + (dev_cap->fs_max_num_qp_per_entry >= + (dev->persist->num_vfs + 1))) && + choose_log_fs_mgm_entry_size(dev_cap->fs_max_num_qp_per_entry) >= + MLX4_MIN_MGM_LOG_ENTRY_SIZE) { + dev->oper_log_mgm_entry_size = + choose_log_fs_mgm_entry_size(dev_cap->fs_max_num_qp_per_entry); + dev->caps.steering_mode = MLX4_STEERING_MODE_DEVICE_MANAGED; + dev->caps.num_qp_per_mgm = dev_cap->fs_max_num_qp_per_entry; + dev->caps.fs_log_max_ucast_qp_range_size = + dev_cap->fs_log_max_ucast_qp_range_size; + } else { + if (dev->caps.dmfs_high_steer_mode != + MLX4_STEERING_DMFS_A0_NOT_SUPPORTED) + dev->caps.dmfs_high_steer_mode = MLX4_STEERING_DMFS_A0_DISABLE; + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_UC_STEER && + dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) + dev->caps.steering_mode = MLX4_STEERING_MODE_B0; + else { + dev->caps.steering_mode = MLX4_STEERING_MODE_A0; + + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_UC_STEER || + dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) + mlx4_warn(dev, "Must have both UC_STEER and MC_STEER flags set to use B0 steering - falling back to A0 steering mode\n"); + } + dev->oper_log_mgm_entry_size = + mlx4_log_num_mgm_entry_size > 0 ? + mlx4_log_num_mgm_entry_size : + MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE; + dev->caps.num_qp_per_mgm = mlx4_get_qp_per_mgm(dev); + } + mlx4_dbg(dev, "Steering mode is: %s, oper_log_mgm_entry_size = %d, modparam log_num_mgm_entry_size = %d\n", + mlx4_steering_mode_str(dev->caps.steering_mode), + dev->oper_log_mgm_entry_size, + mlx4_log_num_mgm_entry_size); +} + +static void choose_tunnel_offload_mode(struct mlx4_dev *dev, + struct mlx4_dev_cap *dev_cap) +{ + if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED && + dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_VXLAN_OFFLOADS) + dev->caps.tunnel_offload_mode = MLX4_TUNNEL_OFFLOAD_MODE_VXLAN; + else + dev->caps.tunnel_offload_mode = MLX4_TUNNEL_OFFLOAD_MODE_NONE; + + mlx4_dbg(dev, "Tunneling offload mode is: %s\n", (dev->caps.tunnel_offload_mode + == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) ? "vxlan" : "none"); +} + +static int mlx4_validate_optimized_steering(struct mlx4_dev *dev) +{ + int i; + struct mlx4_port_cap port_cap; + + if (dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_NOT_SUPPORTED) + return -EINVAL; + + for (i = 1; i <= dev->caps.num_ports; i++) { + if (mlx4_dev_port(dev, i, &port_cap)) { + mlx4_err(dev, + "QUERY_DEV_CAP command failed, can't veify DMFS high rate steering.\n"); + } else if ((dev->caps.dmfs_high_steer_mode != + MLX4_STEERING_DMFS_A0_DEFAULT) && + (port_cap.dmfs_optimized_state == + !!(dev->caps.dmfs_high_steer_mode == + MLX4_STEERING_DMFS_A0_DISABLE))) { + mlx4_err(dev, + "DMFS high rate steer mode differ, driver requested %s but %s in FW.\n", + dmfs_high_rate_steering_mode_str( + dev->caps.dmfs_high_steer_mode), + (port_cap.dmfs_optimized_state ? + "enabled" : "disabled")); + } + } + + return 0; +} + +static int mlx4_init_fw(struct mlx4_dev *dev) +{ + struct mlx4_mod_stat_cfg mlx4_cfg; + int err = 0; + + if (!mlx4_is_slave(dev)) { + err = mlx4_QUERY_FW(dev); + if (err) { + if (err == -EACCES) + mlx4_info(dev, "non-primary physical function, skipping\n"); + else + mlx4_err(dev, "QUERY_FW command failed, aborting\n"); + return err; + } + + err = mlx4_load_fw(dev); + if (err) { + mlx4_err(dev, "Failed to start FW, aborting\n"); + return err; + } + + mlx4_cfg.log_pg_sz_m = 1; + mlx4_cfg.log_pg_sz = 0; + err = mlx4_MOD_STAT_CFG(dev, &mlx4_cfg); + if (err) + mlx4_warn(dev, "Failed to override log_pg_sz parameter\n"); + } + + return err; +} + +static int mlx4_init_hca(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_adapter adapter; + struct mlx4_dev_cap dev_cap; + struct mlx4_profile profile; + struct mlx4_init_hca_param init_hca; + u64 icm_size; + struct mlx4_config_dev_params params; + int err; + + if (!mlx4_is_slave(dev)) { + err = mlx4_dev_cap(dev, &dev_cap); + if (err) { + mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n"); + return err; + } + + choose_steering_mode(dev, &dev_cap); + choose_tunnel_offload_mode(dev, &dev_cap); + + if (dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_STATIC && + mlx4_is_master(dev)) + dev->caps.function_caps |= MLX4_FUNC_CAP_DMFS_A0_STATIC; + + err = mlx4_get_phys_port_id(dev); + if (err) + mlx4_err(dev, "Fail to get physical port id\n"); + + if (mlx4_is_master(dev)) + mlx4_parav_master_pf_caps(dev); + + if (mlx4_low_memory_profile()) { + mlx4_info(dev, "Running from within kdump kernel. Using low memory profile\n"); + profile = low_mem_profile; + } else { + profile = default_profile; + } + if (dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) + profile.num_mcg = MLX4_FS_NUM_MCG; + + icm_size = mlx4_make_profile(dev, &profile, &dev_cap, + &init_hca); + if ((long long) icm_size < 0) { + err = icm_size; + return err; + } + + dev->caps.max_fmr_maps = (1 << (32 - ilog2(dev->caps.num_mpts))) - 1; + + if (enable_4k_uar) { + init_hca.log_uar_sz = ilog2(dev->caps.num_uars) + + PAGE_SHIFT - DEFAULT_UAR_PAGE_SHIFT; + init_hca.uar_page_sz = DEFAULT_UAR_PAGE_SHIFT - 12; + } else { + init_hca.log_uar_sz = ilog2(dev->caps.num_uars); + init_hca.uar_page_sz = PAGE_SHIFT - 12; + } + + init_hca.mw_enabled = 0; + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW || + dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN) + init_hca.mw_enabled = INIT_HCA_TPT_MW_ENABLE; + + err = mlx4_init_icm(dev, &dev_cap, &init_hca, icm_size); + if (err) + return err; + + err = mlx4_INIT_HCA(dev, &init_hca); + if (err) { + mlx4_err(dev, "INIT_HCA command failed, aborting\n"); + goto err_free_icm; + } + + if (dev_cap.flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) { + err = mlx4_query_func(dev, &dev_cap); + if (err < 0) { + mlx4_err(dev, "QUERY_FUNC command failed, aborting.\n"); + goto err_close; + } else if (err & MLX4_QUERY_FUNC_NUM_SYS_EQS) { + dev->caps.num_eqs = dev_cap.max_eqs; + dev->caps.reserved_eqs = dev_cap.reserved_eqs; + dev->caps.reserved_uars = dev_cap.reserved_uars; + } + } + + /* + * If TS is supported by FW + * read HCA frequency by QUERY_HCA command + */ + if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) { + memset(&init_hca, 0, sizeof(init_hca)); + err = mlx4_QUERY_HCA(dev, &init_hca); + if (err) { + mlx4_err(dev, "QUERY_HCA command failed, disable timestamp\n"); + dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS; + } else { + dev->caps.hca_core_clock = + init_hca.hca_core_clock; + } + + /* In case we got HCA frequency 0 - disable timestamping + * to avoid dividing by zero + */ + if (!dev->caps.hca_core_clock) { + dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS; + mlx4_err(dev, + "HCA frequency is 0 - timestamping is not supported\n"); + } else if (map_internal_clock(dev)) { + /* + * Map internal clock, + * in case of failure disable timestamping + */ + dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS; + mlx4_err(dev, "Failed to map internal clock. Timestamping is not supported\n"); + } + } + + if (dev->caps.dmfs_high_steer_mode != + MLX4_STEERING_DMFS_A0_NOT_SUPPORTED) { + if (mlx4_validate_optimized_steering(dev)) + mlx4_warn(dev, "Optimized steering validation failed\n"); + + if (dev->caps.dmfs_high_steer_mode == + MLX4_STEERING_DMFS_A0_DISABLE) { + dev->caps.dmfs_high_rate_qpn_base = + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW]; + dev->caps.dmfs_high_rate_qpn_range = + MLX4_A0_STEERING_TABLE_SIZE; + } + + mlx4_dbg(dev, "DMFS high rate steer mode is: %s\n", + dmfs_high_rate_steering_mode_str( + dev->caps.dmfs_high_steer_mode)); + } + } else { + err = mlx4_init_slave(dev); + if (err) { + if (err != -EAGAIN) + mlx4_err(dev, "Failed to initialize slave\n"); + return err; + } + + err = mlx4_slave_cap(dev); + if (err) { + mlx4_err(dev, "Failed to obtain slave caps\n"); + goto err_close; + } + } + + if (map_bf_area(dev)) + mlx4_dbg(dev, "Failed to map blue flame area\n"); + + /*Only the master set the ports, all the rest got it from it.*/ + if (!mlx4_is_slave(dev)) + mlx4_set_port_mask(dev); + + err = mlx4_QUERY_ADAPTER(dev, &adapter); + if (err) { + mlx4_err(dev, "QUERY_ADAPTER command failed, aborting\n"); + goto unmap_bf; + } + + /* Query CONFIG_DEV parameters */ + err = mlx4_config_dev_retrieval(dev, ¶ms); + if (err && err != -ENOTSUPP) { + mlx4_err(dev, "Failed to query CONFIG_DEV parameters\n"); + } else if (!err) { + dev->caps.rx_checksum_flags_port[1] = params.rx_csum_flags_port_1; + dev->caps.rx_checksum_flags_port[2] = params.rx_csum_flags_port_2; + } + priv->eq_table.inta_pin = adapter.inta_pin; + memcpy(dev->board_id, adapter.board_id, sizeof dev->board_id); + + return 0; + +unmap_bf: + unmap_internal_clock(dev); + unmap_bf_area(dev); + + if (mlx4_is_slave(dev)) { + kfree(dev->caps.qp0_qkey); + kfree(dev->caps.qp0_tunnel); + kfree(dev->caps.qp0_proxy); + kfree(dev->caps.qp1_tunnel); + kfree(dev->caps.qp1_proxy); + } + +err_close: + if (mlx4_is_slave(dev)) + mlx4_slave_exit(dev); + else + mlx4_CLOSE_HCA(dev, 0); + +err_free_icm: + if (!mlx4_is_slave(dev)) + mlx4_free_icms(dev); + + return err; +} + +static int mlx4_init_counters_table(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int nent_pow2; + + if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS)) + return -ENOENT; + + if (!dev->caps.max_counters) + return -ENOSPC; + + nent_pow2 = roundup_pow_of_two(dev->caps.max_counters); + /* reserve last counter index for sink counter */ + return mlx4_bitmap_init(&priv->counters_bitmap, nent_pow2, + nent_pow2 - 1, 0, + nent_pow2 - dev->caps.max_counters + 1); +} + +static void mlx4_cleanup_counters_table(struct mlx4_dev *dev) +{ + if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS)) + return; + + if (!dev->caps.max_counters) + return; + + mlx4_bitmap_cleanup(&mlx4_priv(dev)->counters_bitmap); +} + +static void mlx4_cleanup_default_counters(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int port; + + for (port = 0; port < dev->caps.num_ports; port++) + if (priv->def_counter[port] != -1) + mlx4_counter_free(dev, priv->def_counter[port]); +} + +static int mlx4_allocate_default_counters(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int port, err = 0; + u32 idx; + + for (port = 0; port < dev->caps.num_ports; port++) + priv->def_counter[port] = -1; + + for (port = 0; port < dev->caps.num_ports; port++) { + err = mlx4_counter_alloc(dev, &idx); + + if (!err || err == -ENOSPC) { + priv->def_counter[port] = idx; + } else if (err == -ENOENT) { + err = 0; + continue; + } else if (mlx4_is_slave(dev) && err == -EINVAL) { + priv->def_counter[port] = MLX4_SINK_COUNTER_INDEX(dev); + mlx4_warn(dev, "can't allocate counter from old PF driver, using index %d\n", + MLX4_SINK_COUNTER_INDEX(dev)); + err = 0; + } else { + mlx4_err(dev, "%s: failed to allocate default counter port %d err %d\n", + __func__, port + 1, err); + mlx4_cleanup_default_counters(dev); + return err; + } + + mlx4_dbg(dev, "%s: default counter index %d for port %d\n", + __func__, priv->def_counter[port], port + 1); + } + + return err; +} + +int __mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS)) + return -ENOENT; + + *idx = mlx4_bitmap_alloc(&priv->counters_bitmap); + if (*idx == -1) { + *idx = MLX4_SINK_COUNTER_INDEX(dev); + return -ENOSPC; + } + + return 0; +} + +int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx) +{ + u64 out_param; + int err; + + if (mlx4_is_mfunc(dev)) { + err = mlx4_cmd_imm(dev, 0, &out_param, RES_COUNTER, + RES_OP_RESERVE, MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (!err) + *idx = get_param_l(&out_param); + + return err; + } + return __mlx4_counter_alloc(dev, idx); +} +EXPORT_SYMBOL_GPL(mlx4_counter_alloc); + +static int __mlx4_clear_if_stat(struct mlx4_dev *dev, + u8 counter_index) +{ + struct mlx4_cmd_mailbox *if_stat_mailbox; + int err; + u32 if_stat_in_mod = (counter_index & 0xff) | MLX4_QUERY_IF_STAT_RESET; + + if_stat_mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(if_stat_mailbox)) + return PTR_ERR(if_stat_mailbox); + + err = mlx4_cmd_box(dev, 0, if_stat_mailbox->dma, if_stat_in_mod, 0, + MLX4_CMD_QUERY_IF_STAT, MLX4_CMD_TIME_CLASS_C, + MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, if_stat_mailbox); + return err; +} + +void __mlx4_counter_free(struct mlx4_dev *dev, u32 idx) +{ + if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS)) + return; + + if (idx == MLX4_SINK_COUNTER_INDEX(dev)) + return; + + __mlx4_clear_if_stat(dev, idx); + + mlx4_bitmap_free(&mlx4_priv(dev)->counters_bitmap, idx, MLX4_USE_RR); + return; +} + +void mlx4_counter_free(struct mlx4_dev *dev, u32 idx) +{ + u64 in_param = 0; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, idx); + mlx4_cmd(dev, in_param, RES_COUNTER, RES_OP_RESERVE, + MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); + return; + } + __mlx4_counter_free(dev, idx); +} +EXPORT_SYMBOL_GPL(mlx4_counter_free); + +int mlx4_get_default_counter_index(struct mlx4_dev *dev, int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + return priv->def_counter[port - 1]; +} +EXPORT_SYMBOL_GPL(mlx4_get_default_counter_index); + +void mlx4_set_admin_guid(struct mlx4_dev *dev, __be64 guid, int entry, int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + priv->mfunc.master.vf_admin[entry].vport[port].guid = guid; +} +EXPORT_SYMBOL_GPL(mlx4_set_admin_guid); + +__be64 mlx4_get_admin_guid(struct mlx4_dev *dev, int entry, int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + return priv->mfunc.master.vf_admin[entry].vport[port].guid; +} +EXPORT_SYMBOL_GPL(mlx4_get_admin_guid); + +void mlx4_set_random_admin_guid(struct mlx4_dev *dev, int entry, int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + __be64 guid; + + /* hw GUID */ + if (entry == 0) + return; + + get_random_bytes((char *)&guid, sizeof(guid)); + guid &= ~(cpu_to_be64(1ULL << 56)); + guid |= cpu_to_be64(1ULL << 57); + priv->mfunc.master.vf_admin[entry].vport[port].guid = guid; +} + +static int mlx4_setup_hca(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int err; + int port; + __be32 ib_port_default_caps; + + err = mlx4_init_uar_table(dev); + if (err) { + mlx4_err(dev, "Failed to initialize user access region table, aborting\n"); + return err; + } + + err = mlx4_uar_alloc(dev, &priv->driver_uar); + if (err) { + mlx4_err(dev, "Failed to allocate driver access region, aborting\n"); + goto err_uar_table_free; + } + + priv->kar = ioremap((phys_addr_t) priv->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE); + if (!priv->kar) { + mlx4_err(dev, "Couldn't map kernel access region, aborting\n"); + err = -ENOMEM; + goto err_uar_free; + } + + err = mlx4_init_pd_table(dev); + if (err) { + mlx4_err(dev, "Failed to initialize protection domain table, aborting\n"); + goto err_kar_unmap; + } + + err = mlx4_init_xrcd_table(dev); + if (err) { + mlx4_err(dev, "Failed to initialize reliable connection domain table, aborting\n"); + goto err_pd_table_free; + } + + err = mlx4_init_mr_table(dev); + if (err) { + mlx4_err(dev, "Failed to initialize memory region table, aborting\n"); + goto err_xrcd_table_free; + } + + if (!mlx4_is_slave(dev)) { + err = mlx4_init_mcg_table(dev); + if (err) { + mlx4_err(dev, "Failed to initialize multicast group table, aborting\n"); + goto err_mr_table_free; + } + err = mlx4_config_mad_demux(dev); + if (err) { + mlx4_err(dev, "Failed in config_mad_demux, aborting\n"); + goto err_mcg_table_free; + } + } + + err = mlx4_init_eq_table(dev); + if (err) { + mlx4_err(dev, "Failed to initialize event queue table, aborting\n"); + goto err_mcg_table_free; + } + + err = mlx4_cmd_use_events(dev); + if (err) { + mlx4_err(dev, "Failed to switch to event-driven firmware commands, aborting\n"); + goto err_eq_table_free; + } + + err = mlx4_NOP(dev); + if (err) { + if (dev->flags & MLX4_FLAG_MSI_X) { + mlx4_warn(dev, "NOP command failed to generate MSI-X interrupt IRQ %d)\n", + priv->eq_table.eq[MLX4_EQ_ASYNC].irq); + mlx4_warn(dev, "Trying again without MSI-X\n"); + } else { + mlx4_err(dev, "NOP command failed to generate interrupt (IRQ %d), aborting\n", + priv->eq_table.eq[MLX4_EQ_ASYNC].irq); + mlx4_err(dev, "BIOS or ACPI interrupt routing problem?\n"); + } + + goto err_cmd_poll; + } + + mlx4_dbg(dev, "NOP command IRQ test passed\n"); + + err = mlx4_init_cq_table(dev); + if (err) { + mlx4_err(dev, "Failed to initialize completion queue table, aborting\n"); + goto err_cmd_poll; + } + + err = mlx4_init_srq_table(dev); + if (err) { + mlx4_err(dev, "Failed to initialize shared receive queue table, aborting\n"); + goto err_cq_table_free; + } + + err = mlx4_init_qp_table(dev); + if (err) { + mlx4_err(dev, "Failed to initialize queue pair table, aborting\n"); + goto err_srq_table_free; + } + + if (!mlx4_is_slave(dev)) { + err = mlx4_init_counters_table(dev); + if (err && err != -ENOENT) { + mlx4_err(dev, "Failed to initialize counters table, aborting\n"); + goto err_qp_table_free; + } + } + + err = mlx4_allocate_default_counters(dev); + if (err) { + mlx4_err(dev, "Failed to allocate default counters, aborting\n"); + goto err_counters_table_free; + } + + if (!mlx4_is_slave(dev)) { + for (port = 1; port <= dev->caps.num_ports; port++) { + ib_port_default_caps = 0; + err = mlx4_get_port_ib_caps(dev, port, + &ib_port_default_caps); + if (err) + mlx4_warn(dev, "failed to get port %d default ib capabilities (%d). Continuing with caps = 0\n", + port, err); + dev->caps.ib_port_def_cap[port] = ib_port_default_caps; + + /* initialize per-slave default ib port capabilities */ + if (mlx4_is_master(dev)) { + int i; + for (i = 0; i < dev->num_slaves; i++) { + if (i == mlx4_master_func_num(dev)) + continue; + priv->mfunc.master.slave_state[i].ib_cap_mask[port] = + ib_port_default_caps; + } + } + + if (mlx4_is_mfunc(dev)) + dev->caps.port_ib_mtu[port] = IB_MTU_2048; + else + dev->caps.port_ib_mtu[port] = IB_MTU_4096; + + err = mlx4_SET_PORT(dev, port, mlx4_is_master(dev) ? + dev->caps.pkey_table_len[port] : -1); + if (err) { + mlx4_err(dev, "Failed to set port %d, aborting\n", + port); + goto err_default_countes_free; + } + } + } + + return 0; + +err_default_countes_free: + mlx4_cleanup_default_counters(dev); + +err_counters_table_free: + if (!mlx4_is_slave(dev)) + mlx4_cleanup_counters_table(dev); + +err_qp_table_free: + mlx4_cleanup_qp_table(dev); + +err_srq_table_free: + mlx4_cleanup_srq_table(dev); + +err_cq_table_free: + mlx4_cleanup_cq_table(dev); + +err_cmd_poll: + mlx4_cmd_use_polling(dev); + +err_eq_table_free: + mlx4_cleanup_eq_table(dev); + +err_mcg_table_free: + if (!mlx4_is_slave(dev)) + mlx4_cleanup_mcg_table(dev); + +err_mr_table_free: + mlx4_cleanup_mr_table(dev); + +err_xrcd_table_free: + mlx4_cleanup_xrcd_table(dev); + +err_pd_table_free: + mlx4_cleanup_pd_table(dev); + +err_kar_unmap: + iounmap(priv->kar); + +err_uar_free: + mlx4_uar_free(dev, &priv->driver_uar); + +err_uar_table_free: + mlx4_cleanup_uar_table(dev); + return err; +} + +static int mlx4_init_affinity_hint(struct mlx4_dev *dev, int port, int eqn) +{ + int requested_cpu = 0; + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_eq *eq; + int off = 0; + int i; + + if (eqn > dev->caps.num_comp_vectors) + return -EINVAL; + + for (i = 1; i < port; i++) + off += mlx4_get_eqs_per_port(dev, i); + + requested_cpu = eqn - off - !!(eqn > MLX4_EQ_ASYNC); + + /* Meaning EQs are shared, and this call comes from the second port */ + if (requested_cpu < 0) + return 0; + + eq = &priv->eq_table.eq[eqn]; + + eq->affinity_cpu_id = requested_cpu % num_online_cpus(); + + return 0; +} + +static void mlx4_enable_msi_x(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct msix_entry *entries; + int i; + int port = 0; + + if (msi_x) { + int nreq = dev->caps.num_ports * num_online_cpus() + 1; + + nreq = min_t(int, dev->caps.num_eqs - dev->caps.reserved_eqs, + nreq); + if (nreq > MAX_MSIX) + nreq = MAX_MSIX; + + entries = kcalloc(nreq, sizeof *entries, GFP_KERNEL); + if (!entries) + goto no_msi; + + for (i = 0; i < nreq; ++i) + entries[i].entry = i; + + nreq = pci_enable_msix_range(dev->persist->pdev, entries, 2, + nreq); + + if (nreq < 0 || nreq < MLX4_EQ_ASYNC) { + kfree(entries); + goto no_msi; + } + /* 1 is reserved for events (asyncrounous EQ) */ + dev->caps.num_comp_vectors = nreq - 1; + + priv->eq_table.eq[MLX4_EQ_ASYNC].irq = entries[0].vector; + bitmap_zero(priv->eq_table.eq[MLX4_EQ_ASYNC].actv_ports.ports, + dev->caps.num_ports); + + for (i = 0; i < dev->caps.num_comp_vectors + 1; i++) { + if (i == MLX4_EQ_ASYNC) + continue; + + priv->eq_table.eq[i].irq = + entries[i + 1 - !!(i > MLX4_EQ_ASYNC)].vector; + + if (MLX4_IS_LEGACY_EQ_MODE(dev->caps)) { + bitmap_fill(priv->eq_table.eq[i].actv_ports.ports, + dev->caps.num_ports); + /* We don't set affinity hint when there + * aren't enough EQs + */ + } else { + set_bit(port, + priv->eq_table.eq[i].actv_ports.ports); + if (mlx4_init_affinity_hint(dev, port + 1, i)) + mlx4_warn(dev, "Couldn't init hint cpumask for EQ %d\n", + i); + } + /* We divide the Eqs evenly between the two ports. + * (dev->caps.num_comp_vectors / dev->caps.num_ports) + * refers to the number of Eqs per port + * (i.e eqs_per_port). Theoretically, we would like to + * write something like (i + 1) % eqs_per_port == 0. + * However, since there's an asynchronous Eq, we have + * to skip over it by comparing this condition to + * !!((i + 1) > MLX4_EQ_ASYNC). + */ + if ((dev->caps.num_comp_vectors > dev->caps.num_ports) && + ((i + 1) % + (dev->caps.num_comp_vectors / dev->caps.num_ports)) == + !!((i + 1) > MLX4_EQ_ASYNC)) + /* If dev->caps.num_comp_vectors < dev->caps.num_ports, + * everything is shared anyway. + */ + port++; + } + + dev->flags |= MLX4_FLAG_MSI_X; + + kfree(entries); + return; + } + +no_msi: + dev->caps.num_comp_vectors = 1; + + BUG_ON(MLX4_EQ_ASYNC >= 2); + for (i = 0; i < 2; ++i) { + priv->eq_table.eq[i].irq = dev->persist->pdev->irq; + if (i != MLX4_EQ_ASYNC) { + bitmap_fill(priv->eq_table.eq[i].actv_ports.ports, + dev->caps.num_ports); + } + } +} + +static int mlx4_init_port_info(struct mlx4_dev *dev, int port) +{ + struct mlx4_port_info *info = &mlx4_priv(dev)->port[port]; + int err = 0; + + info->dev = dev; + info->port = port; + if (!mlx4_is_slave(dev)) { + mlx4_init_mac_table(dev, &info->mac_table); + mlx4_init_vlan_table(dev, &info->vlan_table); + mlx4_init_roce_gid_table(dev, &info->gid_table); + info->base_qpn = mlx4_get_base_qpn(dev, port); + } + + sprintf(info->dev_name, "mlx4_port%d", port); + info->port_attr.attr.name = info->dev_name; + if (mlx4_is_mfunc(dev)) + info->port_attr.attr.mode = S_IRUGO; + else { + info->port_attr.attr.mode = S_IRUGO | S_IWUSR; + info->port_attr.store = set_port_type; + } + info->port_attr.show = show_port_type; + sysfs_attr_init(&info->port_attr.attr); + + err = device_create_file(&dev->persist->pdev->dev, &info->port_attr); + if (err) { + mlx4_err(dev, "Failed to create file for port %d\n", port); + info->port = -1; + } + + sprintf(info->dev_mtu_name, "mlx4_port%d_mtu", port); + info->port_mtu_attr.attr.name = info->dev_mtu_name; + if (mlx4_is_mfunc(dev)) + info->port_mtu_attr.attr.mode = S_IRUGO; + else { + info->port_mtu_attr.attr.mode = S_IRUGO | S_IWUSR; + info->port_mtu_attr.store = set_port_ib_mtu; + } + info->port_mtu_attr.show = show_port_ib_mtu; + sysfs_attr_init(&info->port_mtu_attr.attr); + + err = device_create_file(&dev->persist->pdev->dev, + &info->port_mtu_attr); + if (err) { + mlx4_err(dev, "Failed to create mtu file for port %d\n", port); + device_remove_file(&info->dev->persist->pdev->dev, + &info->port_attr); + info->port = -1; + } + + return err; +} + +static void mlx4_cleanup_port_info(struct mlx4_port_info *info) +{ + if (info->port < 0) + return; + + device_remove_file(&info->dev->persist->pdev->dev, &info->port_attr); + device_remove_file(&info->dev->persist->pdev->dev, + &info->port_mtu_attr); +#ifdef CONFIG_RFS_ACCEL + free_irq_cpu_rmap(info->rmap); + info->rmap = NULL; +#endif +} + +static int mlx4_init_steering(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int num_entries = dev->caps.num_ports; + int i, j; + + priv->steer = kzalloc(sizeof(struct mlx4_steer) * num_entries, GFP_KERNEL); + if (!priv->steer) + return -ENOMEM; + + for (i = 0; i < num_entries; i++) + for (j = 0; j < MLX4_NUM_STEERS; j++) { + INIT_LIST_HEAD(&priv->steer[i].promisc_qps[j]); + INIT_LIST_HEAD(&priv->steer[i].steer_entries[j]); + } + return 0; +} + +static void mlx4_clear_steering(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_steer_index *entry, *tmp_entry; + struct mlx4_promisc_qp *pqp, *tmp_pqp; + int num_entries = dev->caps.num_ports; + int i, j; + + for (i = 0; i < num_entries; i++) { + for (j = 0; j < MLX4_NUM_STEERS; j++) { + list_for_each_entry_safe(pqp, tmp_pqp, + &priv->steer[i].promisc_qps[j], + list) { + list_del(&pqp->list); + kfree(pqp); + } + list_for_each_entry_safe(entry, tmp_entry, + &priv->steer[i].steer_entries[j], + list) { + list_del(&entry->list); + list_for_each_entry_safe(pqp, tmp_pqp, + &entry->duplicates, + list) { + list_del(&pqp->list); + kfree(pqp); + } + kfree(entry); + } + } + } + kfree(priv->steer); +} + +static int extended_func_num(struct pci_dev *pdev) +{ + return PCI_SLOT(pdev->devfn) * 8 + PCI_FUNC(pdev->devfn); +} + +#define MLX4_OWNER_BASE 0x8069c +#define MLX4_OWNER_SIZE 4 + +static int mlx4_get_ownership(struct mlx4_dev *dev) +{ + void __iomem *owner; + u32 ret; + + if (pci_channel_offline(dev->persist->pdev)) + return -EIO; + + owner = ioremap(pci_resource_start(dev->persist->pdev, 0) + + MLX4_OWNER_BASE, + MLX4_OWNER_SIZE); + if (!owner) { + mlx4_err(dev, "Failed to obtain ownership bit\n"); + return -ENOMEM; + } + + ret = readl(owner); + iounmap(owner); + return (int) !!ret; +} + +static void mlx4_free_ownership(struct mlx4_dev *dev) +{ + void __iomem *owner; + + if (pci_channel_offline(dev->persist->pdev)) + return; + + owner = ioremap(pci_resource_start(dev->persist->pdev, 0) + + MLX4_OWNER_BASE, + MLX4_OWNER_SIZE); + if (!owner) { + mlx4_err(dev, "Failed to obtain ownership bit\n"); + return; + } + writel(0, owner); + msleep(1000); + iounmap(owner); +} + +#define SRIOV_VALID_STATE(flags) (!!((flags) & MLX4_FLAG_SRIOV) ==\ + !!((flags) & MLX4_FLAG_MASTER)) + +static u64 mlx4_enable_sriov(struct mlx4_dev *dev, struct pci_dev *pdev, + u8 total_vfs, int existing_vfs, int reset_flow) +{ + u64 dev_flags = dev->flags; + int err = 0; + + if (reset_flow) { + dev->dev_vfs = kcalloc(total_vfs, sizeof(*dev->dev_vfs), + GFP_KERNEL); + if (!dev->dev_vfs) + goto free_mem; + return dev_flags; + } + + atomic_inc(&pf_loading); + if (dev->flags & MLX4_FLAG_SRIOV) { + if (existing_vfs != total_vfs) { + mlx4_err(dev, "SR-IOV was already enabled, but with num_vfs (%d) different than requested (%d)\n", + existing_vfs, total_vfs); + total_vfs = existing_vfs; + } + } + + dev->dev_vfs = kzalloc(total_vfs * sizeof(*dev->dev_vfs), GFP_KERNEL); + if (NULL == dev->dev_vfs) { + mlx4_err(dev, "Failed to allocate memory for VFs\n"); + goto disable_sriov; + } + + if (!(dev->flags & MLX4_FLAG_SRIOV)) { + mlx4_warn(dev, "Enabling SR-IOV with %d VFs\n", total_vfs); + err = pci_enable_sriov(pdev, total_vfs); + } + if (err) { + mlx4_err(dev, "Failed to enable SR-IOV, continuing without SR-IOV (err = %d)\n", + err); + goto disable_sriov; + } else { + mlx4_warn(dev, "Running in master mode\n"); + dev_flags |= MLX4_FLAG_SRIOV | + MLX4_FLAG_MASTER; + dev_flags &= ~MLX4_FLAG_SLAVE; + dev->persist->num_vfs = total_vfs; + } + return dev_flags; + +disable_sriov: + atomic_dec(&pf_loading); +free_mem: + dev->persist->num_vfs = 0; + kfree(dev->dev_vfs); + dev->dev_vfs = NULL; + return dev_flags & ~MLX4_FLAG_MASTER; +} + +enum { + MLX4_DEV_CAP_CHECK_NUM_VFS_ABOVE_64 = -1, +}; + +static int mlx4_check_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap, + int *nvfs) +{ + int requested_vfs = nvfs[0] + nvfs[1] + nvfs[2]; + /* Checking for 64 VFs as a limitation of CX2 */ + if (!(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_80_VFS) && + requested_vfs >= 64) { + mlx4_err(dev, "Requested %d VFs, but FW does not support more than 64\n", + requested_vfs); + return MLX4_DEV_CAP_CHECK_NUM_VFS_ABOVE_64; + } + return 0; +} + +static int mlx4_pci_enable_device(struct mlx4_dev *dev) +{ + struct pci_dev *pdev = dev->persist->pdev; + int err = 0; + + mutex_lock(&dev->persist->pci_status_mutex); + if (dev->persist->pci_status == MLX4_PCI_STATUS_DISABLED) { + err = pci_enable_device(pdev); + if (!err) + dev->persist->pci_status = MLX4_PCI_STATUS_ENABLED; + } + mutex_unlock(&dev->persist->pci_status_mutex); + + return err; +} + +static void mlx4_pci_disable_device(struct mlx4_dev *dev) +{ + struct pci_dev *pdev = dev->persist->pdev; + + mutex_lock(&dev->persist->pci_status_mutex); + if (dev->persist->pci_status == MLX4_PCI_STATUS_ENABLED) { + pci_disable_device(pdev); + dev->persist->pci_status = MLX4_PCI_STATUS_DISABLED; + } + mutex_unlock(&dev->persist->pci_status_mutex); +} + +static int mlx4_load_one(struct pci_dev *pdev, int pci_dev_data, + int total_vfs, int *nvfs, struct mlx4_priv *priv, + int reset_flow) +{ + struct mlx4_dev *dev; + unsigned sum = 0; + int err; + int port; + int i; + struct mlx4_dev_cap *dev_cap = NULL; + int existing_vfs = 0; + + dev = &priv->dev; + + INIT_LIST_HEAD(&priv->ctx_list); + spin_lock_init(&priv->ctx_lock); + + mutex_init(&priv->port_mutex); + mutex_init(&priv->bond_mutex); + + INIT_LIST_HEAD(&priv->pgdir_list); + mutex_init(&priv->pgdir_mutex); + spin_lock_init(&priv->cmd.context_lock); + + INIT_LIST_HEAD(&priv->bf_list); + mutex_init(&priv->bf_mutex); + + dev->rev_id = pdev->revision; + dev->numa_node = dev_to_node(&pdev->dev); + + /* Detect if this device is a virtual function */ + if (pci_dev_data & MLX4_PCI_DEV_IS_VF) { + mlx4_warn(dev, "Detected virtual function - running in slave mode\n"); + dev->flags |= MLX4_FLAG_SLAVE; + } else { + /* We reset the device and enable SRIOV only for physical + * devices. Try to claim ownership on the device; + * if already taken, skip -- do not allow multiple PFs */ + err = mlx4_get_ownership(dev); + if (err) { + if (err < 0) + return err; + else { + mlx4_warn(dev, "Multiple PFs not yet supported - Skipping PF\n"); + return -EINVAL; + } + } + + atomic_set(&priv->opreq_count, 0); + INIT_WORK(&priv->opreq_task, mlx4_opreq_action); + + /* + * Now reset the HCA before we touch the PCI capabilities or + * attempt a firmware command, since a boot ROM may have left + * the HCA in an undefined state. + */ + err = mlx4_reset(dev); + if (err) { + mlx4_err(dev, "Failed to reset HCA, aborting\n"); + goto err_sriov; + } + + if (total_vfs) { + dev->flags = MLX4_FLAG_MASTER; + existing_vfs = pci_num_vf(pdev); + if (existing_vfs) + dev->flags |= MLX4_FLAG_SRIOV; + dev->persist->num_vfs = total_vfs; + } + } + + /* on load remove any previous indication of internal error, + * device is up. + */ + dev->persist->state = MLX4_DEVICE_STATE_UP; + +slave_start: + err = mlx4_cmd_init(dev); + if (err) { + mlx4_err(dev, "Failed to init command interface, aborting\n"); + goto err_sriov; + } + + /* In slave functions, the communication channel must be initialized + * before posting commands. Also, init num_slaves before calling + * mlx4_init_hca */ + if (mlx4_is_mfunc(dev)) { + if (mlx4_is_master(dev)) { + dev->num_slaves = MLX4_MAX_NUM_SLAVES; + + } else { + dev->num_slaves = 0; + err = mlx4_multi_func_init(dev); + if (err) { + mlx4_err(dev, "Failed to init slave mfunc interface, aborting\n"); + goto err_cmd; + } + } + } + + err = mlx4_init_fw(dev); + if (err) { + mlx4_err(dev, "Failed to init fw, aborting.\n"); + goto err_mfunc; + } + + if (mlx4_is_master(dev)) { + /* when we hit the goto slave_start below, dev_cap already initialized */ + if (!dev_cap) { + dev_cap = kzalloc(sizeof(*dev_cap), GFP_KERNEL); + + if (!dev_cap) { + err = -ENOMEM; + goto err_fw; + } + + err = mlx4_QUERY_DEV_CAP(dev, dev_cap); + if (err) { + mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n"); + goto err_fw; + } + + if (mlx4_check_dev_cap(dev, dev_cap, nvfs)) + goto err_fw; + + if (!(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS)) { + u64 dev_flags = mlx4_enable_sriov(dev, pdev, + total_vfs, + existing_vfs, + reset_flow); + + mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL); + dev->flags = dev_flags; + if (!SRIOV_VALID_STATE(dev->flags)) { + mlx4_err(dev, "Invalid SRIOV state\n"); + goto err_sriov; + } + err = mlx4_reset(dev); + if (err) { + mlx4_err(dev, "Failed to reset HCA, aborting.\n"); + goto err_sriov; + } + goto slave_start; + } + } else { + /* Legacy mode FW requires SRIOV to be enabled before + * doing QUERY_DEV_CAP, since max_eq's value is different if + * SRIOV is enabled. + */ + memset(dev_cap, 0, sizeof(*dev_cap)); + err = mlx4_QUERY_DEV_CAP(dev, dev_cap); + if (err) { + mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n"); + goto err_fw; + } + + if (mlx4_check_dev_cap(dev, dev_cap, nvfs)) + goto err_fw; + } + } + + err = mlx4_init_hca(dev); + if (err) { + if (err == -EACCES) { + /* Not primary Physical function + * Running in slave mode */ + mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL); + /* We're not a PF */ + if (dev->flags & MLX4_FLAG_SRIOV) { + if (!existing_vfs) + pci_disable_sriov(pdev); + if (mlx4_is_master(dev) && !reset_flow) + atomic_dec(&pf_loading); + dev->flags &= ~MLX4_FLAG_SRIOV; + } + if (!mlx4_is_slave(dev)) + mlx4_free_ownership(dev); + dev->flags |= MLX4_FLAG_SLAVE; + dev->flags &= ~MLX4_FLAG_MASTER; + goto slave_start; + } else + goto err_fw; + } + + if (mlx4_is_master(dev) && (dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS)) { + u64 dev_flags = mlx4_enable_sriov(dev, pdev, total_vfs, + existing_vfs, reset_flow); + + if ((dev->flags ^ dev_flags) & (MLX4_FLAG_MASTER | MLX4_FLAG_SLAVE)) { + mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_VHCR); + dev->flags = dev_flags; + err = mlx4_cmd_init(dev); + if (err) { + /* Only VHCR is cleaned up, so could still + * send FW commands + */ + mlx4_err(dev, "Failed to init VHCR command interface, aborting\n"); + goto err_close; + } + } else { + dev->flags = dev_flags; + } + + if (!SRIOV_VALID_STATE(dev->flags)) { + mlx4_err(dev, "Invalid SRIOV state\n"); + goto err_close; + } + } + + /* check if the device is functioning at its maximum possible speed. + * No return code for this call, just warn the user in case of PCI + * express device capabilities are under-satisfied by the bus. + */ + if (!mlx4_is_slave(dev)) + mlx4_check_pcie_caps(dev); + + /* In master functions, the communication channel must be initialized + * after obtaining its address from fw */ + if (mlx4_is_master(dev)) { + if (dev->caps.num_ports < 2 && + num_vfs_argc > 1) { + err = -EINVAL; + mlx4_err(dev, + "Error: Trying to configure VFs on port 2, but HCA has only %d physical ports\n", + dev->caps.num_ports); + goto err_close; + } + memcpy(dev->persist->nvfs, nvfs, sizeof(dev->persist->nvfs)); + + for (i = 0; + i < sizeof(dev->persist->nvfs)/ + sizeof(dev->persist->nvfs[0]); i++) { + unsigned j; + + for (j = 0; j < dev->persist->nvfs[i]; ++sum, ++j) { + dev->dev_vfs[sum].min_port = i < 2 ? i + 1 : 1; + dev->dev_vfs[sum].n_ports = i < 2 ? 1 : + dev->caps.num_ports; + } + } + + /* In master functions, the communication channel + * must be initialized after obtaining its address from fw + */ + err = mlx4_multi_func_init(dev); + if (err) { + mlx4_err(dev, "Failed to init master mfunc interface, aborting.\n"); + goto err_close; + } + } + + err = mlx4_alloc_eq_table(dev); + if (err) + goto err_master_mfunc; + + bitmap_zero(priv->msix_ctl.pool_bm, MAX_MSIX); + mutex_init(&priv->msix_ctl.pool_lock); + + mlx4_enable_msi_x(dev); + if ((mlx4_is_mfunc(dev)) && + !(dev->flags & MLX4_FLAG_MSI_X)) { + err = -ENOSYS; + mlx4_err(dev, "INTx is not supported in multi-function mode, aborting\n"); + goto err_free_eq; + } + + if (!mlx4_is_slave(dev)) { + err = mlx4_init_steering(dev); + if (err) + goto err_disable_msix; + } + + mlx4_init_quotas(dev); + + err = mlx4_setup_hca(dev); + if (err == -EBUSY && (dev->flags & MLX4_FLAG_MSI_X) && + !mlx4_is_mfunc(dev)) { + dev->flags &= ~MLX4_FLAG_MSI_X; + dev->caps.num_comp_vectors = 1; + pci_disable_msix(pdev); + err = mlx4_setup_hca(dev); + } + + if (err) + goto err_steer; + + /* When PF resources are ready arm its comm channel to enable + * getting commands + */ + if (mlx4_is_master(dev)) { + err = mlx4_ARM_COMM_CHANNEL(dev); + if (err) { + mlx4_err(dev, " Failed to arm comm channel eq: %x\n", + err); + goto err_steer; + } + } + + for (port = 1; port <= dev->caps.num_ports; port++) { + err = mlx4_init_port_info(dev, port); + if (err) + goto err_port; + } + + priv->v2p.port1 = 1; + priv->v2p.port2 = 2; + + err = mlx4_register_device(dev); + if (err) + goto err_port; + + mlx4_request_modules(dev); + + mlx4_sense_init(dev); + mlx4_start_sense(dev); + + priv->removed = 0; + + if (mlx4_is_master(dev) && dev->persist->num_vfs && !reset_flow) + atomic_dec(&pf_loading); + + kfree(dev_cap); + return 0; + +err_port: + for (--port; port >= 1; --port) + mlx4_cleanup_port_info(&priv->port[port]); + + mlx4_cleanup_counters_table(dev); + mlx4_cleanup_qp_table(dev); + mlx4_cleanup_srq_table(dev); + mlx4_cleanup_cq_table(dev); + mlx4_cmd_use_polling(dev); + mlx4_cleanup_eq_table(dev); + mlx4_cleanup_mcg_table(dev); + mlx4_cleanup_mr_table(dev); + mlx4_cleanup_xrcd_table(dev); + mlx4_cleanup_pd_table(dev); + mlx4_cleanup_uar_table(dev); + +err_steer: + if (!mlx4_is_slave(dev)) + mlx4_clear_steering(dev); + +err_disable_msix: + if (dev->flags & MLX4_FLAG_MSI_X) + pci_disable_msix(pdev); + +err_free_eq: + mlx4_free_eq_table(dev); + +err_master_mfunc: + if (mlx4_is_master(dev)) { + mlx4_free_resource_tracker(dev, RES_TR_FREE_STRUCTS_ONLY); + mlx4_multi_func_cleanup(dev); + } + + if (mlx4_is_slave(dev)) { + kfree(dev->caps.qp0_qkey); + kfree(dev->caps.qp0_tunnel); + kfree(dev->caps.qp0_proxy); + kfree(dev->caps.qp1_tunnel); + kfree(dev->caps.qp1_proxy); + } + +err_close: + mlx4_close_hca(dev); + +err_fw: + mlx4_close_fw(dev); + +err_mfunc: + if (mlx4_is_slave(dev)) + mlx4_multi_func_cleanup(dev); + +err_cmd: + mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL); + +err_sriov: + if (dev->flags & MLX4_FLAG_SRIOV && !existing_vfs) { + pci_disable_sriov(pdev); + dev->flags &= ~MLX4_FLAG_SRIOV; + } + + if (mlx4_is_master(dev) && dev->persist->num_vfs && !reset_flow) + atomic_dec(&pf_loading); + + kfree(priv->dev.dev_vfs); + + if (!mlx4_is_slave(dev)) + mlx4_free_ownership(dev); + + kfree(dev_cap); + return err; +} + +static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data, + struct mlx4_priv *priv) +{ + int err; + int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0}; + int prb_vf[MLX4_MAX_PORTS + 1] = {0, 0, 0}; + const int param_map[MLX4_MAX_PORTS + 1][MLX4_MAX_PORTS + 1] = { + {2, 0, 0}, {0, 1, 2}, {0, 1, 2} }; + unsigned total_vfs = 0; + unsigned int i; + + pr_info(DRV_NAME ": Initializing %s\n", pci_name(pdev)); + + err = mlx4_pci_enable_device(&priv->dev); + if (err) { + dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); + return err; + } + + /* Due to requirement that all VFs and the PF are *guaranteed* 2 MACS + * per port, we must limit the number of VFs to 63 (since their are + * 128 MACs) + */ + for (i = 0; i < sizeof(nvfs)/sizeof(nvfs[0]) && i < num_vfs_argc; + total_vfs += nvfs[param_map[num_vfs_argc - 1][i]], i++) { + nvfs[param_map[num_vfs_argc - 1][i]] = num_vfs[i]; + if (nvfs[i] < 0) { + dev_err(&pdev->dev, "num_vfs module parameter cannot be negative\n"); + err = -EINVAL; + goto err_disable_pdev; + } + } + for (i = 0; i < sizeof(prb_vf)/sizeof(prb_vf[0]) && i < probe_vfs_argc; + i++) { + prb_vf[param_map[probe_vfs_argc - 1][i]] = probe_vf[i]; + if (prb_vf[i] < 0 || prb_vf[i] > nvfs[i]) { + dev_err(&pdev->dev, "probe_vf module parameter cannot be negative or greater than num_vfs\n"); + err = -EINVAL; + goto err_disable_pdev; + } + } + if (total_vfs > MLX4_MAX_NUM_VF) { + dev_err(&pdev->dev, + "Requested more VF's (%d) than allowed by hw (%d)\n", + total_vfs, MLX4_MAX_NUM_VF); + err = -EINVAL; + goto err_disable_pdev; + } + + for (i = 0; i < MLX4_MAX_PORTS; i++) { + if (nvfs[i] + nvfs[2] > MLX4_MAX_NUM_VF_P_PORT) { + dev_err(&pdev->dev, + "Requested more VF's (%d) for port (%d) than allowed by driver (%d)\n", + nvfs[i] + nvfs[2], i + 1, + MLX4_MAX_NUM_VF_P_PORT); + err = -EINVAL; + goto err_disable_pdev; + } + } + + /* Check for BARs. */ + if (!(pci_dev_data & MLX4_PCI_DEV_IS_VF) && + !(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { + dev_err(&pdev->dev, "Missing DCS, aborting (driver_data: 0x%x, pci_resource_flags(pdev, 0):0x%lx)\n", + pci_dev_data, (long)pci_resource_flags(pdev, 0)); + err = -ENODEV; + goto err_disable_pdev; + } + if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) { + dev_err(&pdev->dev, "Missing UAR, aborting\n"); + err = -ENODEV; + goto err_disable_pdev; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n"); + goto err_disable_pdev; + } + + pci_set_master(pdev); + + err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); + if (err) { + dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n"); + err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); + if (err) { + dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n"); + goto err_release_regions; + } + } + err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); + if (err) { + dev_warn(&pdev->dev, "Warning: couldn't set 64-bit consistent PCI DMA mask\n"); + err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + if (err) { + dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, aborting\n"); + goto err_release_regions; + } + } + + /* Allow large DMA segments, up to the firmware limit of 1 GB */ + dma_set_max_seg_size(&pdev->dev, 1024 * 1024 * 1024); + /* Detect if this device is a virtual function */ + if (pci_dev_data & MLX4_PCI_DEV_IS_VF) { + /* When acting as pf, we normally skip vfs unless explicitly + * requested to probe them. + */ + if (total_vfs) { + unsigned vfs_offset = 0; + + for (i = 0; i < sizeof(nvfs)/sizeof(nvfs[0]) && + vfs_offset + nvfs[i] < extended_func_num(pdev); + vfs_offset += nvfs[i], i++) + ; + if (i == sizeof(nvfs)/sizeof(nvfs[0])) { + err = -ENODEV; + goto err_release_regions; + } + if ((extended_func_num(pdev) - vfs_offset) + > prb_vf[i]) { + dev_warn(&pdev->dev, "Skipping virtual function:%d\n", + extended_func_num(pdev)); + err = -ENODEV; + goto err_release_regions; + } + } + } + + err = mlx4_catas_init(&priv->dev); + if (err) + goto err_release_regions; + + err = mlx4_load_one(pdev, pci_dev_data, total_vfs, nvfs, priv, 0); + if (err) + goto err_catas; + + return 0; + +err_catas: + mlx4_catas_end(&priv->dev); + +err_release_regions: + pci_release_regions(pdev); + +err_disable_pdev: + mlx4_pci_disable_device(&priv->dev); + pci_set_drvdata(pdev, NULL); + return err; +} + +static int mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct mlx4_priv *priv; + struct mlx4_dev *dev; + int ret; + + printk_once(KERN_INFO "%s", mlx4_version); + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + dev = &priv->dev; + dev->persist = kzalloc(sizeof(*dev->persist), GFP_KERNEL); + if (!dev->persist) { + kfree(priv); + return -ENOMEM; + } + dev->persist->pdev = pdev; + dev->persist->dev = dev; + pci_set_drvdata(pdev, dev->persist); + priv->pci_dev_data = id->driver_data; + mutex_init(&dev->persist->device_state_mutex); + mutex_init(&dev->persist->interface_state_mutex); + mutex_init(&dev->persist->pci_status_mutex); + + ret = __mlx4_init_one(pdev, id->driver_data, priv); + if (ret) { + kfree(dev->persist); + kfree(priv); + } else { + pci_save_state(pdev->dev.bsddev); + } + + return ret; +} + +static void mlx4_clean_dev(struct mlx4_dev *dev) +{ + struct mlx4_dev_persistent *persist = dev->persist; + struct mlx4_priv *priv = mlx4_priv(dev); + unsigned long flags = (dev->flags & RESET_PERSIST_MASK_FLAGS); + + memset(priv, 0, sizeof(*priv)); + priv->dev.persist = persist; + priv->dev.flags = flags; +} + +static void mlx4_unload_one(struct pci_dev *pdev) +{ + struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); + struct mlx4_dev *dev = persist->dev; + struct mlx4_priv *priv = mlx4_priv(dev); + int pci_dev_data; + int p, i; + + if (priv->removed) + return; + + /* saving current ports type for further use */ + for (i = 0; i < dev->caps.num_ports; i++) { + dev->persist->curr_port_type[i] = dev->caps.port_type[i + 1]; + dev->persist->curr_port_poss_type[i] = dev->caps. + possible_type[i + 1]; + } + + pci_dev_data = priv->pci_dev_data; + + mlx4_stop_sense(dev); + mlx4_unregister_device(dev); + + for (p = 1; p <= dev->caps.num_ports; p++) { + mlx4_cleanup_port_info(&priv->port[p]); + mlx4_CLOSE_PORT(dev, p); + } + + if (mlx4_is_master(dev)) + mlx4_free_resource_tracker(dev, + RES_TR_FREE_SLAVES_ONLY); + + mlx4_cleanup_default_counters(dev); + if (!mlx4_is_slave(dev)) + mlx4_cleanup_counters_table(dev); + mlx4_cleanup_qp_table(dev); + mlx4_cleanup_srq_table(dev); + mlx4_cleanup_cq_table(dev); + mlx4_cmd_use_polling(dev); + mlx4_cleanup_eq_table(dev); + mlx4_cleanup_mcg_table(dev); + mlx4_cleanup_mr_table(dev); + mlx4_cleanup_xrcd_table(dev); + mlx4_cleanup_pd_table(dev); + + if (mlx4_is_master(dev)) + mlx4_free_resource_tracker(dev, + RES_TR_FREE_STRUCTS_ONLY); + + iounmap(priv->kar); + mlx4_uar_free(dev, &priv->driver_uar); + mlx4_cleanup_uar_table(dev); + if (!mlx4_is_slave(dev)) + mlx4_clear_steering(dev); + mlx4_free_eq_table(dev); + if (mlx4_is_master(dev)) + mlx4_multi_func_cleanup(dev); + mlx4_close_hca(dev); + mlx4_close_fw(dev); + if (mlx4_is_slave(dev)) + mlx4_multi_func_cleanup(dev); + mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL); + + if (dev->flags & MLX4_FLAG_MSI_X) + pci_disable_msix(pdev); + + if (!mlx4_is_slave(dev)) + mlx4_free_ownership(dev); + + kfree(dev->caps.qp0_qkey); + kfree(dev->caps.qp0_tunnel); + kfree(dev->caps.qp0_proxy); + kfree(dev->caps.qp1_tunnel); + kfree(dev->caps.qp1_proxy); + kfree(dev->dev_vfs); + + mlx4_clean_dev(dev); + priv->pci_dev_data = pci_dev_data; + priv->removed = 1; +} + +static void mlx4_remove_one(struct pci_dev *pdev) +{ + struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); + struct mlx4_dev *dev = persist->dev; + struct mlx4_priv *priv = mlx4_priv(dev); + int active_vfs = 0; + + mutex_lock(&persist->interface_state_mutex); + persist->interface_state |= MLX4_INTERFACE_STATE_DELETION; + mutex_unlock(&persist->interface_state_mutex); + + /* Disabling SR-IOV is not allowed while there are active vf's */ + if (mlx4_is_master(dev) && dev->flags & MLX4_FLAG_SRIOV) { + active_vfs = mlx4_how_many_lives_vf(dev); + if (active_vfs) { + pr_warn("Removing PF when there are active VF's !!\n"); + pr_warn("Will not disable SR-IOV.\n"); + } + } + + /* device marked to be under deletion running now without the lock + * letting other tasks to be terminated + */ + if (persist->interface_state & MLX4_INTERFACE_STATE_UP) + mlx4_unload_one(pdev); + else + mlx4_info(dev, "%s: interface is down\n", __func__); + mlx4_catas_end(dev); + if (dev->flags & MLX4_FLAG_SRIOV && !active_vfs) { + mlx4_warn(dev, "Disabling SR-IOV\n"); + pci_disable_sriov(pdev); + } + + pci_release_regions(pdev); + pci_disable_device(pdev); + kfree(dev->persist); + kfree(priv); + pci_set_drvdata(pdev, NULL); +} + +static int restore_current_port_types(struct mlx4_dev *dev, + enum mlx4_port_type *types, + enum mlx4_port_type *poss_types) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int err, i; + + mlx4_stop_sense(dev); + + mutex_lock(&priv->port_mutex); + for (i = 0; i < dev->caps.num_ports; i++) + dev->caps.possible_type[i + 1] = poss_types[i]; + err = mlx4_change_port_types(dev, types); + mutex_unlock(&priv->port_mutex); + + mlx4_start_sense(dev); + + return err; +} + +int mlx4_restart_one(struct pci_dev *pdev) +{ + struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); + struct mlx4_dev *dev = persist->dev; + struct mlx4_priv *priv = mlx4_priv(dev); + int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0}; + int pci_dev_data, err, total_vfs; + + pci_dev_data = priv->pci_dev_data; + total_vfs = dev->persist->num_vfs; + memcpy(nvfs, dev->persist->nvfs, sizeof(dev->persist->nvfs)); + + mlx4_unload_one(pdev); + err = mlx4_load_one(pdev, pci_dev_data, total_vfs, nvfs, priv, 1); + if (err) { + mlx4_err(dev, "%s: ERROR: mlx4_load_one failed, pci_name=%s, err=%d\n", + __func__, pci_name(pdev), err); + return err; + } + + err = restore_current_port_types(dev, dev->persist->curr_port_type, + dev->persist->curr_port_poss_type); + if (err) + mlx4_err(dev, "could not restore original port types (%d)\n", + err); + + return err; +} + +static DEFINE_PCI_DEVICE_TABLE(mlx4_pci_table) = { + /* MT25408 "Hermon" SDR */ + { PCI_VDEVICE(MELLANOX, 0x6340), + .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25408 "Hermon" DDR */ + { PCI_VDEVICE(MELLANOX, 0x634a), + .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25408 "Hermon" QDR */ + { PCI_VDEVICE(MELLANOX, 0x6354), + .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25408 "Hermon" DDR PCIe gen2 */ + { PCI_VDEVICE(MELLANOX, 0x6732), + .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25408 "Hermon" QDR PCIe gen2 */ + { PCI_VDEVICE(MELLANOX, 0x673c), + .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25408 "Hermon" EN 10GigE */ + { PCI_VDEVICE(MELLANOX, 0x6368), + .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25408 "Hermon" EN 10GigE PCIe gen2 */ + { PCI_VDEVICE(MELLANOX, 0x6750), + .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25458 ConnectX EN 10GBASE-T 10GigE */ + { PCI_VDEVICE(MELLANOX, 0x6372), + .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25458 ConnectX EN 10GBASE-T+Gen2 10GigE */ + { PCI_VDEVICE(MELLANOX, 0x675a), + .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT26468 ConnectX EN 10GigE PCIe gen2*/ + { PCI_VDEVICE(MELLANOX, 0x6764), + .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT26438 ConnectX EN 40GigE PCIe gen2 5GT/s */ + { PCI_VDEVICE(MELLANOX, 0x6746), + .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT26478 ConnectX2 40GigE PCIe gen2 */ + { PCI_VDEVICE(MELLANOX, 0x676e), + .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25400 Family [ConnectX-2 Virtual Function] */ + { PCI_VDEVICE(MELLANOX, 0x1002), + .driver_data = MLX4_PCI_DEV_IS_VF }, + /* MT27500 Family [ConnectX-3] */ + { PCI_VDEVICE(MELLANOX, 0x1003) }, + /* MT27500 Family [ConnectX-3 Virtual Function] */ + { PCI_VDEVICE(MELLANOX, 0x1004), + .driver_data = MLX4_PCI_DEV_IS_VF }, + { PCI_VDEVICE(MELLANOX, 0x1005) }, /* MT27510 Family */ + { PCI_VDEVICE(MELLANOX, 0x1006) }, /* MT27511 Family */ + { PCI_VDEVICE(MELLANOX, 0x1007) }, /* MT27520 Family */ + { PCI_VDEVICE(MELLANOX, 0x1008) }, /* MT27521 Family */ + { PCI_VDEVICE(MELLANOX, 0x1009) }, /* MT27530 Family */ + { PCI_VDEVICE(MELLANOX, 0x100a) }, /* MT27531 Family */ + { PCI_VDEVICE(MELLANOX, 0x100b) }, /* MT27540 Family */ + { PCI_VDEVICE(MELLANOX, 0x100c) }, /* MT27541 Family */ + { PCI_VDEVICE(MELLANOX, 0x100d) }, /* MT27550 Family */ + { PCI_VDEVICE(MELLANOX, 0x100e) }, /* MT27551 Family */ + { PCI_VDEVICE(MELLANOX, 0x100f) }, /* MT27560 Family */ + { PCI_VDEVICE(MELLANOX, 0x1010) }, /* MT27561 Family */ + { 0, } +}; + +MODULE_DEVICE_TABLE(pci, mlx4_pci_table); + +static pci_ers_result_t mlx4_pci_err_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); + + mlx4_err(persist->dev, "mlx4_pci_err_detected was called\n"); + mlx4_enter_error_state(persist); + + mutex_lock(&persist->interface_state_mutex); + if (persist->interface_state & MLX4_INTERFACE_STATE_UP) + mlx4_unload_one(pdev); + + mutex_unlock(&persist->interface_state_mutex); + if (state == pci_channel_io_perm_failure) + return PCI_ERS_RESULT_DISCONNECT; + + mlx4_pci_disable_device(persist->dev); + return PCI_ERS_RESULT_NEED_RESET; +} + +static pci_ers_result_t mlx4_pci_slot_reset(struct pci_dev *pdev) +{ + struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); + struct mlx4_dev *dev = persist->dev; + int err; + + mlx4_err(dev, "mlx4_pci_slot_reset was called\n"); + err = mlx4_pci_enable_device(dev); + if (err) { + mlx4_err(dev, "Can not re-enable device, err=%d\n", err); + return PCI_ERS_RESULT_DISCONNECT; + } + + pci_set_master(pdev); + return PCI_ERS_RESULT_RECOVERED; +} + +static void mlx4_pci_resume(struct pci_dev *pdev) +{ + struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); + struct mlx4_dev *dev = persist->dev; + struct mlx4_priv *priv = mlx4_priv(dev); + int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0}; + int total_vfs; + int err; + + mlx4_err(dev, "%s was called\n", __func__); + total_vfs = dev->persist->num_vfs; + memcpy(nvfs, dev->persist->nvfs, sizeof(dev->persist->nvfs)); + + mutex_lock(&persist->interface_state_mutex); + if (!(persist->interface_state & MLX4_INTERFACE_STATE_UP)) { + err = mlx4_load_one(pdev, priv->pci_dev_data, total_vfs, nvfs, + priv, 1); + if (err) { + mlx4_err(dev, "%s: mlx4_load_one failed, err=%d\n", + __func__, err); + goto end; + } + + err = restore_current_port_types(dev, dev->persist-> + curr_port_type, dev->persist-> + curr_port_poss_type); + if (err) + mlx4_err(dev, "could not restore original port types (%d)\n", err); + } +end: + mutex_unlock(&persist->interface_state_mutex); + +} + +static void mlx4_shutdown(struct pci_dev *pdev) +{ + struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); + + mlx4_info(persist->dev, "mlx4_shutdown was called\n"); + mutex_lock(&persist->interface_state_mutex); + if (persist->interface_state & MLX4_INTERFACE_STATE_UP) + mlx4_unload_one(pdev); + mutex_unlock(&persist->interface_state_mutex); +} + +static const struct pci_error_handlers mlx4_err_handler = { + .error_detected = mlx4_pci_err_detected, + .slot_reset = mlx4_pci_slot_reset, + .resume = mlx4_pci_resume, +}; + +static struct pci_driver mlx4_driver = { + .name = DRV_NAME, + .id_table = mlx4_pci_table, + .probe = mlx4_init_one, + .shutdown = mlx4_shutdown, + .remove = mlx4_remove_one, + .err_handler = &mlx4_err_handler, +}; + +static int __init mlx4_verify_params(void) +{ + if ((log_num_mac < 0) || (log_num_mac > 7)) { + pr_warn("mlx4_core: bad num_mac: %d\n", log_num_mac); + return -1; + } + + if (log_num_vlan != 0) + pr_warn("mlx4_core: log_num_vlan - obsolete module param, using %d\n", + MLX4_LOG_NUM_VLANS); + + if (use_prio != 0) + pr_warn("mlx4_core: use_prio - obsolete module param, ignored\n"); + + if ((log_mtts_per_seg < 1) || (log_mtts_per_seg > 7)) { + pr_warn("mlx4_core: bad log_mtts_per_seg: %d\n", + log_mtts_per_seg); + return -1; + } + + /* Check if module param for ports type has legal combination */ + if (port_type_array[0] == false && port_type_array[1] == true) { + pr_warn("Module parameter configuration ETH/IB is not supported. Switching to default configuration IB/IB\n"); + port_type_array[0] = true; + } + + if (mlx4_log_num_mgm_entry_size < -7 || + (mlx4_log_num_mgm_entry_size > 0 && + (mlx4_log_num_mgm_entry_size < MLX4_MIN_MGM_LOG_ENTRY_SIZE || + mlx4_log_num_mgm_entry_size > MLX4_MAX_MGM_LOG_ENTRY_SIZE))) { + pr_warn("mlx4_core: mlx4_log_num_mgm_entry_size (%d) not in legal range (-7..0 or %d..%d)\n", + mlx4_log_num_mgm_entry_size, + MLX4_MIN_MGM_LOG_ENTRY_SIZE, + MLX4_MAX_MGM_LOG_ENTRY_SIZE); + return -1; + } + + return 0; +} + +static int __init mlx4_init(void) +{ + int ret; + + if (mlx4_verify_params()) + return -EINVAL; + + + mlx4_wq = create_singlethread_workqueue("mlx4"); + if (!mlx4_wq) + return -ENOMEM; + + ret = pci_register_driver(&mlx4_driver); + if (ret < 0) + destroy_workqueue(mlx4_wq); + return ret < 0 ? ret : 0; +} + +static void __exit mlx4_cleanup(void) +{ + pci_unregister_driver(&mlx4_driver); + destroy_workqueue(mlx4_wq); +} + +module_init(mlx4_init); +module_exit(mlx4_cleanup); + +static int +mlx4_evhand(module_t mod, int event, void *arg) +{ + return (0); +} + +static moduledata_t mlx4_mod = { + .name = "mlx4", + .evhand = mlx4_evhand, +}; +MODULE_VERSION(mlx4, 1); +DECLARE_MODULE(mlx4, mlx4_mod, SI_SUB_OFED_PREINIT, SI_ORDER_ANY); +MODULE_DEPEND(mlx4, linuxkpi, 1, 1, 1); + Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_main.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_mcg.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_mcg.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_mcg.c (revision 329159) @@ -0,0 +1,1648 @@ +/* + * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include + +#include +#include +#include + +#include "mlx4.h" + +int mlx4_get_mgm_entry_size(struct mlx4_dev *dev) +{ + return 1 << dev->oper_log_mgm_entry_size; +} + +int mlx4_get_qp_per_mgm(struct mlx4_dev *dev) +{ + return 4 * (mlx4_get_mgm_entry_size(dev) / 16 - 2); +} + +static int mlx4_QP_FLOW_STEERING_ATTACH(struct mlx4_dev *dev, + struct mlx4_cmd_mailbox *mailbox, + u32 size, + u64 *reg_id) +{ + u64 imm; + int err = 0; + + err = mlx4_cmd_imm(dev, mailbox->dma, &imm, size, 0, + MLX4_QP_FLOW_STEERING_ATTACH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + return err; + *reg_id = imm; + + return err; +} + +static int mlx4_QP_FLOW_STEERING_DETACH(struct mlx4_dev *dev, u64 regid) +{ + int err = 0; + + err = mlx4_cmd(dev, regid, 0, 0, + MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + + return err; +} + +static int mlx4_READ_ENTRY(struct mlx4_dev *dev, int index, + struct mlx4_cmd_mailbox *mailbox) +{ + return mlx4_cmd_box(dev, 0, mailbox->dma, index, 0, MLX4_CMD_READ_MCG, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); +} + +static int mlx4_WRITE_ENTRY(struct mlx4_dev *dev, int index, + struct mlx4_cmd_mailbox *mailbox) +{ + return mlx4_cmd(dev, mailbox->dma, index, 0, MLX4_CMD_WRITE_MCG, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); +} + +static int mlx4_WRITE_PROMISC(struct mlx4_dev *dev, u8 port, u8 steer, + struct mlx4_cmd_mailbox *mailbox) +{ + u32 in_mod; + + in_mod = (u32) port << 16 | steer << 1; + return mlx4_cmd(dev, mailbox->dma, in_mod, 0x1, + MLX4_CMD_WRITE_MCG, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); +} + +static int mlx4_GID_HASH(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, + u16 *hash, u8 op_mod) +{ + u64 imm; + int err; + + err = mlx4_cmd_imm(dev, mailbox->dma, &imm, 0, op_mod, + MLX4_CMD_MGID_HASH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + + if (!err) + *hash = imm; + + return err; +} + +static struct mlx4_promisc_qp *get_promisc_qp(struct mlx4_dev *dev, u8 port, + enum mlx4_steer_type steer, + u32 qpn) +{ + struct mlx4_steer *s_steer; + struct mlx4_promisc_qp *pqp; + + if (port < 1 || port > dev->caps.num_ports) + return NULL; + + s_steer = &mlx4_priv(dev)->steer[port - 1]; + + list_for_each_entry(pqp, &s_steer->promisc_qps[steer], list) { + if (pqp->qpn == qpn) + return pqp; + } + /* not found */ + return NULL; +} + +/* + * Add new entry to steering data structure. + * All promisc QPs should be added as well + */ +static int new_steering_entry(struct mlx4_dev *dev, u8 port, + enum mlx4_steer_type steer, + unsigned int index, u32 qpn) +{ + struct mlx4_steer *s_steer; + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_mgm *mgm; + u32 members_count; + struct mlx4_steer_index *new_entry; + struct mlx4_promisc_qp *pqp; + struct mlx4_promisc_qp *dqp = NULL; + u32 prot; + int err; + + if (port < 1 || port > dev->caps.num_ports) + return -EINVAL; + + s_steer = &mlx4_priv(dev)->steer[port - 1]; + new_entry = kzalloc(sizeof *new_entry, GFP_KERNEL); + if (!new_entry) + return -ENOMEM; + + INIT_LIST_HEAD(&new_entry->duplicates); + new_entry->index = index; + list_add_tail(&new_entry->list, &s_steer->steer_entries[steer]); + + /* If the given qpn is also a promisc qp, + * it should be inserted to duplicates list + */ + pqp = get_promisc_qp(dev, port, steer, qpn); + if (pqp) { + dqp = kmalloc(sizeof *dqp, GFP_KERNEL); + if (!dqp) { + err = -ENOMEM; + goto out_alloc; + } + dqp->qpn = qpn; + list_add_tail(&dqp->list, &new_entry->duplicates); + } + + /* if no promisc qps for this vep, we are done */ + if (list_empty(&s_steer->promisc_qps[steer])) + return 0; + + /* now need to add all the promisc qps to the new + * steering entry, as they should also receive the packets + * destined to this address */ + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + err = -ENOMEM; + goto out_alloc; + } + mgm = mailbox->buf; + + err = mlx4_READ_ENTRY(dev, index, mailbox); + if (err) + goto out_mailbox; + + members_count = be32_to_cpu(mgm->members_count) & 0xffffff; + prot = be32_to_cpu(mgm->members_count) >> 30; + list_for_each_entry(pqp, &s_steer->promisc_qps[steer], list) { + /* don't add already existing qpn */ + if (pqp->qpn == qpn) + continue; + if (members_count == dev->caps.num_qp_per_mgm) { + /* out of space */ + err = -ENOMEM; + goto out_mailbox; + } + + /* add the qpn */ + mgm->qp[members_count++] = cpu_to_be32(pqp->qpn & MGM_QPN_MASK); + } + /* update the qps count and update the entry with all the promisc qps*/ + mgm->members_count = cpu_to_be32(members_count | (prot << 30)); + err = mlx4_WRITE_ENTRY(dev, index, mailbox); + +out_mailbox: + mlx4_free_cmd_mailbox(dev, mailbox); + if (!err) + return 0; +out_alloc: + if (dqp) { + list_del(&dqp->list); + kfree(dqp); + } + list_del(&new_entry->list); + kfree(new_entry); + return err; +} + +/* update the data structures with existing steering entry */ +static int existing_steering_entry(struct mlx4_dev *dev, u8 port, + enum mlx4_steer_type steer, + unsigned int index, u32 qpn) +{ + struct mlx4_steer *s_steer; + struct mlx4_steer_index *tmp_entry, *entry = NULL; + struct mlx4_promisc_qp *pqp; + struct mlx4_promisc_qp *dqp; + + if (port < 1 || port > dev->caps.num_ports) + return -EINVAL; + + s_steer = &mlx4_priv(dev)->steer[port - 1]; + + pqp = get_promisc_qp(dev, port, steer, qpn); + if (!pqp) + return 0; /* nothing to do */ + + list_for_each_entry(tmp_entry, &s_steer->steer_entries[steer], list) { + if (tmp_entry->index == index) { + entry = tmp_entry; + break; + } + } + if (unlikely(!entry)) { + mlx4_warn(dev, "Steering entry at index %x is not registered\n", index); + return -EINVAL; + } + + /* the given qpn is listed as a promisc qpn + * we need to add it as a duplicate to this entry + * for future references */ + list_for_each_entry(dqp, &entry->duplicates, list) { + if (qpn == dqp->qpn) + return 0; /* qp is already duplicated */ + } + + /* add the qp as a duplicate on this index */ + dqp = kmalloc(sizeof *dqp, GFP_KERNEL); + if (!dqp) + return -ENOMEM; + dqp->qpn = qpn; + list_add_tail(&dqp->list, &entry->duplicates); + + return 0; +} + +/* Check whether a qpn is a duplicate on steering entry + * If so, it should not be removed from mgm */ +static bool check_duplicate_entry(struct mlx4_dev *dev, u8 port, + enum mlx4_steer_type steer, + unsigned int index, u32 qpn) +{ + struct mlx4_steer *s_steer; + struct mlx4_steer_index *tmp_entry, *entry = NULL; + struct mlx4_promisc_qp *dqp, *tmp_dqp; + + if (port < 1 || port > dev->caps.num_ports) + return NULL; + + s_steer = &mlx4_priv(dev)->steer[port - 1]; + + /* if qp is not promisc, it cannot be duplicated */ + if (!get_promisc_qp(dev, port, steer, qpn)) + return false; + + /* The qp is promisc qp so it is a duplicate on this index + * Find the index entry, and remove the duplicate */ + list_for_each_entry(tmp_entry, &s_steer->steer_entries[steer], list) { + if (tmp_entry->index == index) { + entry = tmp_entry; + break; + } + } + if (unlikely(!entry)) { + mlx4_warn(dev, "Steering entry for index %x is not registered\n", index); + return false; + } + list_for_each_entry_safe(dqp, tmp_dqp, &entry->duplicates, list) { + if (dqp->qpn == qpn) { + list_del(&dqp->list); + kfree(dqp); + } + } + return true; +} + +/* Returns true if all the QPs != tqpn contained in this entry + * are Promisc QPs. Returns false otherwise. + */ +static bool promisc_steering_entry(struct mlx4_dev *dev, u8 port, + enum mlx4_steer_type steer, + unsigned int index, u32 tqpn, + u32 *members_count) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_mgm *mgm; + u32 m_count; + bool ret = false; + int i; + + if (port < 1 || port > dev->caps.num_ports) + return false; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return false; + mgm = mailbox->buf; + + if (mlx4_READ_ENTRY(dev, index, mailbox)) + goto out; + m_count = be32_to_cpu(mgm->members_count) & 0xffffff; + if (members_count) + *members_count = m_count; + + for (i = 0; i < m_count; i++) { + u32 qpn = be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK; + if (!get_promisc_qp(dev, port, steer, qpn) && qpn != tqpn) { + /* the qp is not promisc, the entry can't be removed */ + goto out; + } + } + ret = true; +out: + mlx4_free_cmd_mailbox(dev, mailbox); + return ret; +} + +/* IF a steering entry contains only promisc QPs, it can be removed. */ +static bool can_remove_steering_entry(struct mlx4_dev *dev, u8 port, + enum mlx4_steer_type steer, + unsigned int index, u32 tqpn) +{ + struct mlx4_steer *s_steer; + struct mlx4_steer_index *entry = NULL, *tmp_entry; + u32 members_count; + bool ret = false; + + if (port < 1 || port > dev->caps.num_ports) + return NULL; + + s_steer = &mlx4_priv(dev)->steer[port - 1]; + + if (!promisc_steering_entry(dev, port, steer, index, + tqpn, &members_count)) + goto out; + + /* All the qps currently registered for this entry are promiscuous, + * Checking for duplicates */ + ret = true; + list_for_each_entry_safe(entry, tmp_entry, &s_steer->steer_entries[steer], list) { + if (entry->index == index) { + if (list_empty(&entry->duplicates) || + members_count == 1) { + struct mlx4_promisc_qp *pqp, *tmp_pqp; + /* If there is only 1 entry in duplicates then + * this is the QP we want to delete, going over + * the list and deleting the entry. + */ + list_del(&entry->list); + list_for_each_entry_safe(pqp, tmp_pqp, + &entry->duplicates, + list) { + list_del(&pqp->list); + kfree(pqp); + } + kfree(entry); + } else { + /* This entry contains duplicates so it shouldn't be removed */ + ret = false; + goto out; + } + } + } + +out: + return ret; +} + +static int add_promisc_qp(struct mlx4_dev *dev, u8 port, + enum mlx4_steer_type steer, u32 qpn) +{ + struct mlx4_steer *s_steer; + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_mgm *mgm; + struct mlx4_steer_index *entry; + struct mlx4_promisc_qp *pqp; + struct mlx4_promisc_qp *dqp; + u32 members_count; + u32 prot; + int i; + bool found; + int err; + struct mlx4_priv *priv = mlx4_priv(dev); + + if (port < 1 || port > dev->caps.num_ports) + return -EINVAL; + + s_steer = &mlx4_priv(dev)->steer[port - 1]; + + mutex_lock(&priv->mcg_table.mutex); + + if (get_promisc_qp(dev, port, steer, qpn)) { + err = 0; /* Noting to do, already exists */ + goto out_mutex; + } + + pqp = kmalloc(sizeof *pqp, GFP_KERNEL); + if (!pqp) { + err = -ENOMEM; + goto out_mutex; + } + pqp->qpn = qpn; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + err = -ENOMEM; + goto out_alloc; + } + mgm = mailbox->buf; + + if (!(mlx4_is_mfunc(dev) && steer == MLX4_UC_STEER)) { + /* The promisc QP needs to be added for each one of the steering + * entries. If it already exists, needs to be added as + * a duplicate for this entry. + */ + list_for_each_entry(entry, + &s_steer->steer_entries[steer], + list) { + err = mlx4_READ_ENTRY(dev, entry->index, mailbox); + if (err) + goto out_mailbox; + + members_count = be32_to_cpu(mgm->members_count) & + 0xffffff; + prot = be32_to_cpu(mgm->members_count) >> 30; + found = false; + for (i = 0; i < members_count; i++) { + if ((be32_to_cpu(mgm->qp[i]) & + MGM_QPN_MASK) == qpn) { + /* Entry already exists. + * Add to duplicates. + */ + dqp = kmalloc(sizeof(*dqp), GFP_KERNEL); + if (!dqp) { + err = -ENOMEM; + goto out_mailbox; + } + dqp->qpn = qpn; + list_add_tail(&dqp->list, + &entry->duplicates); + found = true; + } + } + if (!found) { + /* Need to add the qpn to mgm */ + if (members_count == + dev->caps.num_qp_per_mgm) { + /* entry is full */ + err = -ENOMEM; + goto out_mailbox; + } + mgm->qp[members_count++] = + cpu_to_be32(qpn & MGM_QPN_MASK); + mgm->members_count = + cpu_to_be32(members_count | + (prot << 30)); + err = mlx4_WRITE_ENTRY(dev, entry->index, + mailbox); + if (err) + goto out_mailbox; + } + } + } + + /* add the new qpn to list of promisc qps */ + list_add_tail(&pqp->list, &s_steer->promisc_qps[steer]); + /* now need to add all the promisc qps to default entry */ + memset(mgm, 0, sizeof *mgm); + members_count = 0; + list_for_each_entry(dqp, &s_steer->promisc_qps[steer], list) { + if (members_count == dev->caps.num_qp_per_mgm) { + /* entry is full */ + err = -ENOMEM; + goto out_list; + } + mgm->qp[members_count++] = cpu_to_be32(dqp->qpn & MGM_QPN_MASK); + } + mgm->members_count = cpu_to_be32(members_count | MLX4_PROT_ETH << 30); + + err = mlx4_WRITE_PROMISC(dev, port, steer, mailbox); + if (err) + goto out_list; + + mlx4_free_cmd_mailbox(dev, mailbox); + mutex_unlock(&priv->mcg_table.mutex); + return 0; + +out_list: + list_del(&pqp->list); +out_mailbox: + mlx4_free_cmd_mailbox(dev, mailbox); +out_alloc: + kfree(pqp); +out_mutex: + mutex_unlock(&priv->mcg_table.mutex); + return err; +} + +static int remove_promisc_qp(struct mlx4_dev *dev, u8 port, + enum mlx4_steer_type steer, u32 qpn) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_steer *s_steer; + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_mgm *mgm; + struct mlx4_steer_index *entry, *tmp_entry; + struct mlx4_promisc_qp *pqp; + struct mlx4_promisc_qp *dqp; + u32 members_count; + bool found; + bool back_to_list = false; + int i; + int err; + + if (port < 1 || port > dev->caps.num_ports) + return -EINVAL; + + s_steer = &mlx4_priv(dev)->steer[port - 1]; + mutex_lock(&priv->mcg_table.mutex); + + pqp = get_promisc_qp(dev, port, steer, qpn); + if (unlikely(!pqp)) { + mlx4_warn(dev, "QP %x is not promiscuous QP\n", qpn); + /* nothing to do */ + err = 0; + goto out_mutex; + } + + /*remove from list of promisc qps */ + list_del(&pqp->list); + + /* set the default entry not to include the removed one */ + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + err = -ENOMEM; + back_to_list = true; + goto out_list; + } + mgm = mailbox->buf; + members_count = 0; + list_for_each_entry(dqp, &s_steer->promisc_qps[steer], list) + mgm->qp[members_count++] = cpu_to_be32(dqp->qpn & MGM_QPN_MASK); + mgm->members_count = cpu_to_be32(members_count | MLX4_PROT_ETH << 30); + + err = mlx4_WRITE_PROMISC(dev, port, steer, mailbox); + if (err) + goto out_mailbox; + + if (!(mlx4_is_mfunc(dev) && steer == MLX4_UC_STEER)) { + /* Remove the QP from all the steering entries */ + list_for_each_entry_safe(entry, tmp_entry, + &s_steer->steer_entries[steer], + list) { + found = false; + list_for_each_entry(dqp, &entry->duplicates, list) { + if (dqp->qpn == qpn) { + found = true; + break; + } + } + if (found) { + /* A duplicate, no need to change the MGM, + * only update the duplicates list + */ + list_del(&dqp->list); + kfree(dqp); + } else { + int loc = -1; + + err = mlx4_READ_ENTRY(dev, + entry->index, + mailbox); + if (err) + goto out_mailbox; + members_count = + be32_to_cpu(mgm->members_count) & + 0xffffff; + if (!members_count) { + mlx4_warn(dev, "QP %06x wasn't found in entry %x mcount=0. deleting entry...\n", + qpn, entry->index); + list_del(&entry->list); + kfree(entry); + continue; + } + + for (i = 0; i < members_count; ++i) + if ((be32_to_cpu(mgm->qp[i]) & + MGM_QPN_MASK) == qpn) { + loc = i; + break; + } + + if (loc < 0) { + mlx4_err(dev, "QP %06x wasn't found in entry %d\n", + qpn, entry->index); + err = -EINVAL; + goto out_mailbox; + } + + /* Copy the last QP in this MGM + * over removed QP + */ + mgm->qp[loc] = mgm->qp[members_count - 1]; + mgm->qp[members_count - 1] = 0; + mgm->members_count = + cpu_to_be32(--members_count | + (MLX4_PROT_ETH << 30)); + + err = mlx4_WRITE_ENTRY(dev, + entry->index, + mailbox); + if (err) + goto out_mailbox; + } + } + } + +out_mailbox: + mlx4_free_cmd_mailbox(dev, mailbox); +out_list: + if (back_to_list) + list_add_tail(&pqp->list, &s_steer->promisc_qps[steer]); + else + kfree(pqp); +out_mutex: + mutex_unlock(&priv->mcg_table.mutex); + return err; +} + +/* + * Caller must hold MCG table semaphore. gid and mgm parameters must + * be properly aligned for command interface. + * + * Returns 0 unless a firmware command error occurs. + * + * If GID is found in MGM or MGM is empty, *index = *hash, *prev = -1 + * and *mgm holds MGM entry. + * + * if GID is found in AMGM, *index = index in AMGM, *prev = index of + * previous entry in hash chain and *mgm holds AMGM entry. + * + * If no AMGM exists for given gid, *index = -1, *prev = index of last + * entry in hash chain and *mgm holds end of hash chain. + */ +static int find_entry(struct mlx4_dev *dev, u8 port, + u8 *gid, enum mlx4_protocol prot, + struct mlx4_cmd_mailbox *mgm_mailbox, + int *prev, int *index) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_mgm *mgm = mgm_mailbox->buf; + u8 *mgid; + int err; + u16 hash; + u8 op_mod = (prot == MLX4_PROT_ETH) ? + !!(dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) : 0; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return -ENOMEM; + mgid = mailbox->buf; + + memcpy(mgid, gid, 16); + + err = mlx4_GID_HASH(dev, mailbox, &hash, op_mod); + mlx4_free_cmd_mailbox(dev, mailbox); + if (err) + return err; + + if (0) { + mlx4_dbg(dev, "Hash for "GID_PRINT_FMT" is %04x\n", + GID_PRINT_ARGS(gid), hash); + } + + *index = hash; + *prev = -1; + + do { + err = mlx4_READ_ENTRY(dev, *index, mgm_mailbox); + if (err) + return err; + + if (!(be32_to_cpu(mgm->members_count) & 0xffffff)) { + if (*index != hash) { + mlx4_err(dev, "Found zero MGID in AMGM\n"); + err = -EINVAL; + } + return err; + } + + if (!memcmp(mgm->gid, gid, 16) && + be32_to_cpu(mgm->members_count) >> 30 == prot) + return err; + + *prev = *index; + *index = be32_to_cpu(mgm->next_gid_index) >> 6; + } while (*index); + + *index = -1; + return err; +} + +static const u8 __promisc_mode[] = { + [MLX4_FS_REGULAR] = 0x0, + [MLX4_FS_ALL_DEFAULT] = 0x1, + [MLX4_FS_MC_DEFAULT] = 0x3, + [MLX4_FS_MIRROR_RX_PORT] = 0x4, + [MLX4_FS_MIRROR_SX_PORT] = 0x5, + [MLX4_FS_UC_SNIFFER] = 0x6, + [MLX4_FS_MC_SNIFFER] = 0x7, +}; + +int mlx4_map_sw_to_hw_steering_mode(struct mlx4_dev *dev, + enum mlx4_net_trans_promisc_mode flow_type) +{ + if (flow_type >= MLX4_FS_MODE_NUM) { + mlx4_err(dev, "Invalid flow type. type = %d\n", flow_type); + return -EINVAL; + } + return __promisc_mode[flow_type]; +} +EXPORT_SYMBOL_GPL(mlx4_map_sw_to_hw_steering_mode); + +static void trans_rule_ctrl_to_hw(struct mlx4_net_trans_rule *ctrl, + struct mlx4_net_trans_rule_hw_ctrl *hw) +{ + u8 flags = 0; + + flags = ctrl->queue_mode == MLX4_NET_TRANS_Q_LIFO ? 1 : 0; + flags |= ctrl->exclusive ? (1 << 2) : 0; + flags |= ctrl->allow_loopback ? (1 << 3) : 0; + + hw->flags = flags; + hw->type = __promisc_mode[ctrl->promisc_mode]; + hw->prio = cpu_to_be16(ctrl->priority); + hw->port = ctrl->port; + hw->qpn = cpu_to_be32(ctrl->qpn); +} + +const u16 __sw_id_hw[] = { + [MLX4_NET_TRANS_RULE_ID_ETH] = 0xE001, + [MLX4_NET_TRANS_RULE_ID_IB] = 0xE005, + [MLX4_NET_TRANS_RULE_ID_IPV6] = 0xE003, + [MLX4_NET_TRANS_RULE_ID_IPV4] = 0xE002, + [MLX4_NET_TRANS_RULE_ID_TCP] = 0xE004, + [MLX4_NET_TRANS_RULE_ID_UDP] = 0xE006, + [MLX4_NET_TRANS_RULE_ID_VXLAN] = 0xE008 +}; + +int mlx4_map_sw_to_hw_steering_id(struct mlx4_dev *dev, + enum mlx4_net_trans_rule_id id) +{ + if (id >= MLX4_NET_TRANS_RULE_NUM) { + mlx4_err(dev, "Invalid network rule id. id = %d\n", id); + return -EINVAL; + } + return __sw_id_hw[id]; +} +EXPORT_SYMBOL_GPL(mlx4_map_sw_to_hw_steering_id); + +static const int __rule_hw_sz[] = { + [MLX4_NET_TRANS_RULE_ID_ETH] = + sizeof(struct mlx4_net_trans_rule_hw_eth), + [MLX4_NET_TRANS_RULE_ID_IB] = + sizeof(struct mlx4_net_trans_rule_hw_ib), + [MLX4_NET_TRANS_RULE_ID_IPV6] = 0, + [MLX4_NET_TRANS_RULE_ID_IPV4] = + sizeof(struct mlx4_net_trans_rule_hw_ipv4), + [MLX4_NET_TRANS_RULE_ID_TCP] = + sizeof(struct mlx4_net_trans_rule_hw_tcp_udp), + [MLX4_NET_TRANS_RULE_ID_UDP] = + sizeof(struct mlx4_net_trans_rule_hw_tcp_udp), + [MLX4_NET_TRANS_RULE_ID_VXLAN] = + sizeof(struct mlx4_net_trans_rule_hw_vxlan) +}; + +int mlx4_hw_rule_sz(struct mlx4_dev *dev, + enum mlx4_net_trans_rule_id id) +{ + if (id >= MLX4_NET_TRANS_RULE_NUM) { + mlx4_err(dev, "Invalid network rule id. id = %d\n", id); + return -EINVAL; + } + + return __rule_hw_sz[id]; +} +EXPORT_SYMBOL_GPL(mlx4_hw_rule_sz); + +static int parse_trans_rule(struct mlx4_dev *dev, struct mlx4_spec_list *spec, + struct _rule_hw *rule_hw) +{ + if (mlx4_hw_rule_sz(dev, spec->id) < 0) + return -EINVAL; + memset(rule_hw, 0, mlx4_hw_rule_sz(dev, spec->id)); + rule_hw->id = cpu_to_be16(__sw_id_hw[spec->id]); + rule_hw->size = mlx4_hw_rule_sz(dev, spec->id) >> 2; + + switch (spec->id) { + case MLX4_NET_TRANS_RULE_ID_ETH: + memcpy(rule_hw->eth.dst_mac, spec->eth.dst_mac, ETH_ALEN); + memcpy(rule_hw->eth.dst_mac_msk, spec->eth.dst_mac_msk, + ETH_ALEN); + memcpy(rule_hw->eth.src_mac, spec->eth.src_mac, ETH_ALEN); + memcpy(rule_hw->eth.src_mac_msk, spec->eth.src_mac_msk, + ETH_ALEN); + if (spec->eth.ether_type_enable) { + rule_hw->eth.ether_type_enable = 1; + rule_hw->eth.ether_type = spec->eth.ether_type; + } + rule_hw->eth.vlan_tag = spec->eth.vlan_id; + rule_hw->eth.vlan_tag_msk = spec->eth.vlan_id_msk; + break; + + case MLX4_NET_TRANS_RULE_ID_IB: + rule_hw->ib.l3_qpn = spec->ib.l3_qpn; + rule_hw->ib.qpn_mask = spec->ib.qpn_msk; + memcpy(&rule_hw->ib.dst_gid, &spec->ib.dst_gid, 16); + memcpy(&rule_hw->ib.dst_gid_msk, &spec->ib.dst_gid_msk, 16); + break; + + case MLX4_NET_TRANS_RULE_ID_IPV6: + return -EOPNOTSUPP; + + case MLX4_NET_TRANS_RULE_ID_IPV4: + rule_hw->ipv4.src_ip = spec->ipv4.src_ip; + rule_hw->ipv4.src_ip_msk = spec->ipv4.src_ip_msk; + rule_hw->ipv4.dst_ip = spec->ipv4.dst_ip; + rule_hw->ipv4.dst_ip_msk = spec->ipv4.dst_ip_msk; + break; + + case MLX4_NET_TRANS_RULE_ID_TCP: + case MLX4_NET_TRANS_RULE_ID_UDP: + rule_hw->tcp_udp.dst_port = spec->tcp_udp.dst_port; + rule_hw->tcp_udp.dst_port_msk = spec->tcp_udp.dst_port_msk; + rule_hw->tcp_udp.src_port = spec->tcp_udp.src_port; + rule_hw->tcp_udp.src_port_msk = spec->tcp_udp.src_port_msk; + break; + + case MLX4_NET_TRANS_RULE_ID_VXLAN: + rule_hw->vxlan.vni = + cpu_to_be32(be32_to_cpu(spec->vxlan.vni) << 8); + rule_hw->vxlan.vni_mask = + cpu_to_be32(be32_to_cpu(spec->vxlan.vni_mask) << 8); + break; + + default: + return -EINVAL; + } + + return __rule_hw_sz[spec->id]; +} + +static void mlx4_err_rule(struct mlx4_dev *dev, char *str, + struct mlx4_net_trans_rule *rule) +{ +#define BUF_SIZE 256 + struct mlx4_spec_list *cur; + char buf[BUF_SIZE]; + int len = 0; + + mlx4_err(dev, "%s", str); + len += snprintf(buf + len, BUF_SIZE - len, + "port = %d prio = 0x%x qp = 0x%x ", + rule->port, rule->priority, rule->qpn); + + list_for_each_entry(cur, &rule->list, list) { + switch (cur->id) { + case MLX4_NET_TRANS_RULE_ID_ETH: + len += snprintf(buf + len, BUF_SIZE - len, + "dmac = 0x%02x%02x%02x%02x%02x%02x ", + cur->eth.dst_mac[0], cur->eth.dst_mac[1], + cur->eth.dst_mac[2], cur->eth.dst_mac[3], + cur->eth.dst_mac[4], cur->eth.dst_mac[5]); + if (cur->eth.ether_type) + len += snprintf(buf + len, BUF_SIZE - len, + "ethertype = 0x%x ", + be16_to_cpu(cur->eth.ether_type)); + if (cur->eth.vlan_id) + len += snprintf(buf + len, BUF_SIZE - len, + "vlan-id = %d ", + be16_to_cpu(cur->eth.vlan_id)); + break; + + case MLX4_NET_TRANS_RULE_ID_IPV4: + if (cur->ipv4.src_ip) + len += snprintf(buf + len, BUF_SIZE - len, + "src-ip = %pI4 ", + &cur->ipv4.src_ip); + if (cur->ipv4.dst_ip) + len += snprintf(buf + len, BUF_SIZE - len, + "dst-ip = %pI4 ", + &cur->ipv4.dst_ip); + break; + + case MLX4_NET_TRANS_RULE_ID_TCP: + case MLX4_NET_TRANS_RULE_ID_UDP: + if (cur->tcp_udp.src_port) + len += snprintf(buf + len, BUF_SIZE - len, + "src-port = %d ", + be16_to_cpu(cur->tcp_udp.src_port)); + if (cur->tcp_udp.dst_port) + len += snprintf(buf + len, BUF_SIZE - len, + "dst-port = %d ", + be16_to_cpu(cur->tcp_udp.dst_port)); + break; + + case MLX4_NET_TRANS_RULE_ID_IB: + len += snprintf(buf + len, BUF_SIZE - len, + "dst-gid = "GID_PRINT_FMT"\n", + GID_PRINT_ARGS(cur->ib.dst_gid)); + len += snprintf(buf + len, BUF_SIZE - len, + "dst-gid-mask = "GID_PRINT_FMT"\n", + GID_PRINT_ARGS(cur->ib.dst_gid_msk)); + break; + + case MLX4_NET_TRANS_RULE_ID_VXLAN: + len += snprintf(buf + len, BUF_SIZE - len, + "VNID = %d ", be32_to_cpu(cur->vxlan.vni)); + break; + case MLX4_NET_TRANS_RULE_ID_IPV6: + break; + + default: + break; + } + } + len += snprintf(buf + len, BUF_SIZE - len, "\n"); + mlx4_err(dev, "%s", buf); + + if (len >= BUF_SIZE) + mlx4_err(dev, "Network rule error message was truncated, print buffer is too small\n"); +} + +int mlx4_flow_attach(struct mlx4_dev *dev, + struct mlx4_net_trans_rule *rule, u64 *reg_id) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_spec_list *cur; + u32 size = 0; + int ret; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + trans_rule_ctrl_to_hw(rule, mailbox->buf); + + size += sizeof(struct mlx4_net_trans_rule_hw_ctrl); + + list_for_each_entry(cur, &rule->list, list) { + ret = parse_trans_rule(dev, cur, mailbox->buf + size); + if (ret < 0) { + mlx4_free_cmd_mailbox(dev, mailbox); + return ret; + } + size += ret; + } + + ret = mlx4_QP_FLOW_STEERING_ATTACH(dev, mailbox, size >> 2, reg_id); + if (ret == -ENOMEM) { + mlx4_err_rule(dev, + "mcg table is full. Fail to register network rule\n", + rule); + } else if (ret) { + if (ret == -ENXIO) { + if (dev->caps.steering_mode != MLX4_STEERING_MODE_DEVICE_MANAGED) + mlx4_err_rule(dev, + "DMFS is not enabled, " + "failed to register network rule.\n", + rule); + else + mlx4_err_rule(dev, + "Rule exceeds the dmfs_high_rate_mode limitations, " + "failed to register network rule.\n", + rule); + + } else { + mlx4_err_rule(dev, "Fail to register network rule.\n", rule); + } + } + + mlx4_free_cmd_mailbox(dev, mailbox); + + return ret; +} +EXPORT_SYMBOL_GPL(mlx4_flow_attach); + +int mlx4_flow_detach(struct mlx4_dev *dev, u64 reg_id) +{ + int err; + + err = mlx4_QP_FLOW_STEERING_DETACH(dev, reg_id); + if (err) + mlx4_err(dev, "Fail to detach network rule. registration id = 0x%llx\n", + (unsigned long long)reg_id); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_flow_detach); + +int mlx4_tunnel_steer_add(struct mlx4_dev *dev, unsigned char *addr, + int port, int qpn, u16 prio, u64 *reg_id) +{ + int err; + struct mlx4_spec_list spec_eth_outer = { {NULL} }; + struct mlx4_spec_list spec_vxlan = { {NULL} }; + struct mlx4_spec_list spec_eth_inner = { {NULL} }; + + struct mlx4_net_trans_rule rule = { + .queue_mode = MLX4_NET_TRANS_Q_FIFO, + .exclusive = 0, + .allow_loopback = 1, + .promisc_mode = MLX4_FS_REGULAR, + }; + + __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); + + rule.port = port; + rule.qpn = qpn; + rule.priority = prio; + INIT_LIST_HEAD(&rule.list); + + spec_eth_outer.id = MLX4_NET_TRANS_RULE_ID_ETH; + memcpy(spec_eth_outer.eth.dst_mac, addr, ETH_ALEN); + memcpy(spec_eth_outer.eth.dst_mac_msk, &mac_mask, ETH_ALEN); + + spec_vxlan.id = MLX4_NET_TRANS_RULE_ID_VXLAN; /* any vxlan header */ + spec_eth_inner.id = MLX4_NET_TRANS_RULE_ID_ETH; /* any inner eth header */ + + list_add_tail(&spec_eth_outer.list, &rule.list); + list_add_tail(&spec_vxlan.list, &rule.list); + list_add_tail(&spec_eth_inner.list, &rule.list); + + err = mlx4_flow_attach(dev, &rule, reg_id); + return err; +} +EXPORT_SYMBOL(mlx4_tunnel_steer_add); + +int mlx4_FLOW_STEERING_IB_UC_QP_RANGE(struct mlx4_dev *dev, u32 min_range_qpn, + u32 max_range_qpn) +{ + int err; + u64 in_param; + + in_param = ((u64) min_range_qpn) << 32; + in_param |= ((u64) max_range_qpn) & 0xFFFFFFFF; + + err = mlx4_cmd(dev, in_param, 0, 0, + MLX4_FLOW_STEERING_IB_UC_QP_RANGE, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + + return err; +} +EXPORT_SYMBOL_GPL(mlx4_FLOW_STEERING_IB_UC_QP_RANGE); + +int mlx4_qp_attach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + int block_mcast_loopback, enum mlx4_protocol prot, + enum mlx4_steer_type steer) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_mgm *mgm; + u32 members_count; + int index = -1, prev; + int link = 0; + int i; + int err; + u8 port = gid[5]; + u8 new_entry = 0; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + mgm = mailbox->buf; + + mutex_lock(&priv->mcg_table.mutex); + err = find_entry(dev, port, gid, prot, + mailbox, &prev, &index); + if (err) + goto out; + + if (index != -1) { + if (!(be32_to_cpu(mgm->members_count) & 0xffffff)) { + new_entry = 1; + memcpy(mgm->gid, gid, 16); + } + } else { + link = 1; + + index = mlx4_bitmap_alloc(&priv->mcg_table.bitmap); + if (index == -1) { + mlx4_err(dev, "No AMGM entries left\n"); + err = -ENOMEM; + goto out; + } + index += dev->caps.num_mgms; + + new_entry = 1; + memset(mgm, 0, sizeof *mgm); + memcpy(mgm->gid, gid, 16); + } + + members_count = be32_to_cpu(mgm->members_count) & 0xffffff; + if (members_count == dev->caps.num_qp_per_mgm) { + mlx4_err(dev, "MGM at index %x is full\n", index); + err = -ENOMEM; + goto out; + } + + for (i = 0; i < members_count; ++i) + if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qp->qpn) { + mlx4_dbg(dev, "QP %06x already a member of MGM\n", qp->qpn); + err = 0; + goto out; + } + + if (block_mcast_loopback) + mgm->qp[members_count++] = cpu_to_be32((qp->qpn & MGM_QPN_MASK) | + (1U << MGM_BLCK_LB_BIT)); + else + mgm->qp[members_count++] = cpu_to_be32(qp->qpn & MGM_QPN_MASK); + + mgm->members_count = cpu_to_be32(members_count | (u32) prot << 30); + + err = mlx4_WRITE_ENTRY(dev, index, mailbox); + if (err) + goto out; + + if (!link) + goto out; + + err = mlx4_READ_ENTRY(dev, prev, mailbox); + if (err) + goto out; + + mgm->next_gid_index = cpu_to_be32(index << 6); + + err = mlx4_WRITE_ENTRY(dev, prev, mailbox); + if (err) + goto out; + +out: + if (prot == MLX4_PROT_ETH && index != -1) { + /* manage the steering entry for promisc mode */ + if (new_entry) + err = new_steering_entry(dev, port, steer, + index, qp->qpn); + else + err = existing_steering_entry(dev, port, steer, + index, qp->qpn); + } + if (err && link && index != -1) { + if (index < dev->caps.num_mgms) + mlx4_warn(dev, "Got AMGM index %d < %d\n", + index, dev->caps.num_mgms); + else + mlx4_bitmap_free(&priv->mcg_table.bitmap, + index - dev->caps.num_mgms, MLX4_USE_RR); + } + mutex_unlock(&priv->mcg_table.mutex); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +int mlx4_qp_detach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + enum mlx4_protocol prot, enum mlx4_steer_type steer) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_mgm *mgm; + u32 members_count; + int prev, index; + int i, loc = -1; + int err; + u8 port = gid[5]; + bool removed_entry = false; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + mgm = mailbox->buf; + + mutex_lock(&priv->mcg_table.mutex); + + err = find_entry(dev, port, gid, prot, + mailbox, &prev, &index); + if (err) + goto out; + + if (index == -1) { + mlx4_err(dev, "MGID "GID_PRINT_FMT" not found\n", + GID_PRINT_ARGS(gid)); + err = -EINVAL; + goto out; + } + + /* If this QP is also a promisc QP, it shouldn't be removed only if + * at least one none promisc QP is also attached to this MCG + */ + if (prot == MLX4_PROT_ETH && + check_duplicate_entry(dev, port, steer, index, qp->qpn) && + !promisc_steering_entry(dev, port, steer, index, qp->qpn, NULL)) + goto out; + + members_count = be32_to_cpu(mgm->members_count) & 0xffffff; + for (i = 0; i < members_count; ++i) + if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qp->qpn) { + loc = i; + break; + } + + if (loc == -1) { + mlx4_err(dev, "QP %06x not found in MGM\n", qp->qpn); + err = -EINVAL; + goto out; + } + + /* copy the last QP in this MGM over removed QP */ + mgm->qp[loc] = mgm->qp[members_count - 1]; + mgm->qp[members_count - 1] = 0; + mgm->members_count = cpu_to_be32(--members_count | (u32) prot << 30); + + if (prot == MLX4_PROT_ETH) + removed_entry = can_remove_steering_entry(dev, port, steer, + index, qp->qpn); + if (members_count && (prot != MLX4_PROT_ETH || !removed_entry)) { + err = mlx4_WRITE_ENTRY(dev, index, mailbox); + goto out; + } + + /* We are going to delete the entry, members count should be 0 */ + mgm->members_count = cpu_to_be32((u32) prot << 30); + + if (prev == -1) { + /* Remove entry from MGM */ + int amgm_index = be32_to_cpu(mgm->next_gid_index) >> 6; + if (amgm_index) { + err = mlx4_READ_ENTRY(dev, amgm_index, mailbox); + if (err) + goto out; + } else + memset(mgm->gid, 0, 16); + + err = mlx4_WRITE_ENTRY(dev, index, mailbox); + if (err) + goto out; + + if (amgm_index) { + if (amgm_index < dev->caps.num_mgms) + mlx4_warn(dev, "MGM entry %d had AMGM index %d < %d\n", + index, amgm_index, dev->caps.num_mgms); + else + mlx4_bitmap_free(&priv->mcg_table.bitmap, + amgm_index - dev->caps.num_mgms, MLX4_USE_RR); + } + } else { + /* Remove entry from AMGM */ + int cur_next_index = be32_to_cpu(mgm->next_gid_index) >> 6; + err = mlx4_READ_ENTRY(dev, prev, mailbox); + if (err) + goto out; + + mgm->next_gid_index = cpu_to_be32(cur_next_index << 6); + + err = mlx4_WRITE_ENTRY(dev, prev, mailbox); + if (err) + goto out; + + if (index < dev->caps.num_mgms) + mlx4_warn(dev, "entry %d had next AMGM index %d < %d\n", + prev, index, dev->caps.num_mgms); + else + mlx4_bitmap_free(&priv->mcg_table.bitmap, + index - dev->caps.num_mgms, MLX4_USE_RR); + } + +out: + mutex_unlock(&priv->mcg_table.mutex); + + mlx4_free_cmd_mailbox(dev, mailbox); + if (err && dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) + /* In case device is under an error, return success as a closing command */ + err = 0; + return err; +} + +static int mlx4_QP_ATTACH(struct mlx4_dev *dev, struct mlx4_qp *qp, + u8 gid[16], u8 attach, u8 block_loopback, + enum mlx4_protocol prot) +{ + struct mlx4_cmd_mailbox *mailbox; + int err = 0; + int qpn; + + if (!mlx4_is_mfunc(dev)) + return -EBADF; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + memcpy(mailbox->buf, gid, 16); + qpn = qp->qpn; + qpn |= (prot << 28); + if (attach && block_loopback) + qpn |= (1 << 31); + + err = mlx4_cmd(dev, mailbox->dma, qpn, attach, + MLX4_CMD_QP_ATTACH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); + + mlx4_free_cmd_mailbox(dev, mailbox); + if (err && !attach && + dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) + err = 0; + return err; +} + +int mlx4_trans_to_dmfs_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, + u8 gid[16], u8 port, + int block_mcast_loopback, + enum mlx4_protocol prot, u64 *reg_id) +{ + struct mlx4_spec_list spec = { {NULL} }; + __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); + + struct mlx4_net_trans_rule rule = { + .queue_mode = MLX4_NET_TRANS_Q_FIFO, + .exclusive = 0, + .promisc_mode = MLX4_FS_REGULAR, + .priority = MLX4_DOMAIN_NIC, + }; + + rule.allow_loopback = !block_mcast_loopback; + rule.port = port; + rule.qpn = qp->qpn; + INIT_LIST_HEAD(&rule.list); + + switch (prot) { + case MLX4_PROT_ETH: + spec.id = MLX4_NET_TRANS_RULE_ID_ETH; + memcpy(spec.eth.dst_mac, &gid[10], ETH_ALEN); + memcpy(spec.eth.dst_mac_msk, &mac_mask, ETH_ALEN); + break; + + case MLX4_PROT_IB_IPV6: + spec.id = MLX4_NET_TRANS_RULE_ID_IB; + memcpy(spec.ib.dst_gid, gid, 16); + memset(&spec.ib.dst_gid_msk, 0xff, 16); + break; + default: + return -EINVAL; + } + list_add_tail(&spec.list, &rule.list); + + return mlx4_flow_attach(dev, &rule, reg_id); +} + +int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + u8 port, int block_mcast_loopback, + enum mlx4_protocol prot, u64 *reg_id) +{ + switch (dev->caps.steering_mode) { + case MLX4_STEERING_MODE_A0: + if (prot == MLX4_PROT_ETH) + return 0; + + case MLX4_STEERING_MODE_B0: + if (prot == MLX4_PROT_ETH) + gid[7] |= (MLX4_MC_STEER << 1); + + if (mlx4_is_mfunc(dev)) + return mlx4_QP_ATTACH(dev, qp, gid, 1, + block_mcast_loopback, prot); + return mlx4_qp_attach_common(dev, qp, gid, + block_mcast_loopback, prot, + MLX4_MC_STEER); + + case MLX4_STEERING_MODE_DEVICE_MANAGED: + return mlx4_trans_to_dmfs_attach(dev, qp, gid, port, + block_mcast_loopback, + prot, reg_id); + default: + return -EINVAL; + } +} +EXPORT_SYMBOL_GPL(mlx4_multicast_attach); + +int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + enum mlx4_protocol prot, u64 reg_id) +{ + switch (dev->caps.steering_mode) { + case MLX4_STEERING_MODE_A0: + if (prot == MLX4_PROT_ETH) + return 0; + + case MLX4_STEERING_MODE_B0: + if (prot == MLX4_PROT_ETH) + gid[7] |= (MLX4_MC_STEER << 1); + + if (mlx4_is_mfunc(dev)) + return mlx4_QP_ATTACH(dev, qp, gid, 0, 0, prot); + + return mlx4_qp_detach_common(dev, qp, gid, prot, + MLX4_MC_STEER); + + case MLX4_STEERING_MODE_DEVICE_MANAGED: + return mlx4_flow_detach(dev, reg_id); + + default: + return -EINVAL; + } +} +EXPORT_SYMBOL_GPL(mlx4_multicast_detach); + +int mlx4_flow_steer_promisc_add(struct mlx4_dev *dev, u8 port, + u32 qpn, enum mlx4_net_trans_promisc_mode mode) +{ + struct mlx4_net_trans_rule rule = { + .queue_mode = MLX4_NET_TRANS_Q_FIFO, + .exclusive = 0, + .allow_loopback = 1, + }; + + u64 *regid_p; + + switch (mode) { + case MLX4_FS_ALL_DEFAULT: + regid_p = &dev->regid_promisc_array[port]; + break; + case MLX4_FS_MC_DEFAULT: + regid_p = &dev->regid_allmulti_array[port]; + break; + default: + return -1; + } + + if (*regid_p != 0) + return -1; + + rule.promisc_mode = mode; + rule.port = port; + rule.qpn = qpn; + INIT_LIST_HEAD(&rule.list); + mlx4_err(dev, "going promisc on %x\n", port); + + return mlx4_flow_attach(dev, &rule, regid_p); +} +EXPORT_SYMBOL_GPL(mlx4_flow_steer_promisc_add); + +int mlx4_flow_steer_promisc_remove(struct mlx4_dev *dev, u8 port, + enum mlx4_net_trans_promisc_mode mode) +{ + int ret; + u64 *regid_p; + + switch (mode) { + case MLX4_FS_ALL_DEFAULT: + regid_p = &dev->regid_promisc_array[port]; + break; + case MLX4_FS_MC_DEFAULT: + regid_p = &dev->regid_allmulti_array[port]; + break; + default: + return -1; + } + + if (*regid_p == 0) + return -1; + + ret = mlx4_flow_detach(dev, *regid_p); + if (ret == 0) + *regid_p = 0; + + return ret; +} +EXPORT_SYMBOL_GPL(mlx4_flow_steer_promisc_remove); + +int mlx4_unicast_attach(struct mlx4_dev *dev, + struct mlx4_qp *qp, u8 gid[16], + int block_mcast_loopback, enum mlx4_protocol prot) +{ + if (prot == MLX4_PROT_ETH) + gid[7] |= (MLX4_UC_STEER << 1); + + if (mlx4_is_mfunc(dev)) + return mlx4_QP_ATTACH(dev, qp, gid, 1, + block_mcast_loopback, prot); + + return mlx4_qp_attach_common(dev, qp, gid, block_mcast_loopback, + prot, MLX4_UC_STEER); +} +EXPORT_SYMBOL_GPL(mlx4_unicast_attach); + +int mlx4_unicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, + u8 gid[16], enum mlx4_protocol prot) +{ + if (prot == MLX4_PROT_ETH) + gid[7] |= (MLX4_UC_STEER << 1); + + if (mlx4_is_mfunc(dev)) + return mlx4_QP_ATTACH(dev, qp, gid, 0, 0, prot); + + return mlx4_qp_detach_common(dev, qp, gid, prot, MLX4_UC_STEER); +} +EXPORT_SYMBOL_GPL(mlx4_unicast_detach); + +int mlx4_PROMISC_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + u32 qpn = (u32) vhcr->in_param & 0xffffffff; + int port = mlx4_slave_convert_port(dev, slave, vhcr->in_param >> 62); + enum mlx4_steer_type steer = vhcr->in_modifier; + + if (port < 0) + return -EINVAL; + + /* Promiscuous unicast is not allowed in mfunc */ + if (mlx4_is_mfunc(dev) && steer == MLX4_UC_STEER) + return 0; + + if (vhcr->op_modifier) + return add_promisc_qp(dev, port, steer, qpn); + else + return remove_promisc_qp(dev, port, steer, qpn); +} + +static int mlx4_PROMISC(struct mlx4_dev *dev, u32 qpn, + enum mlx4_steer_type steer, u8 add, u8 port) +{ + return mlx4_cmd(dev, (u64) qpn | (u64) port << 62, (u32) steer, add, + MLX4_CMD_PROMISC, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); +} + +int mlx4_multicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port) +{ + if (mlx4_is_mfunc(dev)) + return mlx4_PROMISC(dev, qpn, MLX4_MC_STEER, 1, port); + + return add_promisc_qp(dev, port, MLX4_MC_STEER, qpn); +} +EXPORT_SYMBOL_GPL(mlx4_multicast_promisc_add); + +int mlx4_multicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port) +{ + if (mlx4_is_mfunc(dev)) + return mlx4_PROMISC(dev, qpn, MLX4_MC_STEER, 0, port); + + return remove_promisc_qp(dev, port, MLX4_MC_STEER, qpn); +} +EXPORT_SYMBOL_GPL(mlx4_multicast_promisc_remove); + +int mlx4_unicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port) +{ + if (mlx4_is_mfunc(dev)) + return mlx4_PROMISC(dev, qpn, MLX4_UC_STEER, 1, port); + + return add_promisc_qp(dev, port, MLX4_UC_STEER, qpn); +} +EXPORT_SYMBOL_GPL(mlx4_unicast_promisc_add); + +int mlx4_unicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port) +{ + if (mlx4_is_mfunc(dev)) + return mlx4_PROMISC(dev, qpn, MLX4_UC_STEER, 0, port); + + return remove_promisc_qp(dev, port, MLX4_UC_STEER, qpn); +} +EXPORT_SYMBOL_GPL(mlx4_unicast_promisc_remove); + +int mlx4_init_mcg_table(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int err; + + /* No need for mcg_table when fw managed the mcg table*/ + if (dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) + return 0; + err = mlx4_bitmap_init(&priv->mcg_table.bitmap, dev->caps.num_amgms, + dev->caps.num_amgms - 1, 0, 0); + if (err) + return err; + + mutex_init(&priv->mcg_table.mutex); + + return 0; +} + +void mlx4_cleanup_mcg_table(struct mlx4_dev *dev) +{ + if (dev->caps.steering_mode != + MLX4_STEERING_MODE_DEVICE_MANAGED) + mlx4_bitmap_cleanup(&mlx4_priv(dev)->mcg_table.bitmap); +} Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_mcg.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_mr.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_mr.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_mr.c (revision 329159) @@ -0,0 +1,1161 @@ +/* + * Copyright (c) 2004 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "mlx4.h" +#include "icm.h" + +static u32 mlx4_buddy_alloc(struct mlx4_buddy *buddy, int order) +{ + int o; + int m; + u32 seg; + + spin_lock(&buddy->lock); + + for (o = order; o <= buddy->max_order; ++o) + if (buddy->num_free[o]) { + m = 1 << (buddy->max_order - o); + seg = find_first_bit(buddy->bits[o], m); + if (seg < m) + goto found; + } + + spin_unlock(&buddy->lock); + return -1; + + found: + clear_bit(seg, buddy->bits[o]); + --buddy->num_free[o]; + + while (o > order) { + --o; + seg <<= 1; + set_bit(seg ^ 1, buddy->bits[o]); + ++buddy->num_free[o]; + } + + spin_unlock(&buddy->lock); + + seg <<= order; + + return seg; +} + +static void mlx4_buddy_free(struct mlx4_buddy *buddy, u32 seg, int order) +{ + seg >>= order; + + spin_lock(&buddy->lock); + + while (test_bit(seg ^ 1, buddy->bits[order])) { + clear_bit(seg ^ 1, buddy->bits[order]); + --buddy->num_free[order]; + seg >>= 1; + ++order; + } + + set_bit(seg, buddy->bits[order]); + ++buddy->num_free[order]; + + spin_unlock(&buddy->lock); +} + +static int mlx4_buddy_init(struct mlx4_buddy *buddy, int max_order) +{ + int i, s; + + buddy->max_order = max_order; + spin_lock_init(&buddy->lock); + + buddy->bits = kcalloc(buddy->max_order + 1, sizeof (long *), + GFP_KERNEL); + buddy->num_free = kcalloc((buddy->max_order + 1), sizeof *buddy->num_free, + GFP_KERNEL); + if (!buddy->bits || !buddy->num_free) + goto err_out; + + for (i = 0; i <= buddy->max_order; ++i) { + s = BITS_TO_LONGS(1 << (buddy->max_order - i)); + buddy->bits[i] = kcalloc(s, sizeof (long), GFP_KERNEL | __GFP_NOWARN); + if (!buddy->bits[i]) { + buddy->bits[i] = vzalloc(s * sizeof(long)); + if (!buddy->bits[i]) + goto err_out_free; + } + } + + set_bit(0, buddy->bits[buddy->max_order]); + buddy->num_free[buddy->max_order] = 1; + + return 0; + +err_out_free: + for (i = 0; i <= buddy->max_order; ++i) + kvfree(buddy->bits[i]); + +err_out: + kfree(buddy->bits); + kfree(buddy->num_free); + + return -ENOMEM; +} + +static void mlx4_buddy_cleanup(struct mlx4_buddy *buddy) +{ + int i; + + for (i = 0; i <= buddy->max_order; ++i) + kvfree(buddy->bits[i]); + + kfree(buddy->bits); + kfree(buddy->num_free); +} + +u32 __mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) +{ + struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; + u32 seg; + int seg_order; + u32 offset; + + seg_order = max_t(int, order - log_mtts_per_seg, 0); + + seg = mlx4_buddy_alloc(&mr_table->mtt_buddy, seg_order); + if (seg == -1) + return -1; + + offset = seg * (1 << log_mtts_per_seg); + + if (mlx4_table_get_range(dev, &mr_table->mtt_table, offset, + offset + (1 << order) - 1)) { + mlx4_buddy_free(&mr_table->mtt_buddy, seg, seg_order); + return -1; + } + + return offset; +} + +static u32 mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) +{ + u64 in_param = 0; + u64 out_param; + int err; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, order); + err = mlx4_cmd_imm(dev, in_param, &out_param, RES_MTT, + RES_OP_RESERVE_AND_MAP, + MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); + if (err) + return -1; + return get_param_l(&out_param); + } + return __mlx4_alloc_mtt_range(dev, order); +} + +int mlx4_mtt_init(struct mlx4_dev *dev, int npages, int page_shift, + struct mlx4_mtt *mtt) +{ + int i; + + if (!npages) { + mtt->order = -1; + mtt->page_shift = MLX4_ICM_PAGE_SHIFT; + return 0; + } else + mtt->page_shift = page_shift; + + for (mtt->order = 0, i = 1; i < npages; i <<= 1) + ++mtt->order; + + mtt->offset = mlx4_alloc_mtt_range(dev, mtt->order); + if (mtt->offset == -1) + return -ENOMEM; + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_mtt_init); + +void __mlx4_free_mtt_range(struct mlx4_dev *dev, u32 offset, int order) +{ + u32 first_seg; + int seg_order; + struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; + + seg_order = max_t(int, order - log_mtts_per_seg, 0); + first_seg = offset / (1 << log_mtts_per_seg); + + mlx4_buddy_free(&mr_table->mtt_buddy, first_seg, seg_order); + mlx4_table_put_range(dev, &mr_table->mtt_table, offset, + offset + (1 << order) - 1); +} + +static void mlx4_free_mtt_range(struct mlx4_dev *dev, u32 offset, int order) +{ + u64 in_param = 0; + int err; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, offset); + set_param_h(&in_param, order); + err = mlx4_cmd(dev, in_param, RES_MTT, RES_OP_RESERVE_AND_MAP, + MLX4_CMD_FREE_RES, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); + if (err) + mlx4_warn(dev, "Failed to free mtt range at:%d order:%d\n", + offset, order); + return; + } + __mlx4_free_mtt_range(dev, offset, order); +} + +void mlx4_mtt_cleanup(struct mlx4_dev *dev, struct mlx4_mtt *mtt) +{ + if (mtt->order < 0) + return; + + mlx4_free_mtt_range(dev, mtt->offset, mtt->order); +} +EXPORT_SYMBOL_GPL(mlx4_mtt_cleanup); + +u64 mlx4_mtt_addr(struct mlx4_dev *dev, struct mlx4_mtt *mtt) +{ + return (u64) mtt->offset * dev->caps.mtt_entry_sz; +} +EXPORT_SYMBOL_GPL(mlx4_mtt_addr); + +static u32 hw_index_to_key(u32 ind) +{ + return (ind >> 24) | (ind << 8); +} + +static u32 key_to_hw_index(u32 key) +{ + return (key << 24) | (key >> 8); +} + +static int mlx4_SW2HW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, + int mpt_index) +{ + return mlx4_cmd(dev, mailbox->dma, mpt_index, + 0, MLX4_CMD_SW2HW_MPT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); +} + +static int mlx4_HW2SW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, + int mpt_index) +{ + return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, mpt_index, + !mailbox, MLX4_CMD_HW2SW_MPT, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); +} + +/* Must protect against concurrent access */ +int mlx4_mr_hw_get_mpt(struct mlx4_dev *dev, struct mlx4_mr *mmr, + struct mlx4_mpt_entry ***mpt_entry) +{ + int err; + int key = key_to_hw_index(mmr->key) & (dev->caps.num_mpts - 1); + struct mlx4_cmd_mailbox *mailbox = NULL; + + if (mmr->enabled != MLX4_MPT_EN_HW) + return -EINVAL; + + err = mlx4_HW2SW_MPT(dev, NULL, key); + if (err) { + mlx4_warn(dev, "HW2SW_MPT failed (%d).", err); + mlx4_warn(dev, "Most likely the MR has MWs bound to it.\n"); + return err; + } + + mmr->enabled = MLX4_MPT_EN_SW; + + if (!mlx4_is_mfunc(dev)) { + **mpt_entry = mlx4_table_find( + &mlx4_priv(dev)->mr_table.dmpt_table, + key, NULL); + } else { + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + err = mlx4_cmd_box(dev, 0, mailbox->dma, key, + 0, MLX4_CMD_QUERY_MPT, + MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); + if (err) + goto free_mailbox; + + *mpt_entry = (struct mlx4_mpt_entry **)&mailbox->buf; + } + + if (!(*mpt_entry) || !(**mpt_entry)) { + err = -ENOMEM; + goto free_mailbox; + } + + return 0; + +free_mailbox: + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_mr_hw_get_mpt); + +int mlx4_mr_hw_write_mpt(struct mlx4_dev *dev, struct mlx4_mr *mmr, + struct mlx4_mpt_entry **mpt_entry) +{ + int err; + + if (!mlx4_is_mfunc(dev)) { + /* Make sure any changes to this entry are flushed */ + wmb(); + + *(u8 *)(*mpt_entry) = MLX4_MPT_STATUS_HW; + + /* Make sure the new status is written */ + wmb(); + + err = mlx4_SYNC_TPT(dev); + } else { + int key = key_to_hw_index(mmr->key) & (dev->caps.num_mpts - 1); + + struct mlx4_cmd_mailbox *mailbox = + container_of((void *)mpt_entry, struct mlx4_cmd_mailbox, + buf); + + err = mlx4_SW2HW_MPT(dev, mailbox, key); + } + + if (!err) { + mmr->pd = be32_to_cpu((*mpt_entry)->pd_flags) & MLX4_MPT_PD_MASK; + mmr->enabled = MLX4_MPT_EN_HW; + } + return err; +} +EXPORT_SYMBOL_GPL(mlx4_mr_hw_write_mpt); + +void mlx4_mr_hw_put_mpt(struct mlx4_dev *dev, + struct mlx4_mpt_entry **mpt_entry) +{ + if (mlx4_is_mfunc(dev)) { + struct mlx4_cmd_mailbox *mailbox = + container_of((void *)mpt_entry, struct mlx4_cmd_mailbox, + buf); + mlx4_free_cmd_mailbox(dev, mailbox); + } +} +EXPORT_SYMBOL_GPL(mlx4_mr_hw_put_mpt); + +int mlx4_mr_hw_change_pd(struct mlx4_dev *dev, struct mlx4_mpt_entry *mpt_entry, + u32 pdn) +{ + u32 pd_flags = be32_to_cpu(mpt_entry->pd_flags) & ~MLX4_MPT_PD_MASK; + /* The wrapper function will put the slave's id here */ + if (mlx4_is_mfunc(dev)) + pd_flags &= ~MLX4_MPT_PD_VF_MASK; + + mpt_entry->pd_flags = cpu_to_be32(pd_flags | + (pdn & MLX4_MPT_PD_MASK) + | MLX4_MPT_PD_FLAG_EN_INV); + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_mr_hw_change_pd); + +int mlx4_mr_hw_change_access(struct mlx4_dev *dev, + struct mlx4_mpt_entry *mpt_entry, + u32 access) +{ + u32 flags = (be32_to_cpu(mpt_entry->flags) & ~MLX4_PERM_MASK) | + (access & MLX4_PERM_MASK); + + mpt_entry->flags = cpu_to_be32(flags); + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_mr_hw_change_access); + +static int mlx4_mr_alloc_reserved(struct mlx4_dev *dev, u32 mridx, u32 pd, + u64 iova, u64 size, u32 access, int npages, + int page_shift, struct mlx4_mr *mr) +{ + mr->iova = iova; + mr->size = size; + mr->pd = pd; + mr->access = access; + mr->enabled = MLX4_MPT_DISABLED; + mr->key = hw_index_to_key(mridx); + + return mlx4_mtt_init(dev, npages, page_shift, &mr->mtt); +} + +static int mlx4_WRITE_MTT(struct mlx4_dev *dev, + struct mlx4_cmd_mailbox *mailbox, + int num_entries) +{ + return mlx4_cmd(dev, mailbox->dma, num_entries, 0, MLX4_CMD_WRITE_MTT, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); +} + +int __mlx4_mpt_reserve(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + return mlx4_bitmap_alloc(&priv->mr_table.mpt_bitmap); +} + +static int mlx4_mpt_reserve(struct mlx4_dev *dev) +{ + u64 out_param; + + if (mlx4_is_mfunc(dev)) { + if (mlx4_cmd_imm(dev, 0, &out_param, RES_MPT, RES_OP_RESERVE, + MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) + return -1; + return get_param_l(&out_param); + } + return __mlx4_mpt_reserve(dev); +} + +void __mlx4_mpt_release(struct mlx4_dev *dev, u32 index) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + mlx4_bitmap_free(&priv->mr_table.mpt_bitmap, index, MLX4_NO_RR); +} + +static void mlx4_mpt_release(struct mlx4_dev *dev, u32 index) +{ + u64 in_param = 0; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, index); + if (mlx4_cmd(dev, in_param, RES_MPT, RES_OP_RESERVE, + MLX4_CMD_FREE_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) + mlx4_warn(dev, "Failed to release mr index:%d\n", + index); + return; + } + __mlx4_mpt_release(dev, index); +} + +int __mlx4_mpt_alloc_icm(struct mlx4_dev *dev, u32 index, gfp_t gfp) +{ + struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; + + return mlx4_table_get(dev, &mr_table->dmpt_table, index, gfp); +} + +static int mlx4_mpt_alloc_icm(struct mlx4_dev *dev, u32 index, gfp_t gfp) +{ + u64 param = 0; + + if (mlx4_is_mfunc(dev)) { + set_param_l(¶m, index); + return mlx4_cmd_imm(dev, param, ¶m, RES_MPT, RES_OP_MAP_ICM, + MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); + } + return __mlx4_mpt_alloc_icm(dev, index, gfp); +} + +void __mlx4_mpt_free_icm(struct mlx4_dev *dev, u32 index) +{ + struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; + + mlx4_table_put(dev, &mr_table->dmpt_table, index); +} + +static void mlx4_mpt_free_icm(struct mlx4_dev *dev, u32 index) +{ + u64 in_param = 0; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, index); + if (mlx4_cmd(dev, in_param, RES_MPT, RES_OP_MAP_ICM, + MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED)) + mlx4_warn(dev, "Failed to free icm of mr index:%d\n", + index); + return; + } + return __mlx4_mpt_free_icm(dev, index); +} + +int mlx4_mr_alloc(struct mlx4_dev *dev, u32 pd, u64 iova, u64 size, u32 access, + int npages, int page_shift, struct mlx4_mr *mr) +{ + u32 index; + int err; + + index = mlx4_mpt_reserve(dev); + if (index == -1) + return -ENOMEM; + + err = mlx4_mr_alloc_reserved(dev, index, pd, iova, size, + access, npages, page_shift, mr); + if (err) + mlx4_mpt_release(dev, index); + + return err; +} +EXPORT_SYMBOL_GPL(mlx4_mr_alloc); + +static int mlx4_mr_free_reserved(struct mlx4_dev *dev, struct mlx4_mr *mr) +{ + int err; + + if (mr->enabled == MLX4_MPT_EN_HW) { + err = mlx4_HW2SW_MPT(dev, NULL, + key_to_hw_index(mr->key) & + (dev->caps.num_mpts - 1)); + if (err) { + mlx4_warn(dev, "HW2SW_MPT failed (%d), MR has MWs bound to it\n", + err); + return err; + } + + mr->enabled = MLX4_MPT_EN_SW; + } + mlx4_mtt_cleanup(dev, &mr->mtt); + + return 0; +} + +int mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr) +{ + int ret; + + ret = mlx4_mr_free_reserved(dev, mr); + if (ret) + return ret; + if (mr->enabled) + mlx4_mpt_free_icm(dev, key_to_hw_index(mr->key)); + mlx4_mpt_release(dev, key_to_hw_index(mr->key)); + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_mr_free); + +void mlx4_mr_rereg_mem_cleanup(struct mlx4_dev *dev, struct mlx4_mr *mr) +{ + mlx4_mtt_cleanup(dev, &mr->mtt); + mr->mtt.order = -1; +} +EXPORT_SYMBOL_GPL(mlx4_mr_rereg_mem_cleanup); + +int mlx4_mr_rereg_mem_write(struct mlx4_dev *dev, struct mlx4_mr *mr, + u64 iova, u64 size, int npages, + int page_shift, struct mlx4_mpt_entry *mpt_entry) +{ + int err; + + err = mlx4_mtt_init(dev, npages, page_shift, &mr->mtt); + if (err) + return err; + + mpt_entry->start = cpu_to_be64(iova); + mpt_entry->length = cpu_to_be64(size); + mpt_entry->entity_size = cpu_to_be32(page_shift); + mpt_entry->flags &= ~(cpu_to_be32(MLX4_MPT_FLAG_FREE | + MLX4_MPT_FLAG_SW_OWNS)); + if (mr->mtt.order < 0) { + mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_PHYSICAL); + mpt_entry->mtt_addr = 0; + } else { + mpt_entry->mtt_addr = cpu_to_be64(mlx4_mtt_addr(dev, + &mr->mtt)); + if (mr->mtt.page_shift == 0) + mpt_entry->mtt_sz = cpu_to_be32(1 << mr->mtt.order); + } + if (mr->mtt.order >= 0 && mr->mtt.page_shift == 0) { + /* fast register MR in free state */ + mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_FREE); + mpt_entry->pd_flags |= cpu_to_be32(MLX4_MPT_PD_FLAG_FAST_REG | + MLX4_MPT_PD_FLAG_RAE); + } else { + mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_SW_OWNS); + } + mr->enabled = MLX4_MPT_EN_SW; + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_mr_rereg_mem_write); + +int mlx4_mr_enable(struct mlx4_dev *dev, struct mlx4_mr *mr) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_mpt_entry *mpt_entry; + int err; + + err = mlx4_mpt_alloc_icm(dev, key_to_hw_index(mr->key), GFP_KERNEL); + if (err) + return err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + err = PTR_ERR(mailbox); + goto err_table; + } + mpt_entry = mailbox->buf; + mpt_entry->flags = cpu_to_be32(MLX4_MPT_FLAG_MIO | + MLX4_MPT_FLAG_REGION | + mr->access); + + mpt_entry->key = cpu_to_be32(key_to_hw_index(mr->key)); + mpt_entry->pd_flags = cpu_to_be32(mr->pd | MLX4_MPT_PD_FLAG_EN_INV); + mpt_entry->start = cpu_to_be64(mr->iova); + mpt_entry->length = cpu_to_be64(mr->size); + mpt_entry->entity_size = cpu_to_be32(mr->mtt.page_shift); + + if (mr->mtt.order < 0) { + mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_PHYSICAL); + mpt_entry->mtt_addr = 0; + } else { + mpt_entry->mtt_addr = cpu_to_be64(mlx4_mtt_addr(dev, + &mr->mtt)); + } + + if (mr->mtt.order >= 0 && mr->mtt.page_shift == 0) { + /* fast register MR in free state */ + mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_FREE); + mpt_entry->pd_flags |= cpu_to_be32(MLX4_MPT_PD_FLAG_FAST_REG | + MLX4_MPT_PD_FLAG_RAE); + mpt_entry->mtt_sz = cpu_to_be32(1 << mr->mtt.order); + } else { + mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_SW_OWNS); + } + + err = mlx4_SW2HW_MPT(dev, mailbox, + key_to_hw_index(mr->key) & (dev->caps.num_mpts - 1)); + if (err) { + mlx4_warn(dev, "SW2HW_MPT failed (%d)\n", err); + goto err_cmd; + } + mr->enabled = MLX4_MPT_EN_HW; + + mlx4_free_cmd_mailbox(dev, mailbox); + + return 0; + +err_cmd: + mlx4_free_cmd_mailbox(dev, mailbox); + +err_table: + mlx4_mpt_free_icm(dev, key_to_hw_index(mr->key)); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_mr_enable); + +static int mlx4_write_mtt_chunk(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + int start_index, int npages, u64 *page_list) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + __be64 *mtts; + dma_addr_t dma_handle; + int i; + + mtts = mlx4_table_find(&priv->mr_table.mtt_table, mtt->offset + + start_index, &dma_handle); + + if (!mtts) + return -ENOMEM; + + dma_sync_single_for_cpu(&dev->persist->pdev->dev, dma_handle, + npages * sizeof (u64), DMA_TO_DEVICE); + + for (i = 0; i < npages; ++i) + mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT); + + dma_sync_single_for_device(&dev->persist->pdev->dev, dma_handle, + npages * sizeof (u64), DMA_TO_DEVICE); + + return 0; +} + +int __mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + int start_index, int npages, u64 *page_list) +{ + int err = 0; + int chunk; + int mtts_per_page; + int max_mtts_first_page; + + /* compute how may mtts fit in the first page */ + mtts_per_page = PAGE_SIZE / sizeof(u64); + max_mtts_first_page = mtts_per_page - (mtt->offset + start_index) + % mtts_per_page; + + chunk = min_t(int, max_mtts_first_page, npages); + + while (npages > 0) { + err = mlx4_write_mtt_chunk(dev, mtt, start_index, chunk, page_list); + if (err) + return err; + npages -= chunk; + start_index += chunk; + page_list += chunk; + + chunk = min_t(int, mtts_per_page, npages); + } + return err; +} + +int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + int start_index, int npages, u64 *page_list) +{ + struct mlx4_cmd_mailbox *mailbox = NULL; + __be64 *inbox = NULL; + int chunk; + int err = 0; + int i; + + if (mtt->order < 0) + return -EINVAL; + + if (mlx4_is_mfunc(dev)) { + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + inbox = mailbox->buf; + + while (npages > 0) { + chunk = min_t(int, MLX4_MAILBOX_SIZE / sizeof(u64) - 2, + npages); + inbox[0] = cpu_to_be64(mtt->offset + start_index); + inbox[1] = 0; + for (i = 0; i < chunk; ++i) + inbox[i + 2] = cpu_to_be64(page_list[i] | + MLX4_MTT_FLAG_PRESENT); + err = mlx4_WRITE_MTT(dev, mailbox, chunk); + if (err) { + mlx4_free_cmd_mailbox(dev, mailbox); + return err; + } + + npages -= chunk; + start_index += chunk; + page_list += chunk; + } + mlx4_free_cmd_mailbox(dev, mailbox); + return err; + } + + return __mlx4_write_mtt(dev, mtt, start_index, npages, page_list); +} +EXPORT_SYMBOL_GPL(mlx4_write_mtt); + +int mlx4_buf_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + struct mlx4_buf *buf, gfp_t gfp) +{ + u64 *page_list; + int err; + int i; + + page_list = kmalloc(buf->npages * sizeof *page_list, + gfp); + if (!page_list) + return -ENOMEM; + + for (i = 0; i < buf->npages; ++i) + if (buf->nbufs == 1) + page_list[i] = buf->direct.map + (i << buf->page_shift); + else + page_list[i] = buf->page_list[i].map; + + err = mlx4_write_mtt(dev, mtt, 0, buf->npages, page_list); + + kfree(page_list); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_buf_write_mtt); + +int mlx4_mw_alloc(struct mlx4_dev *dev, u32 pd, enum mlx4_mw_type type, + struct mlx4_mw *mw) +{ + u32 index; + + if ((type == MLX4_MW_TYPE_1 && + !(dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW)) || + (type == MLX4_MW_TYPE_2 && + !(dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN))) + return -ENOTSUPP; + + index = mlx4_mpt_reserve(dev); + if (index == -1) + return -ENOMEM; + + mw->key = hw_index_to_key(index); + mw->pd = pd; + mw->type = type; + mw->enabled = MLX4_MPT_DISABLED; + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_mw_alloc); + +int mlx4_mw_enable(struct mlx4_dev *dev, struct mlx4_mw *mw) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_mpt_entry *mpt_entry; + int err; + + err = mlx4_mpt_alloc_icm(dev, key_to_hw_index(mw->key), GFP_KERNEL); + if (err) + return err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + err = PTR_ERR(mailbox); + goto err_table; + } + mpt_entry = mailbox->buf; + + /* Note that the MLX4_MPT_FLAG_REGION bit in mpt_entry->flags is turned + * off, thus creating a memory window and not a memory region. + */ + mpt_entry->key = cpu_to_be32(key_to_hw_index(mw->key)); + mpt_entry->pd_flags = cpu_to_be32(mw->pd); + if (mw->type == MLX4_MW_TYPE_2) { + mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_FREE); + mpt_entry->qpn = cpu_to_be32(MLX4_MPT_QP_FLAG_BOUND_QP); + mpt_entry->pd_flags |= cpu_to_be32(MLX4_MPT_PD_FLAG_EN_INV); + } + + err = mlx4_SW2HW_MPT(dev, mailbox, + key_to_hw_index(mw->key) & + (dev->caps.num_mpts - 1)); + if (err) { + mlx4_warn(dev, "SW2HW_MPT failed (%d)\n", err); + goto err_cmd; + } + mw->enabled = MLX4_MPT_EN_HW; + + mlx4_free_cmd_mailbox(dev, mailbox); + + return 0; + +err_cmd: + mlx4_free_cmd_mailbox(dev, mailbox); + +err_table: + mlx4_mpt_free_icm(dev, key_to_hw_index(mw->key)); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_mw_enable); + +void mlx4_mw_free(struct mlx4_dev *dev, struct mlx4_mw *mw) +{ + int err; + + if (mw->enabled == MLX4_MPT_EN_HW) { + err = mlx4_HW2SW_MPT(dev, NULL, + key_to_hw_index(mw->key) & + (dev->caps.num_mpts - 1)); + if (err) + mlx4_warn(dev, "xxx HW2SW_MPT failed (%d)\n", err); + + mw->enabled = MLX4_MPT_EN_SW; + } + if (mw->enabled) + mlx4_mpt_free_icm(dev, key_to_hw_index(mw->key)); + mlx4_mpt_release(dev, key_to_hw_index(mw->key)); +} +EXPORT_SYMBOL_GPL(mlx4_mw_free); + +int mlx4_init_mr_table(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_mr_table *mr_table = &priv->mr_table; + int err; + + /* Nothing to do for slaves - all MR handling is forwarded + * to the master */ + if (mlx4_is_slave(dev)) + return 0; + + if (!is_power_of_2(dev->caps.num_mpts)) + return -EINVAL; + + err = mlx4_bitmap_init(&mr_table->mpt_bitmap, dev->caps.num_mpts, + ~0, dev->caps.reserved_mrws, 0); + if (err) + return err; + + err = mlx4_buddy_init(&mr_table->mtt_buddy, + ilog2((u32)dev->caps.num_mtts / + (1 << log_mtts_per_seg))); + if (err) + goto err_buddy; + + if (dev->caps.reserved_mtts) { + priv->reserved_mtts = + mlx4_alloc_mtt_range(dev, + fls(dev->caps.reserved_mtts - 1)); + if (priv->reserved_mtts < 0) { + mlx4_warn(dev, "MTT table of order %u is too small\n", + mr_table->mtt_buddy.max_order); + err = -ENOMEM; + goto err_reserve_mtts; + } + } + + return 0; + +err_reserve_mtts: + mlx4_buddy_cleanup(&mr_table->mtt_buddy); + +err_buddy: + mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); + + return err; +} + +void mlx4_cleanup_mr_table(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_mr_table *mr_table = &priv->mr_table; + + if (mlx4_is_slave(dev)) + return; + if (priv->reserved_mtts >= 0) + mlx4_free_mtt_range(dev, priv->reserved_mtts, + fls(dev->caps.reserved_mtts - 1)); + mlx4_buddy_cleanup(&mr_table->mtt_buddy); + mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); +} + +static inline int mlx4_check_fmr(struct mlx4_fmr *fmr, u64 *page_list, + int npages, u64 iova) +{ + int i, page_mask; + + if (npages > fmr->max_pages) + return -EINVAL; + + page_mask = (1 << fmr->page_shift) - 1; + + /* We are getting page lists, so va must be page aligned. */ + if (iova & page_mask) + return -EINVAL; + + /* Trust the user not to pass misaligned data in page_list */ + if (0) + for (i = 0; i < npages; ++i) { + if (page_list[i] & ~page_mask) + return -EINVAL; + } + + if (fmr->maps >= fmr->max_maps) + return -EINVAL; + + return 0; +} + +int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list, + int npages, u64 iova, u32 *lkey, u32 *rkey) +{ + u32 key; + int i, err; + + err = mlx4_check_fmr(fmr, page_list, npages, iova); + if (err) + return err; + + ++fmr->maps; + + key = key_to_hw_index(fmr->mr.key); + key += dev->caps.num_mpts; + *lkey = *rkey = fmr->mr.key = hw_index_to_key(key); + + *(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW; + + /* Make sure MPT status is visible before writing MTT entries */ + wmb(); + + dma_sync_single_for_cpu(&dev->persist->pdev->dev, fmr->dma_handle, + npages * sizeof(u64), DMA_TO_DEVICE); + + for (i = 0; i < npages; ++i) + fmr->mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT); + + dma_sync_single_for_device(&dev->persist->pdev->dev, fmr->dma_handle, + npages * sizeof(u64), DMA_TO_DEVICE); + + fmr->mpt->key = cpu_to_be32(key); + fmr->mpt->lkey = cpu_to_be32(key); + fmr->mpt->length = cpu_to_be64(npages * (1ull << fmr->page_shift)); + fmr->mpt->start = cpu_to_be64(iova); + + /* Make MTT entries are visible before setting MPT status */ + wmb(); + + *(u8 *) fmr->mpt = MLX4_MPT_STATUS_HW; + + /* Make sure MPT status is visible before consumer can use FMR */ + wmb(); + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_map_phys_fmr); + +int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages, + int max_maps, u8 page_shift, struct mlx4_fmr *fmr) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int err = -ENOMEM; + + if (max_maps > dev->caps.max_fmr_maps) + return -EINVAL; + + if (page_shift < (ffs(dev->caps.page_size_cap) - 1) || page_shift >= 32) + return -EINVAL; + + /* All MTTs must fit in the same page */ + if (max_pages * sizeof *fmr->mtts > PAGE_SIZE) + return -EINVAL; + + fmr->page_shift = page_shift; + fmr->max_pages = max_pages; + fmr->max_maps = max_maps; + fmr->maps = 0; + + err = mlx4_mr_alloc(dev, pd, 0, 0, access, max_pages, + page_shift, &fmr->mr); + if (err) + return err; + + fmr->mtts = mlx4_table_find(&priv->mr_table.mtt_table, + fmr->mr.mtt.offset, + &fmr->dma_handle); + + if (!fmr->mtts) { + err = -ENOMEM; + goto err_free; + } + + return 0; + +err_free: + (void) mlx4_mr_free(dev, &fmr->mr); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_fmr_alloc); + +int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int err; + + err = mlx4_mr_enable(dev, &fmr->mr); + if (err) + return err; + + fmr->mpt = mlx4_table_find(&priv->mr_table.dmpt_table, + key_to_hw_index(fmr->mr.key), NULL); + if (!fmr->mpt) + return -ENOMEM; + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_fmr_enable); + +void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr, + u32 *lkey, u32 *rkey) +{ + struct mlx4_cmd_mailbox *mailbox; + int err; + + if (!fmr->maps) + return; + + fmr->maps = 0; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + err = PTR_ERR(mailbox); + pr_warn("mlx4_ib: mlx4_alloc_cmd_mailbox failed (%d)\n", err); + return; + } + + err = mlx4_HW2SW_MPT(dev, NULL, + key_to_hw_index(fmr->mr.key) & + (dev->caps.num_mpts - 1)); + mlx4_free_cmd_mailbox(dev, mailbox); + if (err) { + pr_warn("mlx4_ib: mlx4_HW2SW_MPT failed (%d)\n", err); + return; + } + fmr->mr.enabled = MLX4_MPT_EN_SW; +} +EXPORT_SYMBOL_GPL(mlx4_fmr_unmap); + +int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr) +{ + int ret; + + if (fmr->maps) + return -EBUSY; + + ret = mlx4_mr_free(dev, &fmr->mr); + if (ret) + return ret; + fmr->mr.enabled = MLX4_MPT_DISABLED; + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_fmr_free); + +int mlx4_SYNC_TPT(struct mlx4_dev *dev) +{ + return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_SYNC_TPT, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); +} +EXPORT_SYMBOL_GPL(mlx4_SYNC_TPT); Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_mr.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_pd.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_pd.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_pd.c (revision 329159) @@ -0,0 +1,294 @@ +/* + * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2005, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include + +#include + +#include "mlx4.h" +#include "icm.h" + +enum { + MLX4_NUM_RESERVED_UARS = 8 +}; + +int mlx4_pd_alloc(struct mlx4_dev *dev, u32 *pdn) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + *pdn = mlx4_bitmap_alloc(&priv->pd_bitmap); + if (*pdn == -1) + return -ENOMEM; + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_pd_alloc); + +void mlx4_pd_free(struct mlx4_dev *dev, u32 pdn) +{ + mlx4_bitmap_free(&mlx4_priv(dev)->pd_bitmap, pdn, MLX4_USE_RR); +} +EXPORT_SYMBOL_GPL(mlx4_pd_free); + +int __mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + *xrcdn = mlx4_bitmap_alloc(&priv->xrcd_bitmap); + if (*xrcdn == -1) + return -ENOMEM; + + return 0; +} + +int mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn) +{ + u64 out_param; + int err; + + if (mlx4_is_mfunc(dev)) { + err = mlx4_cmd_imm(dev, 0, &out_param, + RES_XRCD, RES_OP_RESERVE, + MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (err) + return err; + + *xrcdn = get_param_l(&out_param); + return 0; + } + return __mlx4_xrcd_alloc(dev, xrcdn); +} +EXPORT_SYMBOL_GPL(mlx4_xrcd_alloc); + +void __mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn) +{ + mlx4_bitmap_free(&mlx4_priv(dev)->xrcd_bitmap, xrcdn, MLX4_USE_RR); +} + +void mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn) +{ + u64 in_param = 0; + int err; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, xrcdn); + err = mlx4_cmd(dev, in_param, RES_XRCD, + RES_OP_RESERVE, MLX4_CMD_FREE_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (err) + mlx4_warn(dev, "Failed to release xrcdn %d\n", xrcdn); + } else + __mlx4_xrcd_free(dev, xrcdn); +} +EXPORT_SYMBOL_GPL(mlx4_xrcd_free); + +int mlx4_init_pd_table(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + return mlx4_bitmap_init(&priv->pd_bitmap, dev->caps.num_pds, + (1 << NOT_MASKED_PD_BITS) - 1, + dev->caps.reserved_pds, 0); +} + +void mlx4_cleanup_pd_table(struct mlx4_dev *dev) +{ + mlx4_bitmap_cleanup(&mlx4_priv(dev)->pd_bitmap); +} + +int mlx4_init_xrcd_table(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + return mlx4_bitmap_init(&priv->xrcd_bitmap, (1 << 16), + (1 << 16) - 1, dev->caps.reserved_xrcds + 1, 0); +} + +void mlx4_cleanup_xrcd_table(struct mlx4_dev *dev) +{ + mlx4_bitmap_cleanup(&mlx4_priv(dev)->xrcd_bitmap); +} + +int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar) +{ + int offset; + + uar->index = mlx4_bitmap_alloc(&mlx4_priv(dev)->uar_table.bitmap); + if (uar->index == -1) + return -ENOMEM; + + if (mlx4_is_slave(dev)) + offset = uar->index % ((int)pci_resource_len(dev->persist->pdev, + 2) / + dev->caps.uar_page_size); + else + offset = uar->index; + uar->pfn = (pci_resource_start(dev->persist->pdev, 2) >> PAGE_SHIFT) + + offset; + uar->map = NULL; + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_uar_alloc); + +void mlx4_uar_free(struct mlx4_dev *dev, struct mlx4_uar *uar) +{ + mlx4_bitmap_free(&mlx4_priv(dev)->uar_table.bitmap, uar->index, MLX4_USE_RR); +} +EXPORT_SYMBOL_GPL(mlx4_uar_free); + +int mlx4_bf_alloc(struct mlx4_dev *dev, struct mlx4_bf *bf, int node) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_uar *uar; + int err = 0; + int idx; + + if (!priv->bf_mapping) + return -ENOMEM; + + mutex_lock(&priv->bf_mutex); + if (!list_empty(&priv->bf_list)) + uar = list_entry(priv->bf_list.next, struct mlx4_uar, bf_list); + else { + if (mlx4_bitmap_avail(&priv->uar_table.bitmap) < MLX4_NUM_RESERVED_UARS) { + err = -ENOMEM; + goto out; + } + uar = kmalloc_node(sizeof(*uar), GFP_KERNEL, node); + if (!uar) { + uar = kmalloc(sizeof(*uar), GFP_KERNEL); + if (!uar) { + err = -ENOMEM; + goto out; + } + } + err = mlx4_uar_alloc(dev, uar); + if (err) + goto free_kmalloc; + + uar->map = ioremap(uar->pfn << PAGE_SHIFT, PAGE_SIZE); + if (!uar->map) { + err = -ENOMEM; + goto free_uar; + } + + uar->bf_map = io_mapping_map_wc(priv->bf_mapping, + uar->index << PAGE_SHIFT, + PAGE_SIZE); + if (!uar->bf_map) { + err = -ENOMEM; + goto unamp_uar; + } + uar->free_bf_bmap = 0; + list_add(&uar->bf_list, &priv->bf_list); + } + + idx = ffz(uar->free_bf_bmap); + uar->free_bf_bmap |= 1 << idx; + bf->uar = uar; + bf->offset = 0; + bf->buf_size = dev->caps.bf_reg_size / 2; + bf->reg = uar->bf_map + idx * dev->caps.bf_reg_size; + if (uar->free_bf_bmap == (1 << dev->caps.bf_regs_per_page) - 1) + list_del_init(&uar->bf_list); + + goto out; + +unamp_uar: + bf->uar = NULL; + iounmap(uar->map); + +free_uar: + mlx4_uar_free(dev, uar); + +free_kmalloc: + kfree(uar); + +out: + mutex_unlock(&priv->bf_mutex); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_bf_alloc); + +void mlx4_bf_free(struct mlx4_dev *dev, struct mlx4_bf *bf) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int idx; + + if (!bf->uar || !bf->uar->bf_map) + return; + + mutex_lock(&priv->bf_mutex); + idx = (bf->reg - bf->uar->bf_map) / dev->caps.bf_reg_size; + bf->uar->free_bf_bmap &= ~(1 << idx); + if (!bf->uar->free_bf_bmap) { + if (!list_empty(&bf->uar->bf_list)) + list_del(&bf->uar->bf_list); + + io_mapping_unmap(bf->uar->bf_map); + iounmap(bf->uar->map); + mlx4_uar_free(dev, bf->uar); + kfree(bf->uar); + } else if (list_empty(&bf->uar->bf_list)) + list_add(&bf->uar->bf_list, &priv->bf_list); + + mutex_unlock(&priv->bf_mutex); +} +EXPORT_SYMBOL_GPL(mlx4_bf_free); + +int mlx4_init_uar_table(struct mlx4_dev *dev) +{ + int num_reserved_uar = mlx4_get_num_reserved_uar(dev); + + mlx4_dbg(dev, "uar_page_shift = %d", dev->uar_page_shift); + mlx4_dbg(dev, "Effective reserved_uars=%d", dev->caps.reserved_uars); + + if (dev->caps.num_uars <= num_reserved_uar) { + mlx4_err(dev, "Only %d UAR pages (need more than %d)\n", + dev->caps.num_uars, num_reserved_uar); + mlx4_err(dev, "Increase firmware log2_uar_bar_megabytes?\n"); + return -ENODEV; + } + + return mlx4_bitmap_init(&mlx4_priv(dev)->uar_table.bitmap, + dev->caps.num_uars, dev->caps.num_uars - 1, + dev->caps.reserved_uars, 0); +} + +void mlx4_cleanup_uar_table(struct mlx4_dev *dev) +{ + mlx4_bitmap_cleanup(&mlx4_priv(dev)->uar_table.bitmap); +} Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_pd.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_port.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_port.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_port.c (revision 329159) @@ -0,0 +1,2015 @@ +/* + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#define LINUXKPI_PARAM_PREFIX mlx4_ + +#include +#include +#include +#include + +#include +#include + +#include + +#include "mlx4.h" +#include + +#define MLX4_MAC_VALID (1ull << 63) + +#define MLX4_VLAN_VALID (1u << 31) +#define MLX4_VLAN_MASK 0xfff + +#define MLX4_STATS_TRAFFIC_COUNTERS_MASK 0xfULL +#define MLX4_STATS_TRAFFIC_DROPS_MASK 0xc0ULL +#define MLX4_STATS_ERROR_COUNTERS_MASK 0x1ffc30ULL +#define MLX4_STATS_PORT_COUNTERS_MASK 0x1fe00000ULL + +#define MLX4_FLAG_V_IGNORE_FCS_MASK 0x2 +#define MLX4_IGNORE_FCS_MASK 0x1 +#define MLX4_TC_MAX_NUMBER 8 + +void mlx4_init_mac_table(struct mlx4_dev *dev, struct mlx4_mac_table *table) +{ + int i; + + mutex_init(&table->mutex); + for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { + table->entries[i] = 0; + table->refs[i] = 0; + table->is_dup[i] = false; + } + table->max = 1 << dev->caps.log_num_macs; + table->total = 0; +} + +void mlx4_init_vlan_table(struct mlx4_dev *dev, struct mlx4_vlan_table *table) +{ + int i; + + mutex_init(&table->mutex); + for (i = 0; i < MLX4_MAX_VLAN_NUM; i++) { + table->entries[i] = 0; + table->refs[i] = 0; + table->is_dup[i] = false; + } + table->max = (1 << dev->caps.log_num_vlans) - MLX4_VLAN_REGULAR; + table->total = 0; +} + +void mlx4_init_roce_gid_table(struct mlx4_dev *dev, + struct mlx4_roce_gid_table *table) +{ + int i; + + mutex_init(&table->mutex); + for (i = 0; i < MLX4_ROCE_MAX_GIDS; i++) + memset(table->roce_gids[i].raw, 0, MLX4_ROCE_GID_ENTRY_SIZE); +} + +static int validate_index(struct mlx4_dev *dev, + struct mlx4_mac_table *table, int index) +{ + int err = 0; + + if (index < 0 || index >= table->max || !table->entries[index]) { + mlx4_warn(dev, "No valid Mac entry for the given index\n"); + err = -EINVAL; + } + return err; +} + +static int find_index(struct mlx4_dev *dev, + struct mlx4_mac_table *table, u64 mac) +{ + int i; + + for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { + if (table->refs[i] && + (MLX4_MAC_MASK & mac) == + (MLX4_MAC_MASK & be64_to_cpu(table->entries[i]))) + return i; + } + /* Mac not found */ + return -EINVAL; +} + +static int mlx4_set_port_mac_table(struct mlx4_dev *dev, u8 port, + __be64 *entries) +{ + struct mlx4_cmd_mailbox *mailbox; + u32 in_mod; + int err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + memcpy(mailbox->buf, entries, MLX4_MAC_TABLE_SIZE); + + in_mod = MLX4_SET_PORT_MAC_TABLE << 8 | port; + + err = mlx4_cmd(dev, mailbox->dma, in_mod, MLX4_SET_PORT_ETH_OPCODE, + MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +int mlx4_find_cached_mac(struct mlx4_dev *dev, u8 port, u64 mac, int *idx) +{ + struct mlx4_port_info *info = &mlx4_priv(dev)->port[port]; + struct mlx4_mac_table *table = &info->mac_table; + int i; + + for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { + if (!table->refs[i]) + continue; + + if (mac == (MLX4_MAC_MASK & be64_to_cpu(table->entries[i]))) { + *idx = i; + return 0; + } + } + + return -ENOENT; +} +EXPORT_SYMBOL_GPL(mlx4_find_cached_mac); + +static bool mlx4_need_mf_bond(struct mlx4_dev *dev) +{ + int i, num_eth_ports = 0; + + if (!mlx4_is_mfunc(dev)) + return false; + mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) + ++num_eth_ports; + + return (num_eth_ports == 2) ? true : false; +} + +int __mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac) +{ + struct mlx4_port_info *info = &mlx4_priv(dev)->port[port]; + struct mlx4_mac_table *table = &info->mac_table; + int i, err = 0; + int free = -1; + int free_for_dup = -1; + bool dup = mlx4_is_mf_bonded(dev); + u8 dup_port = (port == 1) ? 2 : 1; + struct mlx4_mac_table *dup_table = &mlx4_priv(dev)->port[dup_port].mac_table; + bool need_mf_bond = mlx4_need_mf_bond(dev); + bool can_mf_bond = true; + + mlx4_dbg(dev, "Registering MAC: 0x%llx for port %d %s duplicate\n", + (unsigned long long)mac, port, + dup ? "with" : "without"); + + if (need_mf_bond) { + if (port == 1) { + mutex_lock(&table->mutex); + mutex_lock_nested(&dup_table->mutex, SINGLE_DEPTH_NESTING); + } else { + mutex_lock(&dup_table->mutex); + mutex_lock_nested(&table->mutex, SINGLE_DEPTH_NESTING); + } + } else { + mutex_lock(&table->mutex); + } + + if (need_mf_bond) { + int index_at_port = -1; + int index_at_dup_port = -1; + + for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { + if (((MLX4_MAC_MASK & mac) == (MLX4_MAC_MASK & be64_to_cpu(table->entries[i])))) + index_at_port = i; + if (((MLX4_MAC_MASK & mac) == (MLX4_MAC_MASK & be64_to_cpu(dup_table->entries[i])))) + index_at_dup_port = i; + } + + /* check that same mac is not in the tables at different indices */ + if ((index_at_port != index_at_dup_port) && + (index_at_port >= 0) && + (index_at_dup_port >= 0)) + can_mf_bond = false; + + /* If the mac is already in the primary table, the slot must be + * available in the duplicate table as well. + */ + if (index_at_port >= 0 && index_at_dup_port < 0 && + dup_table->refs[index_at_port]) { + can_mf_bond = false; + } + /* If the mac is already in the duplicate table, check that the + * corresponding index is not occupied in the primary table, or + * the primary table already contains the mac at the same index. + * Otherwise, you cannot bond (primary contains a different mac + * at that index). + */ + if (index_at_dup_port >= 0) { + if (!table->refs[index_at_dup_port] || + ((MLX4_MAC_MASK & mac) == (MLX4_MAC_MASK & be64_to_cpu(table->entries[index_at_dup_port])))) + free_for_dup = index_at_dup_port; + else + can_mf_bond = false; + } + } + + for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { + if (!table->refs[i]) { + if (free < 0) + free = i; + if (free_for_dup < 0 && need_mf_bond && can_mf_bond) { + if (!dup_table->refs[i]) + free_for_dup = i; + } + continue; + } + + if ((MLX4_MAC_MASK & mac) == + (MLX4_MAC_MASK & be64_to_cpu(table->entries[i]))) { + /* MAC already registered, increment ref count */ + err = i; + ++table->refs[i]; + if (dup) { + u64 dup_mac = MLX4_MAC_MASK & be64_to_cpu(dup_table->entries[i]); + + if (dup_mac != mac || !dup_table->is_dup[i]) { + mlx4_warn(dev, "register mac: expect duplicate mac 0x%llx on port %d index %d\n", + (long long)mac, dup_port, i); + } + } + goto out; + } + } + + if (need_mf_bond && (free_for_dup < 0)) { + if (dup) { + mlx4_warn(dev, "Fail to allocate duplicate MAC table entry\n"); + mlx4_warn(dev, "High Availability for virtual functions may not work as expected\n"); + dup = false; + } + can_mf_bond = false; + } + + if (need_mf_bond && can_mf_bond) + free = free_for_dup; + + mlx4_dbg(dev, "Free MAC index is %d\n", free); + + if (table->total == table->max) { + /* No free mac entries */ + err = -ENOSPC; + goto out; + } + + /* Register new MAC */ + table->entries[free] = cpu_to_be64(mac | MLX4_MAC_VALID); + + err = mlx4_set_port_mac_table(dev, port, table->entries); + if (unlikely(err)) { + mlx4_err(dev, "Failed adding MAC: 0x%llx\n", + (unsigned long long) mac); + table->entries[free] = 0; + goto out; + } + table->refs[free] = 1; + table->is_dup[free] = false; + ++table->total; + if (dup) { + dup_table->refs[free] = 0; + dup_table->is_dup[free] = true; + dup_table->entries[free] = cpu_to_be64(mac | MLX4_MAC_VALID); + + err = mlx4_set_port_mac_table(dev, dup_port, dup_table->entries); + if (unlikely(err)) { + mlx4_warn(dev, "Failed adding duplicate mac: 0x%llx\n", (long long)mac); + dup_table->is_dup[free] = false; + dup_table->entries[free] = 0; + goto out; + } + ++dup_table->total; + } + err = free; +out: + if (need_mf_bond) { + if (port == 2) { + mutex_unlock(&table->mutex); + mutex_unlock(&dup_table->mutex); + } else { + mutex_unlock(&dup_table->mutex); + mutex_unlock(&table->mutex); + } + } else { + mutex_unlock(&table->mutex); + } + return err; +} +EXPORT_SYMBOL_GPL(__mlx4_register_mac); + +int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac) +{ + u64 out_param = 0; + int err = -EINVAL; + + if (mlx4_is_mfunc(dev)) { + if (!(dev->flags & MLX4_FLAG_OLD_REG_MAC)) { + err = mlx4_cmd_imm(dev, mac, &out_param, + ((u32) port) << 8 | (u32) RES_MAC, + RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + } + if (err && err == -EINVAL && mlx4_is_slave(dev)) { + /* retry using old REG_MAC format */ + set_param_l(&out_param, port); + err = mlx4_cmd_imm(dev, mac, &out_param, RES_MAC, + RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (!err) + dev->flags |= MLX4_FLAG_OLD_REG_MAC; + } + if (err) + return err; + + return get_param_l(&out_param); + } + return __mlx4_register_mac(dev, port, mac); +} +EXPORT_SYMBOL_GPL(mlx4_register_mac); + +int mlx4_get_base_qpn(struct mlx4_dev *dev, u8 port) +{ + return dev->caps.reserved_qps_base[MLX4_QP_REGION_ETH_ADDR] + + (port - 1) * (1 << dev->caps.log_num_macs); +} +EXPORT_SYMBOL_GPL(mlx4_get_base_qpn); + +void __mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, u64 mac) +{ + struct mlx4_port_info *info; + struct mlx4_mac_table *table; + int index; + bool dup = mlx4_is_mf_bonded(dev); + u8 dup_port = (port == 1) ? 2 : 1; + struct mlx4_mac_table *dup_table = &mlx4_priv(dev)->port[dup_port].mac_table; + + if (port < 1 || port > dev->caps.num_ports) { + mlx4_warn(dev, "invalid port number (%d), aborting...\n", port); + return; + } + info = &mlx4_priv(dev)->port[port]; + table = &info->mac_table; + + if (dup) { + if (port == 1) { + mutex_lock(&table->mutex); + mutex_lock_nested(&dup_table->mutex, SINGLE_DEPTH_NESTING); + } else { + mutex_lock(&dup_table->mutex); + mutex_lock_nested(&table->mutex, SINGLE_DEPTH_NESTING); + } + } else { + mutex_lock(&table->mutex); + } + + index = find_index(dev, table, mac); + + if (validate_index(dev, table, index)) + goto out; + + if (--table->refs[index] || table->is_dup[index]) { + mlx4_dbg(dev, "Have more references for index %d, no need to modify mac table\n", + index); + if (!table->refs[index]) + dup_table->is_dup[index] = false; + goto out; + } + + table->entries[index] = 0; + if (mlx4_set_port_mac_table(dev, port, table->entries)) + mlx4_warn(dev, "Fail to set mac in port %d during unregister\n", port); + --table->total; + + if (dup) { + dup_table->is_dup[index] = false; + if (dup_table->refs[index]) + goto out; + dup_table->entries[index] = 0; + if (mlx4_set_port_mac_table(dev, dup_port, dup_table->entries)) + mlx4_warn(dev, "Fail to set mac in duplicate port %d during unregister\n", dup_port); + + --table->total; + } +out: + if (dup) { + if (port == 2) { + mutex_unlock(&table->mutex); + mutex_unlock(&dup_table->mutex); + } else { + mutex_unlock(&dup_table->mutex); + mutex_unlock(&table->mutex); + } + } else { + mutex_unlock(&table->mutex); + } +} +EXPORT_SYMBOL_GPL(__mlx4_unregister_mac); + +void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, u64 mac) +{ + u64 out_param = 0; + + if (mlx4_is_mfunc(dev)) { + if (!(dev->flags & MLX4_FLAG_OLD_REG_MAC)) { + (void) mlx4_cmd_imm(dev, mac, &out_param, + ((u32) port) << 8 | (u32) RES_MAC, + RES_OP_RESERVE_AND_MAP, MLX4_CMD_FREE_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + } else { + /* use old unregister mac format */ + set_param_l(&out_param, port); + (void) mlx4_cmd_imm(dev, mac, &out_param, RES_MAC, + RES_OP_RESERVE_AND_MAP, MLX4_CMD_FREE_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + } + return; + } + __mlx4_unregister_mac(dev, port, mac); + return; +} +EXPORT_SYMBOL_GPL(mlx4_unregister_mac); + +int __mlx4_replace_mac(struct mlx4_dev *dev, u8 port, int qpn, u64 new_mac) +{ + struct mlx4_port_info *info = &mlx4_priv(dev)->port[port]; + struct mlx4_mac_table *table = &info->mac_table; + int index = qpn - info->base_qpn; + int err = 0; + bool dup = mlx4_is_mf_bonded(dev); + u8 dup_port = (port == 1) ? 2 : 1; + struct mlx4_mac_table *dup_table = &mlx4_priv(dev)->port[dup_port].mac_table; + + /* CX1 doesn't support multi-functions */ + if (dup) { + if (port == 1) { + mutex_lock(&table->mutex); + mutex_lock_nested(&dup_table->mutex, SINGLE_DEPTH_NESTING); + } else { + mutex_lock(&dup_table->mutex); + mutex_lock_nested(&table->mutex, SINGLE_DEPTH_NESTING); + } + } else { + mutex_lock(&table->mutex); + } + + err = validate_index(dev, table, index); + if (err) + goto out; + + table->entries[index] = cpu_to_be64(new_mac | MLX4_MAC_VALID); + + err = mlx4_set_port_mac_table(dev, port, table->entries); + if (unlikely(err)) { + mlx4_err(dev, "Failed adding MAC: 0x%llx\n", + (unsigned long long) new_mac); + table->entries[index] = 0; + } else { + if (dup) { + dup_table->entries[index] = cpu_to_be64(new_mac | MLX4_MAC_VALID); + + err = mlx4_set_port_mac_table(dev, dup_port, dup_table->entries); + if (unlikely(err)) { + mlx4_err(dev, "Failed adding duplicate MAC: 0x%llx\n", + (unsigned long long)new_mac); + dup_table->entries[index] = 0; + } + } + } +out: + if (dup) { + if (port == 2) { + mutex_unlock(&table->mutex); + mutex_unlock(&dup_table->mutex); + } else { + mutex_unlock(&dup_table->mutex); + mutex_unlock(&table->mutex); + } + } else { + mutex_unlock(&table->mutex); + } + return err; +} +EXPORT_SYMBOL_GPL(__mlx4_replace_mac); + +static int mlx4_set_port_vlan_table(struct mlx4_dev *dev, u8 port, + __be32 *entries) +{ + struct mlx4_cmd_mailbox *mailbox; + u32 in_mod; + int err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + memcpy(mailbox->buf, entries, MLX4_VLAN_TABLE_SIZE); + in_mod = MLX4_SET_PORT_VLAN_TABLE << 8 | port; + err = mlx4_cmd(dev, mailbox->dma, in_mod, MLX4_SET_PORT_ETH_OPCODE, + MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, mailbox); + + return err; +} + +int mlx4_find_cached_vlan(struct mlx4_dev *dev, u8 port, u16 vid, int *idx) +{ + struct mlx4_vlan_table *table = &mlx4_priv(dev)->port[port].vlan_table; + int i; + + for (i = 0; i < MLX4_MAX_VLAN_NUM; ++i) { + if (table->refs[i] && + (vid == (MLX4_VLAN_MASK & + be32_to_cpu(table->entries[i])))) { + /* VLAN already registered, increase reference count */ + *idx = i; + return 0; + } + } + + return -ENOENT; +} +EXPORT_SYMBOL_GPL(mlx4_find_cached_vlan); + +int __mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, + int *index) +{ + struct mlx4_vlan_table *table = &mlx4_priv(dev)->port[port].vlan_table; + int i, err = 0; + int free = -1; + int free_for_dup = -1; + bool dup = mlx4_is_mf_bonded(dev); + u8 dup_port = (port == 1) ? 2 : 1; + struct mlx4_vlan_table *dup_table = &mlx4_priv(dev)->port[dup_port].vlan_table; + bool need_mf_bond = mlx4_need_mf_bond(dev); + bool can_mf_bond = true; + + mlx4_dbg(dev, "Registering VLAN: %d for port %d %s duplicate\n", + vlan, port, + dup ? "with" : "without"); + + if (need_mf_bond) { + if (port == 1) { + mutex_lock(&table->mutex); + mutex_lock_nested(&dup_table->mutex, SINGLE_DEPTH_NESTING); + } else { + mutex_lock(&dup_table->mutex); + mutex_lock_nested(&table->mutex, SINGLE_DEPTH_NESTING); + } + } else { + mutex_lock(&table->mutex); + } + + if (table->total == table->max) { + /* No free vlan entries */ + err = -ENOSPC; + goto out; + } + + if (need_mf_bond) { + int index_at_port = -1; + int index_at_dup_port = -1; + + for (i = MLX4_VLAN_REGULAR; i < MLX4_MAX_VLAN_NUM; i++) { + if (vlan == (MLX4_VLAN_MASK & be32_to_cpu(table->entries[i]))) + index_at_port = i; + if (vlan == (MLX4_VLAN_MASK & be32_to_cpu(dup_table->entries[i]))) + index_at_dup_port = i; + } + /* check that same vlan is not in the tables at different indices */ + if ((index_at_port != index_at_dup_port) && + (index_at_port >= 0) && + (index_at_dup_port >= 0)) + can_mf_bond = false; + + /* If the vlan is already in the primary table, the slot must be + * available in the duplicate table as well. + */ + if (index_at_port >= 0 && index_at_dup_port < 0 && + dup_table->refs[index_at_port]) { + can_mf_bond = false; + } + /* If the vlan is already in the duplicate table, check that the + * corresponding index is not occupied in the primary table, or + * the primary table already contains the vlan at the same index. + * Otherwise, you cannot bond (primary contains a different vlan + * at that index). + */ + if (index_at_dup_port >= 0) { + if (!table->refs[index_at_dup_port] || + (vlan == (MLX4_VLAN_MASK & be32_to_cpu(dup_table->entries[index_at_dup_port])))) + free_for_dup = index_at_dup_port; + else + can_mf_bond = false; + } + } + + for (i = MLX4_VLAN_REGULAR; i < MLX4_MAX_VLAN_NUM; i++) { + if (!table->refs[i]) { + if (free < 0) + free = i; + if (free_for_dup < 0 && need_mf_bond && can_mf_bond) { + if (!dup_table->refs[i]) + free_for_dup = i; + } + } + + if ((table->refs[i] || table->is_dup[i]) && + (vlan == (MLX4_VLAN_MASK & + be32_to_cpu(table->entries[i])))) { + /* Vlan already registered, increase references count */ + mlx4_dbg(dev, "vlan %u is already registered.\n", vlan); + *index = i; + ++table->refs[i]; + if (dup) { + u16 dup_vlan = MLX4_VLAN_MASK & be32_to_cpu(dup_table->entries[i]); + + if (dup_vlan != vlan || !dup_table->is_dup[i]) { + mlx4_warn(dev, "register vlan: expected duplicate vlan %u on port %d index %d\n", + vlan, dup_port, i); + } + } + goto out; + } + } + + if (need_mf_bond && (free_for_dup < 0)) { + if (dup) { + mlx4_warn(dev, "Fail to allocate duplicate VLAN table entry\n"); + mlx4_warn(dev, "High Availability for virtual functions may not work as expected\n"); + dup = false; + } + can_mf_bond = false; + } + + if (need_mf_bond && can_mf_bond) + free = free_for_dup; + + if (free < 0) { + err = -ENOMEM; + goto out; + } + + /* Register new VLAN */ + table->refs[free] = 1; + table->is_dup[free] = false; + table->entries[free] = cpu_to_be32(vlan | MLX4_VLAN_VALID); + + err = mlx4_set_port_vlan_table(dev, port, table->entries); + if (unlikely(err)) { + mlx4_warn(dev, "Failed adding vlan: %u\n", vlan); + table->refs[free] = 0; + table->entries[free] = 0; + goto out; + } + ++table->total; + if (dup) { + dup_table->refs[free] = 0; + dup_table->is_dup[free] = true; + dup_table->entries[free] = cpu_to_be32(vlan | MLX4_VLAN_VALID); + + err = mlx4_set_port_vlan_table(dev, dup_port, dup_table->entries); + if (unlikely(err)) { + mlx4_warn(dev, "Failed adding duplicate vlan: %u\n", vlan); + dup_table->is_dup[free] = false; + dup_table->entries[free] = 0; + goto out; + } + ++dup_table->total; + } + + *index = free; +out: + if (need_mf_bond) { + if (port == 2) { + mutex_unlock(&table->mutex); + mutex_unlock(&dup_table->mutex); + } else { + mutex_unlock(&dup_table->mutex); + mutex_unlock(&table->mutex); + } + } else { + mutex_unlock(&table->mutex); + } + return err; +} + +int mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, int *index) +{ + u64 out_param = 0; + int err; + + if (vlan > 4095) + return -EINVAL; + + if (mlx4_is_mfunc(dev)) { + err = mlx4_cmd_imm(dev, vlan, &out_param, + ((u32) port) << 8 | (u32) RES_VLAN, + RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (!err) + *index = get_param_l(&out_param); + + return err; + } + return __mlx4_register_vlan(dev, port, vlan, index); +} +EXPORT_SYMBOL_GPL(mlx4_register_vlan); + +void __mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, u16 vlan) +{ + struct mlx4_vlan_table *table = &mlx4_priv(dev)->port[port].vlan_table; + int index; + bool dup = mlx4_is_mf_bonded(dev); + u8 dup_port = (port == 1) ? 2 : 1; + struct mlx4_vlan_table *dup_table = &mlx4_priv(dev)->port[dup_port].vlan_table; + + if (dup) { + if (port == 1) { + mutex_lock(&table->mutex); + mutex_lock_nested(&dup_table->mutex, SINGLE_DEPTH_NESTING); + } else { + mutex_lock(&dup_table->mutex); + mutex_lock_nested(&table->mutex, SINGLE_DEPTH_NESTING); + } + } else { + mutex_lock(&table->mutex); + } + + if (mlx4_find_cached_vlan(dev, port, vlan, &index)) { + mlx4_warn(dev, "vlan 0x%x is not in the vlan table\n", vlan); + goto out; + } + + if (index < MLX4_VLAN_REGULAR) { + mlx4_warn(dev, "Trying to free special vlan index %d\n", index); + goto out; + } + + if (--table->refs[index] || table->is_dup[index]) { + mlx4_dbg(dev, "Have %d more references for index %d, no need to modify vlan table\n", + table->refs[index], index); + if (!table->refs[index]) + dup_table->is_dup[index] = false; + goto out; + } + table->entries[index] = 0; + if (mlx4_set_port_vlan_table(dev, port, table->entries)) + mlx4_warn(dev, "Fail to set vlan in port %d during unregister\n", port); + --table->total; + if (dup) { + dup_table->is_dup[index] = false; + if (dup_table->refs[index]) + goto out; + dup_table->entries[index] = 0; + if (mlx4_set_port_vlan_table(dev, dup_port, dup_table->entries)) + mlx4_warn(dev, "Fail to set vlan in duplicate port %d during unregister\n", dup_port); + --dup_table->total; + } +out: + if (dup) { + if (port == 2) { + mutex_unlock(&table->mutex); + mutex_unlock(&dup_table->mutex); + } else { + mutex_unlock(&dup_table->mutex); + mutex_unlock(&table->mutex); + } + } else { + mutex_unlock(&table->mutex); + } +} + +void mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, u16 vlan) +{ + u64 out_param = 0; + + if (mlx4_is_mfunc(dev)) { + (void) mlx4_cmd_imm(dev, vlan, &out_param, + ((u32) port) << 8 | (u32) RES_VLAN, + RES_OP_RESERVE_AND_MAP, + MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); + return; + } + __mlx4_unregister_vlan(dev, port, vlan); +} +EXPORT_SYMBOL_GPL(mlx4_unregister_vlan); + +int mlx4_bond_mac_table(struct mlx4_dev *dev) +{ + struct mlx4_mac_table *t1 = &mlx4_priv(dev)->port[1].mac_table; + struct mlx4_mac_table *t2 = &mlx4_priv(dev)->port[2].mac_table; + int ret = 0; + int i; + bool update1 = false; + bool update2 = false; + + mutex_lock(&t1->mutex); + mutex_lock(&t2->mutex); + for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { + if ((t1->entries[i] != t2->entries[i]) && + t1->entries[i] && t2->entries[i]) { + mlx4_warn(dev, "can't duplicate entry %d in mac table\n", i); + ret = -EINVAL; + goto unlock; + } + } + + for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { + if (t1->entries[i] && !t2->entries[i]) { + t2->entries[i] = t1->entries[i]; + t2->is_dup[i] = true; + update2 = true; + } else if (!t1->entries[i] && t2->entries[i]) { + t1->entries[i] = t2->entries[i]; + t1->is_dup[i] = true; + update1 = true; + } else if (t1->entries[i] && t2->entries[i]) { + t1->is_dup[i] = true; + t2->is_dup[i] = true; + } + } + + if (update1) { + ret = mlx4_set_port_mac_table(dev, 1, t1->entries); + if (ret) + mlx4_warn(dev, "failed to set MAC table for port 1 (%d)\n", ret); + } + if (!ret && update2) { + ret = mlx4_set_port_mac_table(dev, 2, t2->entries); + if (ret) + mlx4_warn(dev, "failed to set MAC table for port 2 (%d)\n", ret); + } + + if (ret) + mlx4_warn(dev, "failed to create mirror MAC tables\n"); +unlock: + mutex_unlock(&t2->mutex); + mutex_unlock(&t1->mutex); + return ret; +} + +int mlx4_unbond_mac_table(struct mlx4_dev *dev) +{ + struct mlx4_mac_table *t1 = &mlx4_priv(dev)->port[1].mac_table; + struct mlx4_mac_table *t2 = &mlx4_priv(dev)->port[2].mac_table; + int ret = 0; + int ret1; + int i; + bool update1 = false; + bool update2 = false; + + mutex_lock(&t1->mutex); + mutex_lock(&t2->mutex); + for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { + if (t1->entries[i] != t2->entries[i]) { + mlx4_warn(dev, "mac table is in an unexpected state when trying to unbond\n"); + ret = -EINVAL; + goto unlock; + } + } + + for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { + if (!t1->entries[i]) + continue; + t1->is_dup[i] = false; + if (!t1->refs[i]) { + t1->entries[i] = 0; + update1 = true; + } + t2->is_dup[i] = false; + if (!t2->refs[i]) { + t2->entries[i] = 0; + update2 = true; + } + } + + if (update1) { + ret = mlx4_set_port_mac_table(dev, 1, t1->entries); + if (ret) + mlx4_warn(dev, "failed to unmirror MAC tables for port 1(%d)\n", ret); + } + if (update2) { + ret1 = mlx4_set_port_mac_table(dev, 2, t2->entries); + if (ret1) { + mlx4_warn(dev, "failed to unmirror MAC tables for port 2(%d)\n", ret1); + ret = ret1; + } + } +unlock: + mutex_unlock(&t2->mutex); + mutex_unlock(&t1->mutex); + return ret; +} + +int mlx4_bond_vlan_table(struct mlx4_dev *dev) +{ + struct mlx4_vlan_table *t1 = &mlx4_priv(dev)->port[1].vlan_table; + struct mlx4_vlan_table *t2 = &mlx4_priv(dev)->port[2].vlan_table; + int ret = 0; + int i; + bool update1 = false; + bool update2 = false; + + mutex_lock(&t1->mutex); + mutex_lock(&t2->mutex); + for (i = 0; i < MLX4_MAX_VLAN_NUM; i++) { + if ((t1->entries[i] != t2->entries[i]) && + t1->entries[i] && t2->entries[i]) { + mlx4_warn(dev, "can't duplicate entry %d in vlan table\n", i); + ret = -EINVAL; + goto unlock; + } + } + + for (i = 0; i < MLX4_MAX_VLAN_NUM; i++) { + if (t1->entries[i] && !t2->entries[i]) { + t2->entries[i] = t1->entries[i]; + t2->is_dup[i] = true; + update2 = true; + } else if (!t1->entries[i] && t2->entries[i]) { + t1->entries[i] = t2->entries[i]; + t1->is_dup[i] = true; + update1 = true; + } else if (t1->entries[i] && t2->entries[i]) { + t1->is_dup[i] = true; + t2->is_dup[i] = true; + } + } + + if (update1) { + ret = mlx4_set_port_vlan_table(dev, 1, t1->entries); + if (ret) + mlx4_warn(dev, "failed to set VLAN table for port 1 (%d)\n", ret); + } + if (!ret && update2) { + ret = mlx4_set_port_vlan_table(dev, 2, t2->entries); + if (ret) + mlx4_warn(dev, "failed to set VLAN table for port 2 (%d)\n", ret); + } + + if (ret) + mlx4_warn(dev, "failed to create mirror VLAN tables\n"); +unlock: + mutex_unlock(&t2->mutex); + mutex_unlock(&t1->mutex); + return ret; +} + +int mlx4_unbond_vlan_table(struct mlx4_dev *dev) +{ + struct mlx4_vlan_table *t1 = &mlx4_priv(dev)->port[1].vlan_table; + struct mlx4_vlan_table *t2 = &mlx4_priv(dev)->port[2].vlan_table; + int ret = 0; + int ret1; + int i; + bool update1 = false; + bool update2 = false; + + mutex_lock(&t1->mutex); + mutex_lock(&t2->mutex); + for (i = 0; i < MLX4_MAX_VLAN_NUM; i++) { + if (t1->entries[i] != t2->entries[i]) { + mlx4_warn(dev, "vlan table is in an unexpected state when trying to unbond\n"); + ret = -EINVAL; + goto unlock; + } + } + + for (i = 0; i < MLX4_MAX_VLAN_NUM; i++) { + if (!t1->entries[i]) + continue; + t1->is_dup[i] = false; + if (!t1->refs[i]) { + t1->entries[i] = 0; + update1 = true; + } + t2->is_dup[i] = false; + if (!t2->refs[i]) { + t2->entries[i] = 0; + update2 = true; + } + } + + if (update1) { + ret = mlx4_set_port_vlan_table(dev, 1, t1->entries); + if (ret) + mlx4_warn(dev, "failed to unmirror VLAN tables for port 1(%d)\n", ret); + } + if (update2) { + ret1 = mlx4_set_port_vlan_table(dev, 2, t2->entries); + if (ret1) { + mlx4_warn(dev, "failed to unmirror VLAN tables for port 2(%d)\n", ret1); + ret = ret1; + } + } +unlock: + mutex_unlock(&t2->mutex); + mutex_unlock(&t1->mutex); + return ret; +} + +int mlx4_get_port_ib_caps(struct mlx4_dev *dev, u8 port, __be32 *caps) +{ + struct mlx4_cmd_mailbox *inmailbox, *outmailbox; + u8 *inbuf, *outbuf; + int err; + + inmailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(inmailbox)) + return PTR_ERR(inmailbox); + + outmailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(outmailbox)) { + mlx4_free_cmd_mailbox(dev, inmailbox); + return PTR_ERR(outmailbox); + } + + inbuf = inmailbox->buf; + outbuf = outmailbox->buf; + inbuf[0] = 1; + inbuf[1] = 1; + inbuf[2] = 1; + inbuf[3] = 1; + *(__be16 *) (&inbuf[16]) = cpu_to_be16(0x0015); + *(__be32 *) (&inbuf[20]) = cpu_to_be32(port); + + err = mlx4_cmd_box(dev, inmailbox->dma, outmailbox->dma, port, 3, + MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C, + MLX4_CMD_NATIVE); + if (!err) + *caps = *(__be32 *) (outbuf + 84); + mlx4_free_cmd_mailbox(dev, inmailbox); + mlx4_free_cmd_mailbox(dev, outmailbox); + return err; +} +static struct mlx4_roce_gid_entry zgid_entry; + +int mlx4_get_slave_num_gids(struct mlx4_dev *dev, int slave, int port) +{ + int vfs; + int slave_gid = slave; + unsigned i; + struct mlx4_slaves_pport slaves_pport; + struct mlx4_active_ports actv_ports; + unsigned max_port_p_one; + + if (slave == 0) + return MLX4_ROCE_PF_GIDS; + + /* Slave is a VF */ + slaves_pport = mlx4_phys_to_slaves_pport(dev, port); + actv_ports = mlx4_get_active_ports(dev, slave); + max_port_p_one = find_first_bit(actv_ports.ports, dev->caps.num_ports) + + bitmap_weight(actv_ports.ports, dev->caps.num_ports) + 1; + + for (i = 1; i < max_port_p_one; i++) { + struct mlx4_active_ports exclusive_ports; + struct mlx4_slaves_pport slaves_pport_actv; + bitmap_zero(exclusive_ports.ports, dev->caps.num_ports); + set_bit(i - 1, exclusive_ports.ports); + if (i == port) + continue; + slaves_pport_actv = mlx4_phys_to_slaves_pport_actv( + dev, &exclusive_ports); + slave_gid -= bitmap_weight(slaves_pport_actv.slaves, + dev->persist->num_vfs + 1); + } + vfs = bitmap_weight(slaves_pport.slaves, dev->persist->num_vfs + 1) - 1; + if (slave_gid <= ((MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS) % vfs)) + return ((MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS) / vfs) + 1; + return (MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS) / vfs; +} + +int mlx4_get_base_gid_ix(struct mlx4_dev *dev, int slave, int port) +{ + int gids; + unsigned i; + int slave_gid = slave; + int vfs; + + struct mlx4_slaves_pport slaves_pport; + struct mlx4_active_ports actv_ports; + unsigned max_port_p_one; + + if (slave == 0) + return 0; + + slaves_pport = mlx4_phys_to_slaves_pport(dev, port); + actv_ports = mlx4_get_active_ports(dev, slave); + max_port_p_one = find_first_bit(actv_ports.ports, dev->caps.num_ports) + + bitmap_weight(actv_ports.ports, dev->caps.num_ports) + 1; + + for (i = 1; i < max_port_p_one; i++) { + struct mlx4_active_ports exclusive_ports; + struct mlx4_slaves_pport slaves_pport_actv; + bitmap_zero(exclusive_ports.ports, dev->caps.num_ports); + set_bit(i - 1, exclusive_ports.ports); + if (i == port) + continue; + slaves_pport_actv = mlx4_phys_to_slaves_pport_actv( + dev, &exclusive_ports); + slave_gid -= bitmap_weight(slaves_pport_actv.slaves, + dev->persist->num_vfs + 1); + } + gids = MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS; + vfs = bitmap_weight(slaves_pport.slaves, dev->persist->num_vfs + 1) - 1; + if (slave_gid <= gids % vfs) + return MLX4_ROCE_PF_GIDS + ((gids / vfs) + 1) * (slave_gid - 1); + + return MLX4_ROCE_PF_GIDS + (gids % vfs) + + ((gids / vfs) * (slave_gid - 1)); +} +EXPORT_SYMBOL_GPL(mlx4_get_base_gid_ix); + +static int mlx4_reset_roce_port_gids(struct mlx4_dev *dev, int slave, + int port, struct mlx4_cmd_mailbox *mailbox) +{ + struct mlx4_roce_gid_entry *gid_entry_mbox; + struct mlx4_priv *priv = mlx4_priv(dev); + int num_gids, base, offset; + int i, err; + + num_gids = mlx4_get_slave_num_gids(dev, slave, port); + base = mlx4_get_base_gid_ix(dev, slave, port); + + memset(mailbox->buf, 0, MLX4_MAILBOX_SIZE); + + mutex_lock(&(priv->port[port].gid_table.mutex)); + /* Zero-out gids belonging to that slave in the port GID table */ + for (i = 0, offset = base; i < num_gids; offset++, i++) + memcpy(priv->port[port].gid_table.roce_gids[offset].raw, + zgid_entry.raw, MLX4_ROCE_GID_ENTRY_SIZE); + + /* Now, copy roce port gids table to mailbox for passing to FW */ + gid_entry_mbox = (struct mlx4_roce_gid_entry *)mailbox->buf; + for (i = 0; i < MLX4_ROCE_MAX_GIDS; gid_entry_mbox++, i++) + memcpy(gid_entry_mbox->raw, + priv->port[port].gid_table.roce_gids[i].raw, + MLX4_ROCE_GID_ENTRY_SIZE); + + err = mlx4_cmd(dev, mailbox->dma, + ((u32)port) | (MLX4_SET_PORT_GID_TABLE << 8), + MLX4_SET_PORT_ETH_OPCODE, MLX4_CMD_SET_PORT, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); + mutex_unlock(&(priv->port[port].gid_table.mutex)); + return err; +} + + +void mlx4_reset_roce_gids(struct mlx4_dev *dev, int slave) +{ + struct mlx4_active_ports actv_ports; + struct mlx4_cmd_mailbox *mailbox; + int num_eth_ports, err; + int i; + + if (slave < 0 || slave > dev->persist->num_vfs) + return; + + actv_ports = mlx4_get_active_ports(dev, slave); + + for (i = 0, num_eth_ports = 0; i < dev->caps.num_ports; i++) { + if (test_bit(i, actv_ports.ports)) { + if (dev->caps.port_type[i + 1] != MLX4_PORT_TYPE_ETH) + continue; + num_eth_ports++; + } + } + + if (!num_eth_ports) + return; + + /* have ETH ports. Alloc mailbox for SET_PORT command */ + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return; + + for (i = 0; i < dev->caps.num_ports; i++) { + if (test_bit(i, actv_ports.ports)) { + if (dev->caps.port_type[i + 1] != MLX4_PORT_TYPE_ETH) + continue; + err = mlx4_reset_roce_port_gids(dev, slave, i + 1, mailbox); + if (err) + mlx4_warn(dev, "Could not reset ETH port GID table for slave %d, port %d (%d)\n", + slave, i + 1, err); + } + } + + mlx4_free_cmd_mailbox(dev, mailbox); + return; +} + +static int mlx4_common_set_port(struct mlx4_dev *dev, int slave, u32 in_mod, + u8 op_mod, struct mlx4_cmd_mailbox *inbox) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_port_info *port_info; + struct mlx4_mfunc_master_ctx *master = &priv->mfunc.master; + struct mlx4_slave_state *slave_st = &master->slave_state[slave]; + struct mlx4_set_port_rqp_calc_context *qpn_context; + struct mlx4_set_port_general_context *gen_context; + struct mlx4_roce_gid_entry *gid_entry_tbl, *gid_entry_mbox, *gid_entry_mb1; + int reset_qkey_viols; + int port; + int is_eth; + int num_gids; + int base; + u32 in_modifier; + u32 promisc; + u16 mtu, prev_mtu; + int err; + int i, j; + int offset; + __be32 agg_cap_mask; + __be32 slave_cap_mask; + __be32 new_cap_mask; + + port = in_mod & 0xff; + in_modifier = in_mod >> 8; + is_eth = op_mod; + port_info = &priv->port[port]; + + /* Slaves cannot perform SET_PORT operations except changing MTU */ + if (is_eth) { + if (slave != dev->caps.function && + in_modifier != MLX4_SET_PORT_GENERAL && + in_modifier != MLX4_SET_PORT_GID_TABLE) { + mlx4_warn(dev, "denying SET_PORT for slave:%d\n", + slave); + return -EINVAL; + } + switch (in_modifier) { + case MLX4_SET_PORT_RQP_CALC: + qpn_context = inbox->buf; + qpn_context->base_qpn = + cpu_to_be32(port_info->base_qpn); + qpn_context->n_mac = 0x7; + promisc = be32_to_cpu(qpn_context->promisc) >> + SET_PORT_PROMISC_SHIFT; + qpn_context->promisc = cpu_to_be32( + promisc << SET_PORT_PROMISC_SHIFT | + port_info->base_qpn); + promisc = be32_to_cpu(qpn_context->mcast) >> + SET_PORT_MC_PROMISC_SHIFT; + qpn_context->mcast = cpu_to_be32( + promisc << SET_PORT_MC_PROMISC_SHIFT | + port_info->base_qpn); + break; + case MLX4_SET_PORT_GENERAL: + gen_context = inbox->buf; + /* Mtu is configured as the max MTU among all the + * the functions on the port. */ + mtu = be16_to_cpu(gen_context->mtu); + mtu = min_t(int, mtu, dev->caps.eth_mtu_cap[port] + + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN); + prev_mtu = slave_st->mtu[port]; + slave_st->mtu[port] = mtu; + if (mtu > master->max_mtu[port]) + master->max_mtu[port] = mtu; + if (mtu < prev_mtu && prev_mtu == + master->max_mtu[port]) { + slave_st->mtu[port] = mtu; + master->max_mtu[port] = mtu; + for (i = 0; i < dev->num_slaves; i++) { + master->max_mtu[port] = + max(master->max_mtu[port], + master->slave_state[i].mtu[port]); + } + } + + gen_context->mtu = cpu_to_be16(master->max_mtu[port]); + /* Slave cannot change Global Pause configuration */ + if (slave != mlx4_master_func_num(dev) && + ((gen_context->pptx != master->pptx) || + (gen_context->pprx != master->pprx))) { + gen_context->pptx = master->pptx; + gen_context->pprx = master->pprx; + mlx4_warn(dev, + "denying Global Pause change for slave:%d\n", + slave); + } else { + master->pptx = gen_context->pptx; + master->pprx = gen_context->pprx; + } + break; + case MLX4_SET_PORT_GID_TABLE: + /* change to MULTIPLE entries: number of guest's gids + * need a FOR-loop here over number of gids the guest has. + * 1. Check no duplicates in gids passed by slave + */ + num_gids = mlx4_get_slave_num_gids(dev, slave, port); + base = mlx4_get_base_gid_ix(dev, slave, port); + gid_entry_mbox = (struct mlx4_roce_gid_entry *)(inbox->buf); + for (i = 0; i < num_gids; gid_entry_mbox++, i++) { + if (!memcmp(gid_entry_mbox->raw, zgid_entry.raw, + sizeof(zgid_entry))) + continue; + gid_entry_mb1 = gid_entry_mbox + 1; + for (j = i + 1; j < num_gids; gid_entry_mb1++, j++) { + if (!memcmp(gid_entry_mb1->raw, + zgid_entry.raw, sizeof(zgid_entry))) + continue; + if (!memcmp(gid_entry_mb1->raw, gid_entry_mbox->raw, + sizeof(gid_entry_mbox->raw))) { + /* found duplicate */ + return -EINVAL; + } + } + } + + /* 2. Check that do not have duplicates in OTHER + * entries in the port GID table + */ + + mutex_lock(&(priv->port[port].gid_table.mutex)); + for (i = 0; i < MLX4_ROCE_MAX_GIDS; i++) { + if (i >= base && i < base + num_gids) + continue; /* don't compare to slave's current gids */ + gid_entry_tbl = &priv->port[port].gid_table.roce_gids[i]; + if (!memcmp(gid_entry_tbl->raw, zgid_entry.raw, sizeof(zgid_entry))) + continue; + gid_entry_mbox = (struct mlx4_roce_gid_entry *)(inbox->buf); + for (j = 0; j < num_gids; gid_entry_mbox++, j++) { + if (!memcmp(gid_entry_mbox->raw, zgid_entry.raw, + sizeof(zgid_entry))) + continue; + if (!memcmp(gid_entry_mbox->raw, gid_entry_tbl->raw, + sizeof(gid_entry_tbl->raw))) { + /* found duplicate */ + mlx4_warn(dev, "requested gid entry for slave:%d is a duplicate of gid at index %d\n", + slave, i); + mutex_unlock(&(priv->port[port].gid_table.mutex)); + return -EINVAL; + } + } + } + + /* insert slave GIDs with memcpy, starting at slave's base index */ + gid_entry_mbox = (struct mlx4_roce_gid_entry *)(inbox->buf); + for (i = 0, offset = base; i < num_gids; gid_entry_mbox++, offset++, i++) + memcpy(priv->port[port].gid_table.roce_gids[offset].raw, + gid_entry_mbox->raw, MLX4_ROCE_GID_ENTRY_SIZE); + + /* Now, copy roce port gids table to current mailbox for passing to FW */ + gid_entry_mbox = (struct mlx4_roce_gid_entry *)(inbox->buf); + for (i = 0; i < MLX4_ROCE_MAX_GIDS; gid_entry_mbox++, i++) + memcpy(gid_entry_mbox->raw, + priv->port[port].gid_table.roce_gids[i].raw, + MLX4_ROCE_GID_ENTRY_SIZE); + + err = mlx4_cmd(dev, inbox->dma, in_mod & 0xffff, op_mod, + MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); + mutex_unlock(&(priv->port[port].gid_table.mutex)); + return err; + } + + return mlx4_cmd(dev, inbox->dma, in_mod & 0xffff, op_mod, + MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); + } + + /* Slaves are not allowed to SET_PORT beacon (LED) blink */ + if (op_mod == MLX4_SET_PORT_BEACON_OPCODE) { + mlx4_warn(dev, "denying SET_PORT Beacon slave:%d\n", slave); + return -EPERM; + } + + /* For IB, we only consider: + * - The capability mask, which is set to the aggregate of all + * slave function capabilities + * - The QKey violatin counter - reset according to each request. + */ + + if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { + reset_qkey_viols = (*(u8 *) inbox->buf) & 0x40; + new_cap_mask = ((__be32 *) inbox->buf)[2]; + } else { + reset_qkey_viols = ((u8 *) inbox->buf)[3] & 0x1; + new_cap_mask = ((__be32 *) inbox->buf)[1]; + } + + /* slave may not set the IS_SM capability for the port */ + if (slave != mlx4_master_func_num(dev) && + (be32_to_cpu(new_cap_mask) & MLX4_PORT_CAP_IS_SM)) + return -EINVAL; + + /* No DEV_MGMT in multifunc mode */ + if (mlx4_is_mfunc(dev) && + (be32_to_cpu(new_cap_mask) & MLX4_PORT_CAP_DEV_MGMT_SUP)) + return -EINVAL; + + agg_cap_mask = 0; + slave_cap_mask = + priv->mfunc.master.slave_state[slave].ib_cap_mask[port]; + priv->mfunc.master.slave_state[slave].ib_cap_mask[port] = new_cap_mask; + for (i = 0; i < dev->num_slaves; i++) + agg_cap_mask |= + priv->mfunc.master.slave_state[i].ib_cap_mask[port]; + + /* only clear mailbox for guests. Master may be setting + * MTU or PKEY table size + */ + if (slave != dev->caps.function) + memset(inbox->buf, 0, 256); + if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { + *(u8 *) inbox->buf |= !!reset_qkey_viols << 6; + ((__be32 *) inbox->buf)[2] = agg_cap_mask; + } else { + ((u8 *) inbox->buf)[3] |= !!reset_qkey_viols; + ((__be32 *) inbox->buf)[1] = agg_cap_mask; + } + + err = mlx4_cmd(dev, inbox->dma, port, is_eth, MLX4_CMD_SET_PORT, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); + if (err) + priv->mfunc.master.slave_state[slave].ib_cap_mask[port] = + slave_cap_mask; + return err; +} + +int mlx4_SET_PORT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int port = mlx4_slave_convert_port( + dev, slave, vhcr->in_modifier & 0xFF); + + if (port < 0) + return -EINVAL; + + vhcr->in_modifier = (vhcr->in_modifier & ~0xFF) | + (port & 0xFF); + + return mlx4_common_set_port(dev, slave, vhcr->in_modifier, + vhcr->op_modifier, inbox); +} + +/* bit locations for set port command with zero op modifier */ +enum { + MLX4_SET_PORT_VL_CAP = 4, /* bits 7:4 */ + MLX4_SET_PORT_MTU_CAP = 12, /* bits 15:12 */ + MLX4_CHANGE_PORT_PKEY_TBL_SZ = 20, + MLX4_CHANGE_PORT_VL_CAP = 21, + MLX4_CHANGE_PORT_MTU_CAP = 22, +}; + +int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port, int pkey_tbl_sz) +{ + struct mlx4_cmd_mailbox *mailbox; + int err, vl_cap, pkey_tbl_flag = 0; + + if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) + return 0; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + ((__be32 *) mailbox->buf)[1] = dev->caps.ib_port_def_cap[port]; + + if (pkey_tbl_sz >= 0 && mlx4_is_master(dev)) { + pkey_tbl_flag = 1; + ((__be16 *) mailbox->buf)[20] = cpu_to_be16(pkey_tbl_sz); + } + + /* IB VL CAP enum isn't used by the firmware, just numerical values */ + for (vl_cap = 8; vl_cap >= 1; vl_cap >>= 1) { + ((__be32 *) mailbox->buf)[0] = cpu_to_be32( + (1 << MLX4_CHANGE_PORT_MTU_CAP) | + (1 << MLX4_CHANGE_PORT_VL_CAP) | + (pkey_tbl_flag << MLX4_CHANGE_PORT_PKEY_TBL_SZ) | + (dev->caps.port_ib_mtu[port] << MLX4_SET_PORT_MTU_CAP) | + (vl_cap << MLX4_SET_PORT_VL_CAP)); + err = mlx4_cmd(dev, mailbox->dma, port, + MLX4_SET_PORT_IB_OPCODE, MLX4_CMD_SET_PORT, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); + if (err != -ENOMEM) + break; + } + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +#define SET_PORT_ROCE_2_FLAGS 0x10 +#define MLX4_SET_PORT_ROCE_V1_V2 0x2 +int mlx4_SET_PORT_general(struct mlx4_dev *dev, u8 port, int mtu, + u8 pptx, u8 pfctx, u8 pprx, u8 pfcrx) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_set_port_general_context *context; + int err; + u32 in_mod; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + context = mailbox->buf; + context->flags = SET_PORT_GEN_ALL_VALID; + context->mtu = cpu_to_be16(mtu); + context->pptx = (pptx * (!pfctx)) << 7; + context->pfctx = pfctx; + context->pprx = (pprx * (!pfcrx)) << 7; + context->pfcrx = pfcrx; + + if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) { + context->flags |= SET_PORT_ROCE_2_FLAGS; + context->roce_mode |= + MLX4_SET_PORT_ROCE_V1_V2 << 4; + } + in_mod = MLX4_SET_PORT_GENERAL << 8 | port; + err = mlx4_cmd(dev, mailbox->dma, in_mod, MLX4_SET_PORT_ETH_OPCODE, + MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL(mlx4_SET_PORT_general); + +int mlx4_SET_PORT_qpn_calc(struct mlx4_dev *dev, u8 port, u32 base_qpn, + u8 promisc) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_set_port_rqp_calc_context *context; + int err; + u32 in_mod; + u32 m_promisc = (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) ? + MCAST_DIRECT : MCAST_DEFAULT; + + if (dev->caps.steering_mode != MLX4_STEERING_MODE_A0) + return 0; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + context = mailbox->buf; + context->base_qpn = cpu_to_be32(base_qpn); + context->n_mac = dev->caps.log_num_macs; + context->promisc = cpu_to_be32(promisc << SET_PORT_PROMISC_SHIFT | + base_qpn); + context->mcast = cpu_to_be32(m_promisc << SET_PORT_MC_PROMISC_SHIFT | + base_qpn); + context->intra_no_vlan = 0; + context->no_vlan = MLX4_NO_VLAN_IDX; + context->intra_vlan_miss = 0; + context->vlan_miss = MLX4_VLAN_MISS_IDX; + + in_mod = MLX4_SET_PORT_RQP_CALC << 8 | port; + err = mlx4_cmd(dev, mailbox->dma, in_mod, MLX4_SET_PORT_ETH_OPCODE, + MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL(mlx4_SET_PORT_qpn_calc); + +int mlx4_SET_PORT_fcs_check(struct mlx4_dev *dev, u8 port, u8 ignore_fcs_value) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_set_port_general_context *context; + u32 in_mod; + int err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + context = mailbox->buf; + context->v_ignore_fcs |= MLX4_FLAG_V_IGNORE_FCS_MASK; + if (ignore_fcs_value) + context->ignore_fcs |= MLX4_IGNORE_FCS_MASK; + else + context->ignore_fcs &= ~MLX4_IGNORE_FCS_MASK; + + in_mod = MLX4_SET_PORT_GENERAL << 8 | port; + err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL(mlx4_SET_PORT_fcs_check); + +enum { + VXLAN_ENABLE_MODIFY = 1 << 7, + VXLAN_STEERING_MODIFY = 1 << 6, + + VXLAN_ENABLE = 1 << 7, +}; + +struct mlx4_set_port_vxlan_context { + u32 reserved1; + u8 modify_flags; + u8 reserved2; + u8 enable_flags; + u8 steering; +}; + +int mlx4_SET_PORT_VXLAN(struct mlx4_dev *dev, u8 port, u8 steering, int enable) +{ + int err; + u32 in_mod; + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_set_port_vxlan_context *context; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + context = mailbox->buf; + memset(context, 0, sizeof(*context)); + + context->modify_flags = VXLAN_ENABLE_MODIFY | VXLAN_STEERING_MODIFY; + if (enable) + context->enable_flags = VXLAN_ENABLE; + context->steering = steering; + + in_mod = MLX4_SET_PORT_VXLAN << 8 | port; + err = mlx4_cmd(dev, mailbox->dma, in_mod, MLX4_SET_PORT_ETH_OPCODE, + MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL(mlx4_SET_PORT_VXLAN); + +int mlx4_SET_PORT_BEACON(struct mlx4_dev *dev, u8 port, u16 time) +{ + int err; + struct mlx4_cmd_mailbox *mailbox; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + *((__be32 *)mailbox->buf) = cpu_to_be32(time); + + err = mlx4_cmd(dev, mailbox->dma, port, MLX4_SET_PORT_BEACON_OPCODE, + MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL(mlx4_SET_PORT_BEACON); + +int mlx4_SET_MCAST_FLTR_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err = 0; + + return err; +} + +int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, + u64 mac, u64 clear, u8 mode) +{ + return mlx4_cmd(dev, (mac | (clear << 63)), port, mode, + MLX4_CMD_SET_MCAST_FLTR, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); +} +EXPORT_SYMBOL(mlx4_SET_MCAST_FLTR); + +int mlx4_SET_VLAN_FLTR_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err = 0; + + return err; +} + +int mlx4_DUMP_ETH_STATS_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + return 0; +} + +int mlx4_get_slave_from_roce_gid(struct mlx4_dev *dev, int port, u8 *gid, + int *slave_id) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int i, found_ix = -1; + int vf_gids = MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS; + struct mlx4_slaves_pport slaves_pport; + unsigned num_vfs; + int slave_gid; + + if (!mlx4_is_mfunc(dev)) + return -EINVAL; + + slaves_pport = mlx4_phys_to_slaves_pport(dev, port); + num_vfs = bitmap_weight(slaves_pport.slaves, + dev->persist->num_vfs + 1) - 1; + + for (i = 0; i < MLX4_ROCE_MAX_GIDS; i++) { + if (!memcmp(priv->port[port].gid_table.roce_gids[i].raw, gid, + MLX4_ROCE_GID_ENTRY_SIZE)) { + found_ix = i; + break; + } + } + + if (found_ix >= 0) { + /* Calculate a slave_gid which is the slave number in the gid + * table and not a globally unique slave number. + */ + if (found_ix < MLX4_ROCE_PF_GIDS) + slave_gid = 0; + else if (found_ix < MLX4_ROCE_PF_GIDS + (vf_gids % num_vfs) * + (vf_gids / num_vfs + 1)) + slave_gid = ((found_ix - MLX4_ROCE_PF_GIDS) / + (vf_gids / num_vfs + 1)) + 1; + else + slave_gid = + ((found_ix - MLX4_ROCE_PF_GIDS - + ((vf_gids % num_vfs) * ((vf_gids / num_vfs + 1)))) / + (vf_gids / num_vfs)) + vf_gids % num_vfs + 1; + + /* Calculate the globally unique slave id */ + if (slave_gid) { + struct mlx4_active_ports exclusive_ports; + struct mlx4_active_ports actv_ports; + struct mlx4_slaves_pport slaves_pport_actv; + unsigned max_port_p_one; + int num_vfs_before = 0; + int candidate_slave_gid; + + /* Calculate how many VFs are on the previous port, if exists */ + for (i = 1; i < port; i++) { + bitmap_zero(exclusive_ports.ports, dev->caps.num_ports); + set_bit(i - 1, exclusive_ports.ports); + slaves_pport_actv = + mlx4_phys_to_slaves_pport_actv( + dev, &exclusive_ports); + num_vfs_before += bitmap_weight( + slaves_pport_actv.slaves, + dev->persist->num_vfs + 1); + } + + /* candidate_slave_gid isn't necessarily the correct slave, but + * it has the same number of ports and is assigned to the same + * ports as the real slave we're looking for. On dual port VF, + * slave_gid = [single port VFs on port ] + + * [offset of the current slave from the first dual port VF] + + * 1 (for the PF). + */ + candidate_slave_gid = slave_gid + num_vfs_before; + + actv_ports = mlx4_get_active_ports(dev, candidate_slave_gid); + max_port_p_one = find_first_bit( + actv_ports.ports, dev->caps.num_ports) + + bitmap_weight(actv_ports.ports, + dev->caps.num_ports) + 1; + + /* Calculate the real slave number */ + for (i = 1; i < max_port_p_one; i++) { + if (i == port) + continue; + bitmap_zero(exclusive_ports.ports, + dev->caps.num_ports); + set_bit(i - 1, exclusive_ports.ports); + slaves_pport_actv = + mlx4_phys_to_slaves_pport_actv( + dev, &exclusive_ports); + slave_gid += bitmap_weight( + slaves_pport_actv.slaves, + dev->persist->num_vfs + 1); + } + } + *slave_id = slave_gid; + } + + return (found_ix >= 0) ? 0 : -EINVAL; +} +EXPORT_SYMBOL(mlx4_get_slave_from_roce_gid); + +int mlx4_get_roce_gid_from_slave(struct mlx4_dev *dev, int port, int slave_id, + u8 *gid) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + if (!mlx4_is_master(dev)) + return -EINVAL; + + memcpy(gid, priv->port[port].gid_table.roce_gids[slave_id].raw, + MLX4_ROCE_GID_ENTRY_SIZE); + return 0; +} +EXPORT_SYMBOL(mlx4_get_roce_gid_from_slave); + +/* Cable Module Info */ +#define MODULE_INFO_MAX_READ 48 + +#define I2C_ADDR_LOW 0x50 +#define I2C_ADDR_HIGH 0x51 +#define I2C_PAGE_SIZE 256 + +/* Module Info Data */ +struct mlx4_cable_info { + u8 i2c_addr; + u8 page_num; + __be16 dev_mem_address; + __be16 reserved1; + __be16 size; + __be32 reserved2[2]; + u8 data[MODULE_INFO_MAX_READ]; +}; + +enum cable_info_err { + CABLE_INF_INV_PORT = 0x1, + CABLE_INF_OP_NOSUP = 0x2, + CABLE_INF_NOT_CONN = 0x3, + CABLE_INF_NO_EEPRM = 0x4, + CABLE_INF_PAGE_ERR = 0x5, + CABLE_INF_INV_ADDR = 0x6, + CABLE_INF_I2C_ADDR = 0x7, + CABLE_INF_QSFP_VIO = 0x8, + CABLE_INF_I2C_BUSY = 0x9, +}; + +#define MAD_STATUS_2_CABLE_ERR(mad_status) ((mad_status >> 8) & 0xFF) + +#ifdef DEBUG +static inline const char *cable_info_mad_err_str(u16 mad_status) +{ + u8 err = MAD_STATUS_2_CABLE_ERR(mad_status); + + switch (err) { + case CABLE_INF_INV_PORT: + return "invalid port selected"; + case CABLE_INF_OP_NOSUP: + return "operation not supported for this port (the port is of type CX4 or internal)"; + case CABLE_INF_NOT_CONN: + return "cable is not connected"; + case CABLE_INF_NO_EEPRM: + return "the connected cable has no EPROM (passive copper cable)"; + case CABLE_INF_PAGE_ERR: + return "page number is greater than 15"; + case CABLE_INF_INV_ADDR: + return "invalid device_address or size (that is, size equals 0 or address+size is greater than 256)"; + case CABLE_INF_I2C_ADDR: + return "invalid I2C slave address"; + case CABLE_INF_QSFP_VIO: + return "at least one cable violates the QSFP specification and ignores the modsel signal"; + case CABLE_INF_I2C_BUSY: + return "I2C bus is constantly busy"; + } + return "Unknown Error"; +} +#endif /* DEBUG */ + +/** + * mlx4_get_module_info - Read cable module eeprom data + * @dev: mlx4_dev. + * @port: port number. + * @offset: byte offset in eeprom to start reading data from. + * @size: num of bytes to read. + * @data: output buffer to put the requested data into. + * + * Reads cable module eeprom data, puts the outcome data into + * data pointer paramer. + * Returns num of read bytes on success or a negative error + * code. + */ +int mlx4_get_module_info(struct mlx4_dev *dev, u8 port, + u16 offset, u16 size, u8 *data) +{ + struct mlx4_cmd_mailbox *inbox, *outbox; + struct mlx4_mad_ifc *inmad, *outmad; + struct mlx4_cable_info *cable_info; + u16 i2c_addr; + int ret; + + if (size > MODULE_INFO_MAX_READ) + size = MODULE_INFO_MAX_READ; + + inbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(inbox)) + return PTR_ERR(inbox); + + outbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(outbox)) { + mlx4_free_cmd_mailbox(dev, inbox); + return PTR_ERR(outbox); + } + + inmad = (struct mlx4_mad_ifc *)(inbox->buf); + outmad = (struct mlx4_mad_ifc *)(outbox->buf); + + inmad->method = 0x1; /* Get */ + inmad->class_version = 0x1; + inmad->mgmt_class = 0x1; + inmad->base_version = 0x1; + inmad->attr_id = cpu_to_be16(0xFF60); /* Module Info */ + + if (offset < I2C_PAGE_SIZE && offset + size > I2C_PAGE_SIZE) + /* Cross pages reads are not allowed + * read until offset 256 in low page + */ + size -= offset + size - I2C_PAGE_SIZE; + + i2c_addr = I2C_ADDR_LOW; + if (offset >= I2C_PAGE_SIZE) { + /* Reset offset to high page */ + i2c_addr = I2C_ADDR_HIGH; + offset -= I2C_PAGE_SIZE; + } + + cable_info = (struct mlx4_cable_info *)inmad->data; + cable_info->dev_mem_address = cpu_to_be16(offset); + cable_info->page_num = 0; + cable_info->i2c_addr = i2c_addr; + cable_info->size = cpu_to_be16(size); + + ret = mlx4_cmd_box(dev, inbox->dma, outbox->dma, port, 3, + MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C, + MLX4_CMD_NATIVE); + if (ret) + goto out; + + if (be16_to_cpu(outmad->status)) { + /* Mad returned with bad status */ + ret = be16_to_cpu(outmad->status); +#ifdef DEBUG + mlx4_warn(dev, + "MLX4_CMD_MAD_IFC Get Module info attr(%x) port(%d) i2c_addr(%x) offset(%d) size(%d): Response Mad Status(%x) - %s\n", + 0xFF60, port, i2c_addr, offset, size, + ret, cable_info_mad_err_str(ret)); +#endif + if (i2c_addr == I2C_ADDR_HIGH && + MAD_STATUS_2_CABLE_ERR(ret) == CABLE_INF_I2C_ADDR) + /* Some SFP cables do not support i2c slave + * address 0x51 (high page), abort silently. + */ + ret = 0; + else + ret = -ret; + goto out; + } + cable_info = (struct mlx4_cable_info *)outmad->data; + memcpy(data, cable_info->data, size); + ret = size; +out: + mlx4_free_cmd_mailbox(dev, inbox); + mlx4_free_cmd_mailbox(dev, outbox); + return ret; +} +EXPORT_SYMBOL(mlx4_get_module_info); Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_port.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_profile.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_profile.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_profile.c (revision 329159) @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2014 Mellanox Technologies. All rights reserved. + * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include + +#include "mlx4.h" +#include "fw.h" + +enum { + MLX4_RES_QP, + MLX4_RES_RDMARC, + MLX4_RES_ALTC, + MLX4_RES_AUXC, + MLX4_RES_SRQ, + MLX4_RES_CQ, + MLX4_RES_EQ, + MLX4_RES_DMPT, + MLX4_RES_CMPT, + MLX4_RES_MTT, + MLX4_RES_MCG, + MLX4_RES_NUM +}; + +static const char *res_name[] = { + [MLX4_RES_QP] = "QP", + [MLX4_RES_RDMARC] = "RDMARC", + [MLX4_RES_ALTC] = "ALTC", + [MLX4_RES_AUXC] = "AUXC", + [MLX4_RES_SRQ] = "SRQ", + [MLX4_RES_CQ] = "CQ", + [MLX4_RES_EQ] = "EQ", + [MLX4_RES_DMPT] = "DMPT", + [MLX4_RES_CMPT] = "CMPT", + [MLX4_RES_MTT] = "MTT", + [MLX4_RES_MCG] = "MCG", +}; + +u64 mlx4_make_profile(struct mlx4_dev *dev, + struct mlx4_profile *request, + struct mlx4_dev_cap *dev_cap, + struct mlx4_init_hca_param *init_hca) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource { + u64 size; + u64 start; + int type; + u32 num; + int log_num; + }; + + u64 total_size = 0; + struct mlx4_resource *profile; + int i, j; + + profile = kcalloc(MLX4_RES_NUM, sizeof(*profile), GFP_KERNEL); + if (!profile) + return -ENOMEM; + + profile[MLX4_RES_QP].size = dev_cap->qpc_entry_sz; + profile[MLX4_RES_RDMARC].size = dev_cap->rdmarc_entry_sz; + profile[MLX4_RES_ALTC].size = dev_cap->altc_entry_sz; + profile[MLX4_RES_AUXC].size = dev_cap->aux_entry_sz; + profile[MLX4_RES_SRQ].size = dev_cap->srq_entry_sz; + profile[MLX4_RES_CQ].size = dev_cap->cqc_entry_sz; + profile[MLX4_RES_EQ].size = dev_cap->eqc_entry_sz; + profile[MLX4_RES_DMPT].size = dev_cap->dmpt_entry_sz; + profile[MLX4_RES_CMPT].size = dev_cap->cmpt_entry_sz; + profile[MLX4_RES_MTT].size = dev_cap->mtt_entry_sz; + profile[MLX4_RES_MCG].size = mlx4_get_mgm_entry_size(dev); + + profile[MLX4_RES_QP].num = request->num_qp; + profile[MLX4_RES_RDMARC].num = request->num_qp * request->rdmarc_per_qp; + profile[MLX4_RES_ALTC].num = request->num_qp; + profile[MLX4_RES_AUXC].num = request->num_qp; + profile[MLX4_RES_SRQ].num = request->num_srq; + profile[MLX4_RES_CQ].num = request->num_cq; + profile[MLX4_RES_EQ].num = mlx4_is_mfunc(dev) ? dev->phys_caps.num_phys_eqs : + min_t(unsigned, dev_cap->max_eqs, MAX_MSIX); + profile[MLX4_RES_DMPT].num = request->num_mpt; + profile[MLX4_RES_CMPT].num = MLX4_NUM_CMPTS; + profile[MLX4_RES_MTT].num = request->num_mtt * (1 << log_mtts_per_seg); + profile[MLX4_RES_MCG].num = request->num_mcg; + + for (i = 0; i < MLX4_RES_NUM; ++i) { + profile[i].type = i; + profile[i].num = roundup_pow_of_two(profile[i].num); + profile[i].log_num = ilog2(profile[i].num); + profile[i].size *= profile[i].num; + profile[i].size = max(profile[i].size, (u64) PAGE_SIZE); + } + + /* + * Sort the resources in decreasing order of size. Since they + * all have sizes that are powers of 2, we'll be able to keep + * resources aligned to their size and pack them without gaps + * using the sorted order. + */ + for (i = MLX4_RES_NUM; i > 0; --i) + for (j = 1; j < i; ++j) { + if (profile[j].size > profile[j - 1].size) + swap(profile[j], profile[j - 1]); + } + + for (i = 0; i < MLX4_RES_NUM; ++i) { + if (profile[i].size) { + profile[i].start = total_size; + total_size += profile[i].size; + } + + if (total_size > dev_cap->max_icm_sz) { + mlx4_err(dev, "Profile requires 0x%llx bytes; won't fit in 0x%llx bytes of context memory\n", + (unsigned long long) total_size, + (unsigned long long) dev_cap->max_icm_sz); + kfree(profile); + return -ENOMEM; + } + + if (profile[i].size) + mlx4_dbg(dev, " profile[%2d] (%6s): 2^%02d entries @ 0x%10llx, size 0x%10llx\n", + i, res_name[profile[i].type], + profile[i].log_num, + (unsigned long long) profile[i].start, + (unsigned long long) profile[i].size); + } + + mlx4_dbg(dev, "HCA context memory: reserving %d KB\n", + (int) (total_size >> 10)); + + for (i = 0; i < MLX4_RES_NUM; ++i) { + switch (profile[i].type) { + case MLX4_RES_QP: + dev->caps.num_qps = profile[i].num; + init_hca->qpc_base = profile[i].start; + init_hca->log_num_qps = profile[i].log_num; + break; + case MLX4_RES_RDMARC: + for (priv->qp_table.rdmarc_shift = 0; + request->num_qp << priv->qp_table.rdmarc_shift < profile[i].num; + ++priv->qp_table.rdmarc_shift) + ; /* nothing */ + dev->caps.max_qp_dest_rdma = 1 << priv->qp_table.rdmarc_shift; + priv->qp_table.rdmarc_base = (u32) profile[i].start; + init_hca->rdmarc_base = profile[i].start; + init_hca->log_rd_per_qp = priv->qp_table.rdmarc_shift; + break; + case MLX4_RES_ALTC: + init_hca->altc_base = profile[i].start; + break; + case MLX4_RES_AUXC: + init_hca->auxc_base = profile[i].start; + break; + case MLX4_RES_SRQ: + dev->caps.num_srqs = profile[i].num; + init_hca->srqc_base = profile[i].start; + init_hca->log_num_srqs = profile[i].log_num; + break; + case MLX4_RES_CQ: + dev->caps.num_cqs = profile[i].num; + init_hca->cqc_base = profile[i].start; + init_hca->log_num_cqs = profile[i].log_num; + break; + case MLX4_RES_EQ: + if (dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) { + init_hca->log_num_eqs = 0x1f; + init_hca->eqc_base = profile[i].start; + init_hca->num_sys_eqs = dev_cap->num_sys_eqs; + } else { + dev->caps.num_eqs = roundup_pow_of_two( + min_t(unsigned, + dev_cap->max_eqs, + MAX_MSIX)); + init_hca->eqc_base = profile[i].start; + init_hca->log_num_eqs = ilog2(dev->caps.num_eqs); + } + break; + case MLX4_RES_DMPT: + dev->caps.num_mpts = profile[i].num; + priv->mr_table.mpt_base = profile[i].start; + init_hca->dmpt_base = profile[i].start; + init_hca->log_mpt_sz = profile[i].log_num; + break; + case MLX4_RES_CMPT: + init_hca->cmpt_base = profile[i].start; + break; + case MLX4_RES_MTT: + dev->caps.num_mtts = profile[i].num; + priv->mr_table.mtt_base = profile[i].start; + init_hca->mtt_base = profile[i].start; + break; + case MLX4_RES_MCG: + init_hca->mc_base = profile[i].start; + init_hca->log_mc_entry_sz = + ilog2(mlx4_get_mgm_entry_size(dev)); + init_hca->log_mc_table_sz = profile[i].log_num; + if (dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) { + dev->caps.num_mgms = profile[i].num; + } else { + init_hca->log_mc_hash_sz = + profile[i].log_num - 1; + dev->caps.num_mgms = profile[i].num >> 1; + dev->caps.num_amgms = profile[i].num >> 1; + } + break; + default: + break; + } + } + + /* + * PDs don't take any HCA memory, but we assign them as part + * of the HCA profile anyway. + */ + dev->caps.num_pds = MLX4_NUM_PDS; + + kfree(profile); + return total_size; +} Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_profile.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_qp.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_qp.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_qp.c (revision 329159) @@ -0,0 +1,950 @@ +/* + * Copyright (c) 2004 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * Copyright (c) 2004 Voltaire, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include + +#include +#include + +#include "mlx4.h" +#include "icm.h" + +/* QP to support BF should have bits 6,7 cleared */ +#define MLX4_BF_QP_SKIP_MASK 0xc0 +#define MLX4_MAX_BF_QP_RANGE 0x40 + +void mlx4_qp_event(struct mlx4_dev *dev, u32 qpn, int event_type) +{ + struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table; + struct mlx4_qp *qp; + + spin_lock(&qp_table->lock); + + qp = __mlx4_qp_lookup(dev, qpn); + if (qp) + atomic_inc(&qp->refcount); + + spin_unlock(&qp_table->lock); + + if (!qp) { + mlx4_dbg(dev, "Async event for none existent QP %08x\n", qpn); + return; + } + + qp->event(qp, event_type); + + if (atomic_dec_and_test(&qp->refcount)) + complete(&qp->free); +} + +/* used for INIT/CLOSE port logic */ +static int is_master_qp0(struct mlx4_dev *dev, struct mlx4_qp *qp, int *real_qp0, int *proxy_qp0) +{ + /* this procedure is called after we already know we are on the master */ + /* qp0 is either the proxy qp0, or the real qp0 */ + u32 pf_proxy_offset = dev->phys_caps.base_proxy_sqpn + 8 * mlx4_master_func_num(dev); + *proxy_qp0 = qp->qpn >= pf_proxy_offset && qp->qpn <= pf_proxy_offset + 1; + + *real_qp0 = qp->qpn >= dev->phys_caps.base_sqpn && + qp->qpn <= dev->phys_caps.base_sqpn + 1; + + return *real_qp0 || *proxy_qp0; +} + +static int __mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + enum mlx4_qp_state cur_state, enum mlx4_qp_state new_state, + struct mlx4_qp_context *context, + enum mlx4_qp_optpar optpar, + int sqd_event, struct mlx4_qp *qp, int native) +{ + static const u16 op[MLX4_QP_NUM_STATE][MLX4_QP_NUM_STATE] = { + [MLX4_QP_STATE_RST] = { + [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, + [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, + [MLX4_QP_STATE_INIT] = MLX4_CMD_RST2INIT_QP, + }, + [MLX4_QP_STATE_INIT] = { + [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, + [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, + [MLX4_QP_STATE_INIT] = MLX4_CMD_INIT2INIT_QP, + [MLX4_QP_STATE_RTR] = MLX4_CMD_INIT2RTR_QP, + }, + [MLX4_QP_STATE_RTR] = { + [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, + [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, + [MLX4_QP_STATE_RTS] = MLX4_CMD_RTR2RTS_QP, + }, + [MLX4_QP_STATE_RTS] = { + [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, + [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, + [MLX4_QP_STATE_RTS] = MLX4_CMD_RTS2RTS_QP, + [MLX4_QP_STATE_SQD] = MLX4_CMD_RTS2SQD_QP, + }, + [MLX4_QP_STATE_SQD] = { + [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, + [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, + [MLX4_QP_STATE_RTS] = MLX4_CMD_SQD2RTS_QP, + [MLX4_QP_STATE_SQD] = MLX4_CMD_SQD2SQD_QP, + }, + [MLX4_QP_STATE_SQER] = { + [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, + [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, + [MLX4_QP_STATE_RTS] = MLX4_CMD_SQERR2RTS_QP, + }, + [MLX4_QP_STATE_ERR] = { + [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, + [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, + } + }; + + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_cmd_mailbox *mailbox; + int ret = 0; + int real_qp0 = 0; + int proxy_qp0 = 0; + u8 port; + + if (cur_state >= MLX4_QP_NUM_STATE || new_state >= MLX4_QP_NUM_STATE || + !op[cur_state][new_state]) + return -EINVAL; + + if (op[cur_state][new_state] == MLX4_CMD_2RST_QP) { + ret = mlx4_cmd(dev, 0, qp->qpn, 2, + MLX4_CMD_2RST_QP, MLX4_CMD_TIME_CLASS_A, native); + if (mlx4_is_master(dev) && cur_state != MLX4_QP_STATE_ERR && + cur_state != MLX4_QP_STATE_RST && + is_master_qp0(dev, qp, &real_qp0, &proxy_qp0)) { + port = (qp->qpn & 1) + 1; + if (proxy_qp0) + priv->mfunc.master.qp0_state[port].proxy_qp0_active = 0; + else + priv->mfunc.master.qp0_state[port].qp0_active = 0; + } + return ret; + } + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + if (cur_state == MLX4_QP_STATE_RST && new_state == MLX4_QP_STATE_INIT) { + u64 mtt_addr = mlx4_mtt_addr(dev, mtt); + context->mtt_base_addr_h = mtt_addr >> 32; + context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); + context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT; + } + + if ((cur_state == MLX4_QP_STATE_RTR) && + (new_state == MLX4_QP_STATE_RTS) && + dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) + context->roce_entropy = + cpu_to_be16(mlx4_qp_roce_entropy(dev, qp->qpn)); + + *(__be32 *) mailbox->buf = cpu_to_be32(optpar); + memcpy(mailbox->buf + 8, context, sizeof *context); + + ((struct mlx4_qp_context *) (mailbox->buf + 8))->local_qpn = + cpu_to_be32(qp->qpn); + + ret = mlx4_cmd(dev, mailbox->dma, + qp->qpn | (!!sqd_event << 31), + new_state == MLX4_QP_STATE_RST ? 2 : 0, + op[cur_state][new_state], MLX4_CMD_TIME_CLASS_C, native); + + if (mlx4_is_master(dev) && is_master_qp0(dev, qp, &real_qp0, &proxy_qp0)) { + port = (qp->qpn & 1) + 1; + if (cur_state != MLX4_QP_STATE_ERR && + cur_state != MLX4_QP_STATE_RST && + new_state == MLX4_QP_STATE_ERR) { + if (proxy_qp0) + priv->mfunc.master.qp0_state[port].proxy_qp0_active = 0; + else + priv->mfunc.master.qp0_state[port].qp0_active = 0; + } else if (new_state == MLX4_QP_STATE_RTR) { + if (proxy_qp0) + priv->mfunc.master.qp0_state[port].proxy_qp0_active = 1; + else + priv->mfunc.master.qp0_state[port].qp0_active = 1; + } + } + + mlx4_free_cmd_mailbox(dev, mailbox); + return ret; +} + +int mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + enum mlx4_qp_state cur_state, enum mlx4_qp_state new_state, + struct mlx4_qp_context *context, + enum mlx4_qp_optpar optpar, + int sqd_event, struct mlx4_qp *qp) +{ + return __mlx4_qp_modify(dev, mtt, cur_state, new_state, context, + optpar, sqd_event, qp, 0); +} +EXPORT_SYMBOL_GPL(mlx4_qp_modify); + +int __mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, + int *base, u8 flags) +{ + u32 uid; + int bf_qp = !!(flags & (u8)MLX4_RESERVE_ETH_BF_QP); + + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_qp_table *qp_table = &priv->qp_table; + + if (cnt > MLX4_MAX_BF_QP_RANGE && bf_qp) + return -ENOMEM; + + uid = MLX4_QP_TABLE_ZONE_GENERAL; + if (flags & (u8)MLX4_RESERVE_A0_QP) { + if (bf_qp) + uid = MLX4_QP_TABLE_ZONE_RAW_ETH; + else + uid = MLX4_QP_TABLE_ZONE_RSS; + } + + *base = mlx4_zone_alloc_entries(qp_table->zones, uid, cnt, align, + bf_qp ? MLX4_BF_QP_SKIP_MASK : 0, NULL); + if (*base == -1) + return -ENOMEM; + + return 0; +} + +int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, + int *base, u8 flags) +{ + u64 in_param = 0; + u64 out_param; + int err; + + /* Turn off all unsupported QP allocation flags */ + flags &= dev->caps.alloc_res_qp_mask; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, (((u32)flags) << 24) | (u32)cnt); + set_param_h(&in_param, align); + err = mlx4_cmd_imm(dev, in_param, &out_param, + RES_QP, RES_OP_RESERVE, + MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (err) + return err; + + *base = get_param_l(&out_param); + return 0; + } + return __mlx4_qp_reserve_range(dev, cnt, align, base, flags); +} +EXPORT_SYMBOL_GPL(mlx4_qp_reserve_range); + +void __mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_qp_table *qp_table = &priv->qp_table; + + if (mlx4_is_qp_reserved(dev, (u32) base_qpn)) + return; + mlx4_zone_free_entries_unique(qp_table->zones, base_qpn, cnt); +} + +void mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt) +{ + u64 in_param = 0; + int err; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, base_qpn); + set_param_h(&in_param, cnt); + err = mlx4_cmd(dev, in_param, RES_QP, RES_OP_RESERVE, + MLX4_CMD_FREE_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (err) { + mlx4_warn(dev, "Failed to release qp range base:%d cnt:%d\n", + base_qpn, cnt); + } + } else + __mlx4_qp_release_range(dev, base_qpn, cnt); +} +EXPORT_SYMBOL_GPL(mlx4_qp_release_range); + +int __mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn, gfp_t gfp) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_qp_table *qp_table = &priv->qp_table; + int err; + + err = mlx4_table_get(dev, &qp_table->qp_table, qpn, gfp); + if (err) + goto err_out; + + err = mlx4_table_get(dev, &qp_table->auxc_table, qpn, gfp); + if (err) + goto err_put_qp; + + err = mlx4_table_get(dev, &qp_table->altc_table, qpn, gfp); + if (err) + goto err_put_auxc; + + err = mlx4_table_get(dev, &qp_table->rdmarc_table, qpn, gfp); + if (err) + goto err_put_altc; + + err = mlx4_table_get(dev, &qp_table->cmpt_table, qpn, gfp); + if (err) + goto err_put_rdmarc; + + return 0; + +err_put_rdmarc: + mlx4_table_put(dev, &qp_table->rdmarc_table, qpn); + +err_put_altc: + mlx4_table_put(dev, &qp_table->altc_table, qpn); + +err_put_auxc: + mlx4_table_put(dev, &qp_table->auxc_table, qpn); + +err_put_qp: + mlx4_table_put(dev, &qp_table->qp_table, qpn); + +err_out: + return err; +} + +static int mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn, gfp_t gfp) +{ + u64 param = 0; + + if (mlx4_is_mfunc(dev)) { + set_param_l(¶m, qpn); + return mlx4_cmd_imm(dev, param, ¶m, RES_QP, RES_OP_MAP_ICM, + MLX4_CMD_ALLOC_RES, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); + } + return __mlx4_qp_alloc_icm(dev, qpn, gfp); +} + +void __mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_qp_table *qp_table = &priv->qp_table; + + mlx4_table_put(dev, &qp_table->cmpt_table, qpn); + mlx4_table_put(dev, &qp_table->rdmarc_table, qpn); + mlx4_table_put(dev, &qp_table->altc_table, qpn); + mlx4_table_put(dev, &qp_table->auxc_table, qpn); + mlx4_table_put(dev, &qp_table->qp_table, qpn); +} + +static void mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn) +{ + u64 in_param = 0; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, qpn); + if (mlx4_cmd(dev, in_param, RES_QP, RES_OP_MAP_ICM, + MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED)) + mlx4_warn(dev, "Failed to free icm of qp:%d\n", qpn); + } else + __mlx4_qp_free_icm(dev, qpn); +} + +int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp, gfp_t gfp) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_qp_table *qp_table = &priv->qp_table; + int err; + + if (!qpn) + return -EINVAL; + + qp->qpn = qpn; + + err = mlx4_qp_alloc_icm(dev, qpn, gfp); + if (err) + return err; + + spin_lock_irq(&qp_table->lock); + err = radix_tree_insert(&dev->qp_table_tree, qp->qpn & + (dev->caps.num_qps - 1), qp); + spin_unlock_irq(&qp_table->lock); + if (err) + goto err_icm; + + atomic_set(&qp->refcount, 1); + init_completion(&qp->free); + + return 0; + +err_icm: + mlx4_qp_free_icm(dev, qpn); + return err; +} + +EXPORT_SYMBOL_GPL(mlx4_qp_alloc); + +int mlx4_update_qp(struct mlx4_dev *dev, u32 qpn, + enum mlx4_update_qp_attr attr, + struct mlx4_update_qp_params *params) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_update_qp_context *cmd; + u64 pri_addr_path_mask = 0; + u64 qp_mask = 0; + int err = 0; + + if (!attr || (attr & ~MLX4_UPDATE_QP_SUPPORTED_ATTRS)) + return -EINVAL; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + cmd = (struct mlx4_update_qp_context *)mailbox->buf; + + if (attr & MLX4_UPDATE_QP_SMAC) { + pri_addr_path_mask |= 1ULL << MLX4_UPD_QP_PATH_MASK_MAC_INDEX; + cmd->qp_context.pri_path.grh_mylmc = params->smac_index; + } + + if (attr & MLX4_UPDATE_QP_ETH_SRC_CHECK_MC_LB) { + if (!(dev->caps.flags2 + & MLX4_DEV_CAP_FLAG2_UPDATE_QP_SRC_CHECK_LB)) { + mlx4_warn(dev, + "Trying to set src check LB, but it isn't supported\n"); + err = -ENOTSUPP; + goto out; + } + pri_addr_path_mask |= + 1ULL << MLX4_UPD_QP_PATH_MASK_ETH_SRC_CHECK_MC_LB; + if (params->flags & + MLX4_UPDATE_QP_PARAMS_FLAGS_ETH_CHECK_MC_LB) { + cmd->qp_context.pri_path.fl |= + MLX4_FL_ETH_SRC_CHECK_MC_LB; + } + } + + if (attr & MLX4_UPDATE_QP_VSD) { + qp_mask |= 1ULL << MLX4_UPD_QP_MASK_VSD; + if (params->flags & MLX4_UPDATE_QP_PARAMS_FLAGS_VSD_ENABLE) + cmd->qp_context.param3 |= cpu_to_be32(MLX4_STRIP_VLAN); + } + + if (attr & MLX4_UPDATE_QP_RATE_LIMIT) { + qp_mask |= 1ULL << MLX4_UPD_QP_MASK_RATE_LIMIT; + cmd->qp_context.rate_limit_params = cpu_to_be16((params->rate_unit << 14) | params->rate_val); + } + + if (attr & MLX4_UPDATE_QP_QOS_VPORT) { + qp_mask |= 1ULL << MLX4_UPD_QP_MASK_QOS_VPP; + cmd->qp_context.qos_vport = params->qos_vport; + } + + cmd->primary_addr_path_mask = cpu_to_be64(pri_addr_path_mask); + cmd->qp_mask = cpu_to_be64(qp_mask); + + err = mlx4_cmd(dev, mailbox->dma, qpn & 0xffffff, 0, + MLX4_CMD_UPDATE_QP, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); +out: + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_update_qp); + +void mlx4_qp_remove(struct mlx4_dev *dev, struct mlx4_qp *qp) +{ + struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table; + unsigned long flags; + + spin_lock_irqsave(&qp_table->lock, flags); + radix_tree_delete(&dev->qp_table_tree, qp->qpn & (dev->caps.num_qps - 1)); + spin_unlock_irqrestore(&qp_table->lock, flags); +} +EXPORT_SYMBOL_GPL(mlx4_qp_remove); + +void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp) +{ + if (atomic_dec_and_test(&qp->refcount)) + complete(&qp->free); + wait_for_completion(&qp->free); + + mlx4_qp_free_icm(dev, qp->qpn); +} +EXPORT_SYMBOL_GPL(mlx4_qp_free); + +static int mlx4_CONF_SPECIAL_QP(struct mlx4_dev *dev, u32 base_qpn) +{ + return mlx4_cmd(dev, 0, base_qpn, 0, MLX4_CMD_CONF_SPECIAL_QP, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); +} + +#define MLX4_QP_TABLE_RSS_ETH_PRIORITY 2 +#define MLX4_QP_TABLE_RAW_ETH_PRIORITY 1 +#define MLX4_QP_TABLE_RAW_ETH_SIZE 256 + +static int mlx4_create_zones(struct mlx4_dev *dev, + u32 reserved_bottom_general, + u32 reserved_top_general, + u32 reserved_bottom_rss, + u32 start_offset_rss, + u32 max_table_offset) +{ + struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table; + struct mlx4_bitmap (*bitmap)[MLX4_QP_TABLE_ZONE_NUM] = NULL; + int bitmap_initialized = 0; + u32 last_offset; + int k; + int err; + + qp_table->zones = mlx4_zone_allocator_create(MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP); + + if (NULL == qp_table->zones) + return -ENOMEM; + + bitmap = kmalloc(sizeof(*bitmap), GFP_KERNEL); + + if (NULL == bitmap) { + err = -ENOMEM; + goto free_zone; + } + + err = mlx4_bitmap_init(*bitmap + MLX4_QP_TABLE_ZONE_GENERAL, dev->caps.num_qps, + (1 << 23) - 1, reserved_bottom_general, + reserved_top_general); + + if (err) + goto free_bitmap; + + ++bitmap_initialized; + + err = mlx4_zone_add_one(qp_table->zones, *bitmap + MLX4_QP_TABLE_ZONE_GENERAL, + MLX4_ZONE_FALLBACK_TO_HIGHER_PRIO | + MLX4_ZONE_USE_RR, 0, + 0, qp_table->zones_uids + MLX4_QP_TABLE_ZONE_GENERAL); + + if (err) + goto free_bitmap; + + err = mlx4_bitmap_init(*bitmap + MLX4_QP_TABLE_ZONE_RSS, + reserved_bottom_rss, + reserved_bottom_rss - 1, + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], + reserved_bottom_rss - start_offset_rss); + + if (err) + goto free_bitmap; + + ++bitmap_initialized; + + err = mlx4_zone_add_one(qp_table->zones, *bitmap + MLX4_QP_TABLE_ZONE_RSS, + MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO | + MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO | + MLX4_ZONE_USE_RR, MLX4_QP_TABLE_RSS_ETH_PRIORITY, + 0, qp_table->zones_uids + MLX4_QP_TABLE_ZONE_RSS); + + if (err) + goto free_bitmap; + + last_offset = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW]; + /* We have a single zone for the A0 steering QPs area of the FW. This area + * needs to be split into subareas. One set of subareas is for RSS QPs + * (in which qp number bits 6 and/or 7 are set); the other set of subareas + * is for RAW_ETH QPs, which require that both bits 6 and 7 are zero. + * Currently, the values returned by the FW (A0 steering area starting qp number + * and A0 steering area size) are such that there are only two subareas -- one + * for RSS and one for RAW_ETH. + */ + for (k = MLX4_QP_TABLE_ZONE_RSS + 1; k < sizeof(*bitmap)/sizeof((*bitmap)[0]); + k++) { + int size; + u32 offset = start_offset_rss; + u32 bf_mask; + u32 requested_size; + + /* Assuming MLX4_BF_QP_SKIP_MASK is consecutive ones, this calculates + * a mask of all LSB bits set until (and not including) the first + * set bit of MLX4_BF_QP_SKIP_MASK. For example, if MLX4_BF_QP_SKIP_MASK + * is 0xc0, bf_mask will be 0x3f. + */ + bf_mask = (MLX4_BF_QP_SKIP_MASK & ~(MLX4_BF_QP_SKIP_MASK - 1)) - 1; + requested_size = min((u32)MLX4_QP_TABLE_RAW_ETH_SIZE, bf_mask + 1); + + if (((last_offset & MLX4_BF_QP_SKIP_MASK) && + ((int)(max_table_offset - last_offset)) >= + roundup_pow_of_two(MLX4_BF_QP_SKIP_MASK)) || + (!(last_offset & MLX4_BF_QP_SKIP_MASK) && + !((last_offset + requested_size - 1) & + MLX4_BF_QP_SKIP_MASK))) + size = requested_size; + else { + u32 candidate_offset = + (last_offset | MLX4_BF_QP_SKIP_MASK | bf_mask) + 1; + + if (last_offset & MLX4_BF_QP_SKIP_MASK) + last_offset = candidate_offset; + + /* From this point, the BF bits are 0 */ + + if (last_offset > max_table_offset) { + /* need to skip */ + size = -1; + } else { + size = min3(max_table_offset - last_offset, + bf_mask - (last_offset & bf_mask), + requested_size); + if (size < requested_size) { + int candidate_size; + + candidate_size = min3( + max_table_offset - candidate_offset, + bf_mask - (last_offset & bf_mask), + requested_size); + + /* We will not take this path if last_offset was + * already set above to candidate_offset + */ + if (candidate_size > size) { + last_offset = candidate_offset; + size = candidate_size; + } + } + } + } + + if (size > 0) { + /* mlx4_bitmap_alloc_range will find a contiguous range of "size" + * QPs in which both bits 6 and 7 are zero, because we pass it the + * MLX4_BF_SKIP_MASK). + */ + offset = mlx4_bitmap_alloc_range( + *bitmap + MLX4_QP_TABLE_ZONE_RSS, + size, 1, + MLX4_BF_QP_SKIP_MASK); + + if (offset == (u32)-1) { + err = -ENOMEM; + break; + } + + last_offset = offset + size; + + err = mlx4_bitmap_init(*bitmap + k, roundup_pow_of_two(size), + roundup_pow_of_two(size) - 1, 0, + roundup_pow_of_two(size) - size); + } else { + /* Add an empty bitmap, we'll allocate from different zones (since + * at least one is reserved) + */ + err = mlx4_bitmap_init(*bitmap + k, 1, + MLX4_QP_TABLE_RAW_ETH_SIZE - 1, 0, + 0); + mlx4_bitmap_alloc_range(*bitmap + k, 1, 1, 0); + } + + if (err) + break; + + ++bitmap_initialized; + + err = mlx4_zone_add_one(qp_table->zones, *bitmap + k, + MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO | + MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO | + MLX4_ZONE_USE_RR, MLX4_QP_TABLE_RAW_ETH_PRIORITY, + offset, qp_table->zones_uids + k); + + if (err) + break; + } + + if (err) + goto free_bitmap; + + qp_table->bitmap_gen = *bitmap; + + return err; + +free_bitmap: + for (k = 0; k < bitmap_initialized; k++) + mlx4_bitmap_cleanup(*bitmap + k); + kfree(bitmap); +free_zone: + mlx4_zone_allocator_destroy(qp_table->zones); + return err; +} + +static void mlx4_cleanup_qp_zones(struct mlx4_dev *dev) +{ + struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table; + + if (qp_table->zones) { + int i; + + for (i = 0; + i < sizeof(qp_table->zones_uids)/sizeof(qp_table->zones_uids[0]); + i++) { + struct mlx4_bitmap *bitmap = + mlx4_zone_get_bitmap(qp_table->zones, + qp_table->zones_uids[i]); + + mlx4_zone_remove_one(qp_table->zones, qp_table->zones_uids[i]); + if (NULL == bitmap) + continue; + + mlx4_bitmap_cleanup(bitmap); + } + mlx4_zone_allocator_destroy(qp_table->zones); + kfree(qp_table->bitmap_gen); + qp_table->bitmap_gen = NULL; + qp_table->zones = NULL; + } +} + +int mlx4_init_qp_table(struct mlx4_dev *dev) +{ + struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table; + int err; + int reserved_from_top = 0; + int reserved_from_bot; + int k; + int fixed_reserved_from_bot_rv = 0; + int bottom_reserved_for_rss_bitmap; + u32 max_table_offset = dev->caps.dmfs_high_rate_qpn_base + + dev->caps.dmfs_high_rate_qpn_range; + + spin_lock_init(&qp_table->lock); + INIT_RADIX_TREE(&dev->qp_table_tree, GFP_ATOMIC); + if (mlx4_is_slave(dev)) + return 0; + + /* We reserve 2 extra QPs per port for the special QPs. The + * block of special QPs must be aligned to a multiple of 8, so + * round up. + * + * We also reserve the MSB of the 24-bit QP number to indicate + * that a QP is an XRC QP. + */ + for (k = 0; k <= MLX4_QP_REGION_BOTTOM; k++) + fixed_reserved_from_bot_rv += dev->caps.reserved_qps_cnt[k]; + + if (fixed_reserved_from_bot_rv < max_table_offset) + fixed_reserved_from_bot_rv = max_table_offset; + + /* We reserve at least 1 extra for bitmaps that we don't have enough space for*/ + bottom_reserved_for_rss_bitmap = + roundup_pow_of_two(fixed_reserved_from_bot_rv + 1); + dev->phys_caps.base_sqpn = ALIGN(bottom_reserved_for_rss_bitmap, 8); + + { + int sort[MLX4_NUM_QP_REGION]; + int i, j; + int last_base = dev->caps.num_qps; + + for (i = 1; i < MLX4_NUM_QP_REGION; ++i) + sort[i] = i; + + for (i = MLX4_NUM_QP_REGION; i > MLX4_QP_REGION_BOTTOM; --i) { + for (j = MLX4_QP_REGION_BOTTOM + 2; j < i; ++j) { + if (dev->caps.reserved_qps_cnt[sort[j]] > + dev->caps.reserved_qps_cnt[sort[j - 1]]) + swap(sort[j], sort[j - 1]); + } + } + + for (i = MLX4_QP_REGION_BOTTOM + 1; i < MLX4_NUM_QP_REGION; ++i) { + last_base -= dev->caps.reserved_qps_cnt[sort[i]]; + dev->caps.reserved_qps_base[sort[i]] = last_base; + reserved_from_top += + dev->caps.reserved_qps_cnt[sort[i]]; + } + } + + /* Reserve 8 real SQPs in both native and SRIOV modes. + * In addition, in SRIOV mode, reserve 8 proxy SQPs per function + * (for all PFs and VFs), and 8 corresponding tunnel QPs. + * Each proxy SQP works opposite its own tunnel QP. + * + * The QPs are arranged as follows: + * a. 8 real SQPs + * b. All the proxy SQPs (8 per function) + * c. All the tunnel QPs (8 per function) + */ + reserved_from_bot = mlx4_num_reserved_sqps(dev); + if (reserved_from_bot + reserved_from_top > dev->caps.num_qps) { + mlx4_err(dev, "Number of reserved QPs is higher than number of QPs\n"); + return -EINVAL; + } + + err = mlx4_create_zones(dev, reserved_from_bot, reserved_from_bot, + bottom_reserved_for_rss_bitmap, + fixed_reserved_from_bot_rv, + max_table_offset); + + if (err) + return err; + + if (mlx4_is_mfunc(dev)) { + /* for PPF use */ + dev->phys_caps.base_proxy_sqpn = dev->phys_caps.base_sqpn + 8; + dev->phys_caps.base_tunnel_sqpn = dev->phys_caps.base_sqpn + 8 + 8 * MLX4_MFUNC_MAX; + + /* In mfunc, calculate proxy and tunnel qp offsets for the PF here, + * since the PF does not call mlx4_slave_caps */ + dev->caps.qp0_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); + dev->caps.qp0_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); + dev->caps.qp1_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); + dev->caps.qp1_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); + + if (!dev->caps.qp0_tunnel || !dev->caps.qp0_proxy || + !dev->caps.qp1_tunnel || !dev->caps.qp1_proxy) { + err = -ENOMEM; + goto err_mem; + } + + for (k = 0; k < dev->caps.num_ports; k++) { + dev->caps.qp0_proxy[k] = dev->phys_caps.base_proxy_sqpn + + 8 * mlx4_master_func_num(dev) + k; + dev->caps.qp0_tunnel[k] = dev->caps.qp0_proxy[k] + 8 * MLX4_MFUNC_MAX; + dev->caps.qp1_proxy[k] = dev->phys_caps.base_proxy_sqpn + + 8 * mlx4_master_func_num(dev) + MLX4_MAX_PORTS + k; + dev->caps.qp1_tunnel[k] = dev->caps.qp1_proxy[k] + 8 * MLX4_MFUNC_MAX; + } + } + + + err = mlx4_CONF_SPECIAL_QP(dev, dev->phys_caps.base_sqpn); + if (err) + goto err_mem; + + return err; + +err_mem: + kfree(dev->caps.qp0_tunnel); + kfree(dev->caps.qp0_proxy); + kfree(dev->caps.qp1_tunnel); + kfree(dev->caps.qp1_proxy); + dev->caps.qp0_tunnel = dev->caps.qp0_proxy = + dev->caps.qp1_tunnel = dev->caps.qp1_proxy = NULL; + mlx4_cleanup_qp_zones(dev); + return err; +} + +void mlx4_cleanup_qp_table(struct mlx4_dev *dev) +{ + if (mlx4_is_slave(dev)) + return; + + mlx4_CONF_SPECIAL_QP(dev, 0); + + mlx4_cleanup_qp_zones(dev); +} + +int mlx4_qp_query(struct mlx4_dev *dev, struct mlx4_qp *qp, + struct mlx4_qp_context *context) +{ + struct mlx4_cmd_mailbox *mailbox; + int err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + err = mlx4_cmd_box(dev, 0, mailbox->dma, qp->qpn, 0, + MLX4_CMD_QUERY_QP, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); + if (!err) + memcpy(context, mailbox->buf + 8, sizeof *context); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_qp_query); + +int mlx4_qp_to_ready(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + struct mlx4_qp_context *context, + struct mlx4_qp *qp, enum mlx4_qp_state *qp_state) +{ + int err; + int i; + enum mlx4_qp_state states[] = { + MLX4_QP_STATE_RST, + MLX4_QP_STATE_INIT, + MLX4_QP_STATE_RTR, + MLX4_QP_STATE_RTS + }; + + for (i = 0; i < ARRAY_SIZE(states) - 1; i++) { + context->flags &= cpu_to_be32(~(0xf << 28)); + context->flags |= cpu_to_be32(states[i + 1] << 28); + if (states[i + 1] != MLX4_QP_STATE_RTR) + context->params2 &= ~MLX4_QP_BIT_FPP; + err = mlx4_qp_modify(dev, mtt, states[i], states[i + 1], + context, 0, 0, qp); + if (err) { + mlx4_err(dev, "Failed to bring QP to state: %d with error: %d\n", + states[i + 1], err); + return err; + } + + *qp_state = states[i + 1]; + } + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_qp_to_ready); + +u16 mlx4_qp_roce_entropy(struct mlx4_dev *dev, u32 qpn) +{ + struct mlx4_qp_context context; + struct mlx4_qp qp; + int err; + + qp.qpn = qpn; + err = mlx4_qp_query(dev, &qp, &context); + if (!err) { + u32 dest_qpn = be32_to_cpu(context.remote_qpn) & 0xffffff; + u16 folded_dst = folded_qp(dest_qpn); + u16 folded_src = folded_qp(qpn); + + return (dest_qpn != qpn) ? + ((folded_dst ^ folded_src) | 0xC000) : + folded_src | 0xC000; + } + return 0xdead; +} Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_qp.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_reset.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_reset.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_reset.c (revision 329159) @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include + +#include "mlx4.h" + +int mlx4_reset(struct mlx4_dev *dev) +{ + void __iomem *reset; + u32 *hca_header = NULL; + int pcie_cap; + u16 devctl; + u16 linkctl; + u16 vendor; + unsigned long end; + u32 sem; + int i; + int err = 0; + +#define MLX4_RESET_BASE 0xf0000 +#define MLX4_RESET_SIZE 0x400 +#define MLX4_SEM_OFFSET 0x3fc +#define MLX4_RESET_OFFSET 0x10 +#define MLX4_RESET_VALUE swab32(1) + +#define MLX4_SEM_TIMEOUT_JIFFIES (10 * HZ) +#define MLX4_RESET_TIMEOUT_JIFFIES (2 * HZ) + + /* + * Reset the chip. This is somewhat ugly because we have to + * save off the PCI header before reset and then restore it + * after the chip reboots. We skip config space offsets 22 + * and 23 since those have a special meaning. + */ + + /* Do we need to save off the full 4K PCI Express header?? */ + hca_header = kmalloc(256, GFP_KERNEL); + if (!hca_header) { + err = -ENOMEM; + mlx4_err(dev, "Couldn't allocate memory to save HCA PCI header, aborting\n"); + goto out; + } + + pcie_cap = pci_pcie_cap(dev->persist->pdev); + + for (i = 0; i < 64; ++i) { + if (i == 22 || i == 23) + continue; + if (pci_read_config_dword(dev->persist->pdev, i * 4, + hca_header + i)) { + err = -ENODEV; + mlx4_err(dev, "Couldn't save HCA PCI header, aborting\n"); + goto out; + } + } + + reset = ioremap(pci_resource_start(dev->persist->pdev, 0) + + MLX4_RESET_BASE, + MLX4_RESET_SIZE); + if (!reset) { + err = -ENOMEM; + mlx4_err(dev, "Couldn't map HCA reset register, aborting\n"); + goto out; + } + + /* grab HW semaphore to lock out flash updates */ + end = jiffies + MLX4_SEM_TIMEOUT_JIFFIES; + do { + sem = readl(reset + MLX4_SEM_OFFSET); + if (!sem) + break; + + msleep(1); + } while (time_before(jiffies, end)); + + if (sem) { + mlx4_err(dev, "Failed to obtain HW semaphore, aborting\n"); + err = -EAGAIN; + iounmap(reset); + goto out; + } + + /* actually hit reset */ + writel(MLX4_RESET_VALUE, reset + MLX4_RESET_OFFSET); + iounmap(reset); + + /* Docs say to wait one second before accessing device */ + msleep(1000); + + end = jiffies + MLX4_RESET_TIMEOUT_JIFFIES; + do { + if (!pci_read_config_word(dev->persist->pdev, PCI_VENDOR_ID, + &vendor) && vendor != 0xffff) + break; + + msleep(1); + } while (time_before(jiffies, end)); + + if (vendor == 0xffff) { + err = -ENODEV; + mlx4_err(dev, "PCI device did not come back after reset, aborting\n"); + goto out; + } + + /* Now restore the PCI headers */ + if (pcie_cap) { + devctl = hca_header[(pcie_cap + PCI_EXP_DEVCTL) / 4]; + if (pcie_capability_write_word(dev->persist->pdev, + PCI_EXP_DEVCTL, + devctl)) { + err = -ENODEV; + mlx4_err(dev, "Couldn't restore HCA PCI Express Device Control register, aborting\n"); + goto out; + } + linkctl = hca_header[(pcie_cap + PCI_EXP_LNKCTL) / 4]; + if (pcie_capability_write_word(dev->persist->pdev, + PCI_EXP_LNKCTL, + linkctl)) { + err = -ENODEV; + mlx4_err(dev, "Couldn't restore HCA PCI Express Link control register, aborting\n"); + goto out; + } + } + + for (i = 0; i < 16; ++i) { + if (i * 4 == PCI_COMMAND) + continue; + + if (pci_write_config_dword(dev->persist->pdev, i * 4, + hca_header[i])) { + err = -ENODEV; + mlx4_err(dev, "Couldn't restore HCA reg %x, aborting\n", + i); + goto out; + } + } + + if (pci_write_config_dword(dev->persist->pdev, PCI_COMMAND, + hca_header[PCI_COMMAND / 4])) { + err = -ENODEV; + mlx4_err(dev, "Couldn't restore HCA COMMAND, aborting\n"); + goto out; + } + +out: + kfree(hca_header); + + return err; +} Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_reset.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_resource_tracker.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_resource_tracker.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_resource_tracker.c (revision 329159) @@ -0,0 +1,5383 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. + * All rights reserved. + * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mlx4.h" +#include "fw.h" + +#define MLX4_MAC_VALID (1ull << 63) +#define MLX4_PF_COUNTERS_PER_PORT 2 +#define MLX4_VF_COUNTERS_PER_PORT 1 + +struct mac_res { + struct list_head list; + u64 mac; + int ref_count; + u8 smac_index; + u8 port; +}; + +struct vlan_res { + struct list_head list; + u16 vlan; + int ref_count; + int vlan_index; + u8 port; +}; + +struct res_common { + struct list_head list; + struct rb_node node; + u64 res_id; + int owner; + int state; + int from_state; + int to_state; + int removing; +}; + +enum { + RES_ANY_BUSY = 1 +}; + +struct res_gid { + struct list_head list; + u8 gid[16]; + enum mlx4_protocol prot; + enum mlx4_steer_type steer; + u64 reg_id; +}; + +enum res_qp_states { + RES_QP_BUSY = RES_ANY_BUSY, + + /* QP number was allocated */ + RES_QP_RESERVED, + + /* ICM memory for QP context was mapped */ + RES_QP_MAPPED, + + /* QP is in hw ownership */ + RES_QP_HW +}; + +struct res_qp { + struct res_common com; + struct res_mtt *mtt; + struct res_cq *rcq; + struct res_cq *scq; + struct res_srq *srq; + struct list_head mcg_list; + spinlock_t mcg_spl; + int local_qpn; + atomic_t ref_count; + u32 qpc_flags; + /* saved qp params before VST enforcement in order to restore on VGT */ + u8 sched_queue; + __be32 param3; + u8 vlan_control; + u8 fvl_rx; + u8 pri_path_fl; + u8 vlan_index; + u8 feup; +}; + +enum res_mtt_states { + RES_MTT_BUSY = RES_ANY_BUSY, + RES_MTT_ALLOCATED, +}; + +static inline const char *mtt_states_str(enum res_mtt_states state) +{ + switch (state) { + case RES_MTT_BUSY: return "RES_MTT_BUSY"; + case RES_MTT_ALLOCATED: return "RES_MTT_ALLOCATED"; + default: return "Unknown"; + } +} + +struct res_mtt { + struct res_common com; + int order; + atomic_t ref_count; +}; + +enum res_mpt_states { + RES_MPT_BUSY = RES_ANY_BUSY, + RES_MPT_RESERVED, + RES_MPT_MAPPED, + RES_MPT_HW, +}; + +struct res_mpt { + struct res_common com; + struct res_mtt *mtt; + int key; +}; + +enum res_eq_states { + RES_EQ_BUSY = RES_ANY_BUSY, + RES_EQ_RESERVED, + RES_EQ_HW, +}; + +struct res_eq { + struct res_common com; + struct res_mtt *mtt; +}; + +enum res_cq_states { + RES_CQ_BUSY = RES_ANY_BUSY, + RES_CQ_ALLOCATED, + RES_CQ_HW, +}; + +struct res_cq { + struct res_common com; + struct res_mtt *mtt; + atomic_t ref_count; +}; + +enum res_srq_states { + RES_SRQ_BUSY = RES_ANY_BUSY, + RES_SRQ_ALLOCATED, + RES_SRQ_HW, +}; + +struct res_srq { + struct res_common com; + struct res_mtt *mtt; + struct res_cq *cq; + atomic_t ref_count; +}; + +enum res_counter_states { + RES_COUNTER_BUSY = RES_ANY_BUSY, + RES_COUNTER_ALLOCATED, +}; + +struct res_counter { + struct res_common com; + int port; +}; + +enum res_xrcdn_states { + RES_XRCD_BUSY = RES_ANY_BUSY, + RES_XRCD_ALLOCATED, +}; + +struct res_xrcdn { + struct res_common com; + int port; +}; + +enum res_fs_rule_states { + RES_FS_RULE_BUSY = RES_ANY_BUSY, + RES_FS_RULE_ALLOCATED, +}; + +struct res_fs_rule { + struct res_common com; + int qpn; + /* VF DMFS mbox with port flipped */ + void *mirr_mbox; + /* > 0 --> apply mirror when getting into HA mode */ + /* = 0 --> un-apply mirror when getting out of HA mode */ + u32 mirr_mbox_size; + struct list_head mirr_list; + u64 mirr_rule_id; +}; + +static void *res_tracker_lookup(struct rb_root *root, u64 res_id) +{ + struct rb_node *node = root->rb_node; + + while (node) { + struct res_common *res = container_of(node, struct res_common, + node); + + if (res_id < res->res_id) + node = node->rb_left; + else if (res_id > res->res_id) + node = node->rb_right; + else + return res; + } + return NULL; +} + +static int res_tracker_insert(struct rb_root *root, struct res_common *res) +{ + struct rb_node **new = &(root->rb_node), *parent = NULL; + + /* Figure out where to put new node */ + while (*new) { + struct res_common *this = container_of(*new, struct res_common, + node); + + parent = *new; + if (res->res_id < this->res_id) + new = &((*new)->rb_left); + else if (res->res_id > this->res_id) + new = &((*new)->rb_right); + else + return -EEXIST; + } + + /* Add new node and rebalance tree. */ + rb_link_node(&res->node, parent, new); + rb_insert_color(&res->node, root); + + return 0; +} + +enum qp_transition { + QP_TRANS_INIT2RTR, + QP_TRANS_RTR2RTS, + QP_TRANS_RTS2RTS, + QP_TRANS_SQERR2RTS, + QP_TRANS_SQD2SQD, + QP_TRANS_SQD2RTS +}; + +/* For Debug uses */ +static const char *resource_str(enum mlx4_resource rt) +{ + switch (rt) { + case RES_QP: return "RES_QP"; + case RES_CQ: return "RES_CQ"; + case RES_SRQ: return "RES_SRQ"; + case RES_MPT: return "RES_MPT"; + case RES_MTT: return "RES_MTT"; + case RES_MAC: return "RES_MAC"; + case RES_VLAN: return "RES_VLAN"; + case RES_EQ: return "RES_EQ"; + case RES_COUNTER: return "RES_COUNTER"; + case RES_FS_RULE: return "RES_FS_RULE"; + case RES_XRCD: return "RES_XRCD"; + default: return "Unknown resource type !!!"; + }; +} + +static void rem_slave_vlans(struct mlx4_dev *dev, int slave); +static inline int mlx4_grant_resource(struct mlx4_dev *dev, int slave, + enum mlx4_resource res_type, int count, + int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct resource_allocator *res_alloc = + &priv->mfunc.master.res_tracker.res_alloc[res_type]; + int err = -EINVAL; + int allocated, free, reserved, guaranteed, from_free; + int from_rsvd; + + if (slave > dev->persist->num_vfs) + return -EINVAL; + + spin_lock(&res_alloc->alloc_lock); + allocated = (port > 0) ? + res_alloc->allocated[(port - 1) * + (dev->persist->num_vfs + 1) + slave] : + res_alloc->allocated[slave]; + free = (port > 0) ? res_alloc->res_port_free[port - 1] : + res_alloc->res_free; + reserved = (port > 0) ? res_alloc->res_port_rsvd[port - 1] : + res_alloc->res_reserved; + guaranteed = res_alloc->guaranteed[slave]; + + if (allocated + count > res_alloc->quota[slave]) { + mlx4_warn(dev, "VF %d port %d res %s: quota exceeded, count %d alloc %d quota %d\n", + slave, port, resource_str(res_type), count, + allocated, res_alloc->quota[slave]); + goto out; + } + + if (allocated + count <= guaranteed) { + err = 0; + from_rsvd = count; + } else { + /* portion may need to be obtained from free area */ + if (guaranteed - allocated > 0) + from_free = count - (guaranteed - allocated); + else + from_free = count; + + from_rsvd = count - from_free; + + if (free - from_free >= reserved) + err = 0; + else + mlx4_warn(dev, "VF %d port %d res %s: free pool empty, free %d from_free %d rsvd %d\n", + slave, port, resource_str(res_type), free, + from_free, reserved); + } + + if (!err) { + /* grant the request */ + if (port > 0) { + res_alloc->allocated[(port - 1) * + (dev->persist->num_vfs + 1) + slave] += count; + res_alloc->res_port_free[port - 1] -= count; + res_alloc->res_port_rsvd[port - 1] -= from_rsvd; + } else { + res_alloc->allocated[slave] += count; + res_alloc->res_free -= count; + res_alloc->res_reserved -= from_rsvd; + } + } + +out: + spin_unlock(&res_alloc->alloc_lock); + return err; +} + +static inline void mlx4_release_resource(struct mlx4_dev *dev, int slave, + enum mlx4_resource res_type, int count, + int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct resource_allocator *res_alloc = + &priv->mfunc.master.res_tracker.res_alloc[res_type]; + int allocated, guaranteed, from_rsvd; + + if (slave > dev->persist->num_vfs) + return; + + spin_lock(&res_alloc->alloc_lock); + + allocated = (port > 0) ? + res_alloc->allocated[(port - 1) * + (dev->persist->num_vfs + 1) + slave] : + res_alloc->allocated[slave]; + guaranteed = res_alloc->guaranteed[slave]; + + if (allocated - count >= guaranteed) { + from_rsvd = 0; + } else { + /* portion may need to be returned to reserved area */ + if (allocated - guaranteed > 0) + from_rsvd = count - (allocated - guaranteed); + else + from_rsvd = count; + } + + if (port > 0) { + res_alloc->allocated[(port - 1) * + (dev->persist->num_vfs + 1) + slave] -= count; + res_alloc->res_port_free[port - 1] += count; + res_alloc->res_port_rsvd[port - 1] += from_rsvd; + } else { + res_alloc->allocated[slave] -= count; + res_alloc->res_free += count; + res_alloc->res_reserved += from_rsvd; + } + + spin_unlock(&res_alloc->alloc_lock); + return; +} + +static inline void initialize_res_quotas(struct mlx4_dev *dev, + struct resource_allocator *res_alloc, + enum mlx4_resource res_type, + int vf, int num_instances) +{ + res_alloc->guaranteed[vf] = num_instances / + (2 * (dev->persist->num_vfs + 1)); + res_alloc->quota[vf] = (num_instances / 2) + res_alloc->guaranteed[vf]; + if (vf == mlx4_master_func_num(dev)) { + res_alloc->res_free = num_instances; + if (res_type == RES_MTT) { + /* reserved mtts will be taken out of the PF allocation */ + res_alloc->res_free += dev->caps.reserved_mtts; + res_alloc->guaranteed[vf] += dev->caps.reserved_mtts; + res_alloc->quota[vf] += dev->caps.reserved_mtts; + } + } +} + +void mlx4_init_quotas(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int pf; + + /* quotas for VFs are initialized in mlx4_slave_cap */ + if (mlx4_is_slave(dev)) + return; + + if (!mlx4_is_mfunc(dev)) { + dev->quotas.qp = dev->caps.num_qps - dev->caps.reserved_qps - + mlx4_num_reserved_sqps(dev); + dev->quotas.cq = dev->caps.num_cqs - dev->caps.reserved_cqs; + dev->quotas.srq = dev->caps.num_srqs - dev->caps.reserved_srqs; + dev->quotas.mtt = dev->caps.num_mtts - dev->caps.reserved_mtts; + dev->quotas.mpt = dev->caps.num_mpts - dev->caps.reserved_mrws; + return; + } + + pf = mlx4_master_func_num(dev); + dev->quotas.qp = + priv->mfunc.master.res_tracker.res_alloc[RES_QP].quota[pf]; + dev->quotas.cq = + priv->mfunc.master.res_tracker.res_alloc[RES_CQ].quota[pf]; + dev->quotas.srq = + priv->mfunc.master.res_tracker.res_alloc[RES_SRQ].quota[pf]; + dev->quotas.mtt = + priv->mfunc.master.res_tracker.res_alloc[RES_MTT].quota[pf]; + dev->quotas.mpt = + priv->mfunc.master.res_tracker.res_alloc[RES_MPT].quota[pf]; +} + +static int get_max_gauranteed_vfs_counter(struct mlx4_dev *dev) +{ + /* reduce the sink counter */ + return (dev->caps.max_counters - 1 - + (MLX4_PF_COUNTERS_PER_PORT * MLX4_MAX_PORTS)) + / MLX4_MAX_PORTS; +} + +int mlx4_init_resource_tracker(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int i, j; + int t; + int max_vfs_guarantee_counter = get_max_gauranteed_vfs_counter(dev); + + priv->mfunc.master.res_tracker.slave_list = + kzalloc(dev->num_slaves * sizeof(struct slave_list), + GFP_KERNEL); + if (!priv->mfunc.master.res_tracker.slave_list) + return -ENOMEM; + + for (i = 0 ; i < dev->num_slaves; i++) { + for (t = 0; t < MLX4_NUM_OF_RESOURCE_TYPE; ++t) + INIT_LIST_HEAD(&priv->mfunc.master.res_tracker. + slave_list[i].res_list[t]); + mutex_init(&priv->mfunc.master.res_tracker.slave_list[i].mutex); + } + + mlx4_dbg(dev, "Started init_resource_tracker: %ld slaves\n", + dev->num_slaves); + for (i = 0 ; i < MLX4_NUM_OF_RESOURCE_TYPE; i++) + priv->mfunc.master.res_tracker.res_tree[i] = RB_ROOT; + + for (i = 0; i < MLX4_NUM_OF_RESOURCE_TYPE; i++) { + struct resource_allocator *res_alloc = + &priv->mfunc.master.res_tracker.res_alloc[i]; + res_alloc->quota = kmalloc((dev->persist->num_vfs + 1) * + sizeof(int), GFP_KERNEL); + res_alloc->guaranteed = kmalloc((dev->persist->num_vfs + 1) * + sizeof(int), GFP_KERNEL); + if (i == RES_MAC || i == RES_VLAN) + res_alloc->allocated = kzalloc(MLX4_MAX_PORTS * + (dev->persist->num_vfs + + 1) * + sizeof(int), GFP_KERNEL); + else + res_alloc->allocated = kzalloc((dev->persist-> + num_vfs + 1) * + sizeof(int), GFP_KERNEL); + /* Reduce the sink counter */ + if (i == RES_COUNTER) + res_alloc->res_free = dev->caps.max_counters - 1; + + if (!res_alloc->quota || !res_alloc->guaranteed || + !res_alloc->allocated) + goto no_mem_err; + + spin_lock_init(&res_alloc->alloc_lock); + for (t = 0; t < dev->persist->num_vfs + 1; t++) { + struct mlx4_active_ports actv_ports = + mlx4_get_active_ports(dev, t); + switch (i) { + case RES_QP: + initialize_res_quotas(dev, res_alloc, RES_QP, + t, dev->caps.num_qps - + dev->caps.reserved_qps - + mlx4_num_reserved_sqps(dev)); + break; + case RES_CQ: + initialize_res_quotas(dev, res_alloc, RES_CQ, + t, dev->caps.num_cqs - + dev->caps.reserved_cqs); + break; + case RES_SRQ: + initialize_res_quotas(dev, res_alloc, RES_SRQ, + t, dev->caps.num_srqs - + dev->caps.reserved_srqs); + break; + case RES_MPT: + initialize_res_quotas(dev, res_alloc, RES_MPT, + t, dev->caps.num_mpts - + dev->caps.reserved_mrws); + break; + case RES_MTT: + initialize_res_quotas(dev, res_alloc, RES_MTT, + t, dev->caps.num_mtts - + dev->caps.reserved_mtts); + break; + case RES_MAC: + if (t == mlx4_master_func_num(dev)) { + int max_vfs_pport = 0; + /* Calculate the max vfs per port for */ + /* both ports. */ + for (j = 0; j < dev->caps.num_ports; + j++) { + struct mlx4_slaves_pport slaves_pport = + mlx4_phys_to_slaves_pport(dev, j + 1); + unsigned current_slaves = + bitmap_weight(slaves_pport.slaves, + dev->caps.num_ports) - 1; + if (max_vfs_pport < current_slaves) + max_vfs_pport = + current_slaves; + } + res_alloc->quota[t] = + MLX4_MAX_MAC_NUM - + 2 * max_vfs_pport; + res_alloc->guaranteed[t] = 2; + for (j = 0; j < MLX4_MAX_PORTS; j++) + res_alloc->res_port_free[j] = + MLX4_MAX_MAC_NUM; + } else { + res_alloc->quota[t] = MLX4_MAX_MAC_NUM; + res_alloc->guaranteed[t] = 2; + } + break; + case RES_VLAN: + if (t == mlx4_master_func_num(dev)) { + res_alloc->quota[t] = MLX4_MAX_VLAN_NUM; + res_alloc->guaranteed[t] = MLX4_MAX_VLAN_NUM / 2; + for (j = 0; j < MLX4_MAX_PORTS; j++) + res_alloc->res_port_free[j] = + res_alloc->quota[t]; + } else { + res_alloc->quota[t] = MLX4_MAX_VLAN_NUM / 2; + res_alloc->guaranteed[t] = 0; + } + break; + case RES_COUNTER: + res_alloc->quota[t] = dev->caps.max_counters; + if (t == mlx4_master_func_num(dev)) + res_alloc->guaranteed[t] = + MLX4_PF_COUNTERS_PER_PORT * + MLX4_MAX_PORTS; + else if (t <= max_vfs_guarantee_counter) + res_alloc->guaranteed[t] = + MLX4_VF_COUNTERS_PER_PORT * + MLX4_MAX_PORTS; + else + res_alloc->guaranteed[t] = 0; + res_alloc->res_free -= res_alloc->guaranteed[t]; + break; + default: + break; + } + if (i == RES_MAC || i == RES_VLAN) { + for (j = 0; j < dev->caps.num_ports; j++) + if (test_bit(j, actv_ports.ports)) + res_alloc->res_port_rsvd[j] += + res_alloc->guaranteed[t]; + } else { + res_alloc->res_reserved += res_alloc->guaranteed[t]; + } + } + } + spin_lock_init(&priv->mfunc.master.res_tracker.lock); + return 0; + +no_mem_err: + for (i = 0; i < MLX4_NUM_OF_RESOURCE_TYPE; i++) { + kfree(priv->mfunc.master.res_tracker.res_alloc[i].allocated); + priv->mfunc.master.res_tracker.res_alloc[i].allocated = NULL; + kfree(priv->mfunc.master.res_tracker.res_alloc[i].guaranteed); + priv->mfunc.master.res_tracker.res_alloc[i].guaranteed = NULL; + kfree(priv->mfunc.master.res_tracker.res_alloc[i].quota); + priv->mfunc.master.res_tracker.res_alloc[i].quota = NULL; + } + return -ENOMEM; +} + +void mlx4_free_resource_tracker(struct mlx4_dev *dev, + enum mlx4_res_tracker_free_type type) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int i; + + if (priv->mfunc.master.res_tracker.slave_list) { + if (type != RES_TR_FREE_STRUCTS_ONLY) { + for (i = 0; i < dev->num_slaves; i++) { + if (type == RES_TR_FREE_ALL || + dev->caps.function != i) + mlx4_delete_all_resources_for_slave(dev, i); + } + /* free master's vlans */ + i = dev->caps.function; + mlx4_reset_roce_gids(dev, i); + mutex_lock(&priv->mfunc.master.res_tracker.slave_list[i].mutex); + rem_slave_vlans(dev, i); + mutex_unlock(&priv->mfunc.master.res_tracker.slave_list[i].mutex); + } + + if (type != RES_TR_FREE_SLAVES_ONLY) { + for (i = 0; i < MLX4_NUM_OF_RESOURCE_TYPE; i++) { + kfree(priv->mfunc.master.res_tracker.res_alloc[i].allocated); + priv->mfunc.master.res_tracker.res_alloc[i].allocated = NULL; + kfree(priv->mfunc.master.res_tracker.res_alloc[i].guaranteed); + priv->mfunc.master.res_tracker.res_alloc[i].guaranteed = NULL; + kfree(priv->mfunc.master.res_tracker.res_alloc[i].quota); + priv->mfunc.master.res_tracker.res_alloc[i].quota = NULL; + } + kfree(priv->mfunc.master.res_tracker.slave_list); + priv->mfunc.master.res_tracker.slave_list = NULL; + } + } +} + +static void update_pkey_index(struct mlx4_dev *dev, int slave, + struct mlx4_cmd_mailbox *inbox) +{ + u8 sched = *(u8 *)(inbox->buf + 64); + u8 orig_index = *(u8 *)(inbox->buf + 35); + u8 new_index; + struct mlx4_priv *priv = mlx4_priv(dev); + int port; + + port = (sched >> 6 & 1) + 1; + + new_index = priv->virt2phys_pkey[slave][port - 1][orig_index]; + *(u8 *)(inbox->buf + 35) = new_index; +} + +static void update_gid(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *inbox, + u8 slave) +{ + struct mlx4_qp_context *qp_ctx = inbox->buf + 8; + enum mlx4_qp_optpar optpar = be32_to_cpu(*(__be32 *) inbox->buf); + u32 ts = (be32_to_cpu(qp_ctx->flags) >> 16) & 0xff; + int port; + + if (MLX4_QP_ST_UD == ts) { + port = (qp_ctx->pri_path.sched_queue >> 6 & 1) + 1; + if (mlx4_is_eth(dev, port)) + qp_ctx->pri_path.mgid_index = + mlx4_get_base_gid_ix(dev, slave, port) | 0x80; + else + qp_ctx->pri_path.mgid_index = slave | 0x80; + + } else if (MLX4_QP_ST_RC == ts || MLX4_QP_ST_XRC == ts || MLX4_QP_ST_UC == ts) { + if (optpar & MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH) { + port = (qp_ctx->pri_path.sched_queue >> 6 & 1) + 1; + if (mlx4_is_eth(dev, port)) { + qp_ctx->pri_path.mgid_index += + mlx4_get_base_gid_ix(dev, slave, port); + qp_ctx->pri_path.mgid_index &= 0x7f; + } else { + qp_ctx->pri_path.mgid_index = slave & 0x7F; + } + } + if (optpar & MLX4_QP_OPTPAR_ALT_ADDR_PATH) { + port = (qp_ctx->alt_path.sched_queue >> 6 & 1) + 1; + if (mlx4_is_eth(dev, port)) { + qp_ctx->alt_path.mgid_index += + mlx4_get_base_gid_ix(dev, slave, port); + qp_ctx->alt_path.mgid_index &= 0x7f; + } else { + qp_ctx->alt_path.mgid_index = slave & 0x7F; + } + } + } +} + +static int handle_counter(struct mlx4_dev *dev, struct mlx4_qp_context *qpc, + u8 slave, int port); + +static int update_vport_qp_param(struct mlx4_dev *dev, + struct mlx4_cmd_mailbox *inbox, + u8 slave, u32 qpn) +{ + struct mlx4_qp_context *qpc = inbox->buf + 8; + struct mlx4_vport_oper_state *vp_oper; + struct mlx4_priv *priv; + u32 qp_type; + int port, err = 0; + + port = (qpc->pri_path.sched_queue & 0x40) ? 2 : 1; + priv = mlx4_priv(dev); + vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; + qp_type = (be32_to_cpu(qpc->flags) >> 16) & 0xff; + + err = handle_counter(dev, qpc, slave, port); + if (err) + goto out; + + if (MLX4_VGT != vp_oper->state.default_vlan) { + /* the reserved QPs (special, proxy, tunnel) + * do not operate over vlans + */ + if (mlx4_is_qp_reserved(dev, qpn)) + return 0; + + /* force strip vlan by clear vsd, MLX QP refers to Raw Ethernet */ + if (qp_type == MLX4_QP_ST_UD || + (qp_type == MLX4_QP_ST_MLX && mlx4_is_eth(dev, port))) { + if (dev->caps.bmme_flags & MLX4_BMME_FLAG_VSD_INIT2RTR) { + *(__be32 *)inbox->buf = + cpu_to_be32(be32_to_cpu(*(__be32 *)inbox->buf) | + MLX4_QP_OPTPAR_VLAN_STRIPPING); + qpc->param3 &= ~cpu_to_be32(MLX4_STRIP_VLAN); + } else { + struct mlx4_update_qp_params params = {.flags = 0}; + + err = mlx4_update_qp(dev, qpn, MLX4_UPDATE_QP_VSD, ¶ms); + if (err) + goto out; + } + } + + /* preserve IF_COUNTER flag */ + qpc->pri_path.vlan_control &= + MLX4_CTRL_ETH_SRC_CHECK_IF_COUNTER; + if (1 /*vp_oper->state.link_state == IFLA_VF_LINK_STATE_DISABLE*/ && + dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_UPDATE_QP) { + qpc->pri_path.vlan_control |= + MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED | + MLX4_VLAN_CTRL_ETH_TX_BLOCK_PRIO_TAGGED | + MLX4_VLAN_CTRL_ETH_TX_BLOCK_UNTAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED; + } else if (0 != vp_oper->state.default_vlan) { + if (vp_oper->state.vlan_proto == htons(ETH_P_8021AD)) { + /* vst QinQ should block untagged on TX, + * but cvlan is in payload and phv is set so + * hw see it as untagged. Block tagged instead. + */ + qpc->pri_path.vlan_control |= + MLX4_VLAN_CTRL_ETH_TX_BLOCK_PRIO_TAGGED | + MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED; + } else { /* vst 802.1Q */ + qpc->pri_path.vlan_control |= + MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED; + } + } else { /* priority tagged */ + qpc->pri_path.vlan_control |= + MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED; + } + + qpc->pri_path.fvl_rx |= MLX4_FVL_RX_FORCE_ETH_VLAN; + qpc->pri_path.vlan_index = vp_oper->vlan_idx; + qpc->pri_path.fl |= MLX4_FL_ETH_HIDE_CQE_VLAN; + if (vp_oper->state.vlan_proto == htons(ETH_P_8021AD)) + qpc->pri_path.fl |= MLX4_FL_SV; + else + qpc->pri_path.fl |= MLX4_FL_CV; + qpc->pri_path.feup |= MLX4_FEUP_FORCE_ETH_UP | MLX4_FVL_FORCE_ETH_VLAN; + qpc->pri_path.sched_queue &= 0xC7; + qpc->pri_path.sched_queue |= (vp_oper->state.default_qos) << 3; + qpc->qos_vport = vp_oper->state.qos_vport; + } + if (vp_oper->state.spoofchk) { + qpc->pri_path.feup |= MLX4_FSM_FORCE_ETH_SRC_MAC; + qpc->pri_path.grh_mylmc = (0x80 & qpc->pri_path.grh_mylmc) + vp_oper->mac_idx; + } +out: + return err; +} + +static int mpt_mask(struct mlx4_dev *dev) +{ + return dev->caps.num_mpts - 1; +} + +static void *find_res(struct mlx4_dev *dev, u64 res_id, + enum mlx4_resource type) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + return res_tracker_lookup(&priv->mfunc.master.res_tracker.res_tree[type], + res_id); +} + +static int get_res(struct mlx4_dev *dev, int slave, u64 res_id, + enum mlx4_resource type, + void *res) +{ + struct res_common *r; + int err = 0; + + spin_lock_irq(mlx4_tlock(dev)); + r = find_res(dev, res_id, type); + if (!r) { + err = -ENONET; + goto exit; + } + + if (r->state == RES_ANY_BUSY) { + err = -EBUSY; + goto exit; + } + + if (r->owner != slave) { + err = -EPERM; + goto exit; + } + + r->from_state = r->state; + r->state = RES_ANY_BUSY; + + if (res) + *((struct res_common **)res) = r; + +exit: + spin_unlock_irq(mlx4_tlock(dev)); + return err; +} + +int mlx4_get_slave_from_resource_id(struct mlx4_dev *dev, + enum mlx4_resource type, + u64 res_id, int *slave) +{ + + struct res_common *r; + int err = -ENOENT; + int id = res_id; + + if (type == RES_QP) + id &= 0x7fffff; + spin_lock(mlx4_tlock(dev)); + + r = find_res(dev, id, type); + if (r) { + *slave = r->owner; + err = 0; + } + spin_unlock(mlx4_tlock(dev)); + + return err; +} + +static void put_res(struct mlx4_dev *dev, int slave, u64 res_id, + enum mlx4_resource type) +{ + struct res_common *r; + + spin_lock_irq(mlx4_tlock(dev)); + r = find_res(dev, res_id, type); + if (r) + r->state = r->from_state; + spin_unlock_irq(mlx4_tlock(dev)); +} + +static int counter_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param, int port); + +static int handle_existing_counter(struct mlx4_dev *dev, u8 slave, int port, + int counter_index) +{ + struct res_common *r; + struct res_counter *counter; + int ret = 0; + + if (counter_index == MLX4_SINK_COUNTER_INDEX(dev)) + return ret; + + spin_lock_irq(mlx4_tlock(dev)); + r = find_res(dev, counter_index, RES_COUNTER); + if (!r || r->owner != slave) { + ret = -EINVAL; + } else { + counter = container_of(r, struct res_counter, com); + if (!counter->port) + counter->port = port; + } + + spin_unlock_irq(mlx4_tlock(dev)); + return ret; +} + +static int handle_unexisting_counter(struct mlx4_dev *dev, + struct mlx4_qp_context *qpc, u8 slave, + int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_common *tmp; + struct res_counter *counter; + u64 counter_idx = MLX4_SINK_COUNTER_INDEX(dev); + int err = 0; + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry(tmp, + &tracker->slave_list[slave].res_list[RES_COUNTER], + list) { + counter = container_of(tmp, struct res_counter, com); + if (port == counter->port) { + qpc->pri_path.counter_index = counter->com.res_id; + spin_unlock_irq(mlx4_tlock(dev)); + return 0; + } + } + spin_unlock_irq(mlx4_tlock(dev)); + + /* No existing counter, need to allocate a new counter */ + err = counter_alloc_res(dev, slave, RES_OP_RESERVE, 0, 0, &counter_idx, + port); + if (err == -ENOENT) { + err = 0; + } else if (err && err != -ENOSPC) { + mlx4_err(dev, "%s: failed to create new counter for slave %d err %d\n", + __func__, slave, err); + } else { + qpc->pri_path.counter_index = counter_idx; + mlx4_dbg(dev, "%s: alloc new counter for slave %d index %d\n", + __func__, slave, qpc->pri_path.counter_index); + err = 0; + } + + return err; +} + +static int handle_counter(struct mlx4_dev *dev, struct mlx4_qp_context *qpc, + u8 slave, int port) +{ + if (qpc->pri_path.counter_index != MLX4_SINK_COUNTER_INDEX(dev)) + return handle_existing_counter(dev, slave, port, + qpc->pri_path.counter_index); + + return handle_unexisting_counter(dev, qpc, slave, port); +} + +static struct res_common *alloc_qp_tr(int id) +{ + struct res_qp *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->com.state = RES_QP_RESERVED; + ret->local_qpn = id; + INIT_LIST_HEAD(&ret->mcg_list); + spin_lock_init(&ret->mcg_spl); + atomic_set(&ret->ref_count, 0); + + return &ret->com; +} + +static struct res_common *alloc_mtt_tr(int id, int order) +{ + struct res_mtt *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->order = order; + ret->com.state = RES_MTT_ALLOCATED; + atomic_set(&ret->ref_count, 0); + + return &ret->com; +} + +static struct res_common *alloc_mpt_tr(int id, int key) +{ + struct res_mpt *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->com.state = RES_MPT_RESERVED; + ret->key = key; + + return &ret->com; +} + +static struct res_common *alloc_eq_tr(int id) +{ + struct res_eq *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->com.state = RES_EQ_RESERVED; + + return &ret->com; +} + +static struct res_common *alloc_cq_tr(int id) +{ + struct res_cq *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->com.state = RES_CQ_ALLOCATED; + atomic_set(&ret->ref_count, 0); + + return &ret->com; +} + +static struct res_common *alloc_srq_tr(int id) +{ + struct res_srq *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->com.state = RES_SRQ_ALLOCATED; + atomic_set(&ret->ref_count, 0); + + return &ret->com; +} + +static struct res_common *alloc_counter_tr(int id, int port) +{ + struct res_counter *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->com.state = RES_COUNTER_ALLOCATED; + ret->port = port; + + return &ret->com; +} + +static struct res_common *alloc_xrcdn_tr(int id) +{ + struct res_xrcdn *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->com.state = RES_XRCD_ALLOCATED; + + return &ret->com; +} + +static struct res_common *alloc_fs_rule_tr(u64 id, int qpn) +{ + struct res_fs_rule *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->com.state = RES_FS_RULE_ALLOCATED; + ret->qpn = qpn; + return &ret->com; +} + +static struct res_common *alloc_tr(u64 id, enum mlx4_resource type, int slave, + int extra) +{ + struct res_common *ret; + + switch (type) { + case RES_QP: + ret = alloc_qp_tr(id); + break; + case RES_MPT: + ret = alloc_mpt_tr(id, extra); + break; + case RES_MTT: + ret = alloc_mtt_tr(id, extra); + break; + case RES_EQ: + ret = alloc_eq_tr(id); + break; + case RES_CQ: + ret = alloc_cq_tr(id); + break; + case RES_SRQ: + ret = alloc_srq_tr(id); + break; + case RES_MAC: + pr_err("implementation missing\n"); + return NULL; + case RES_COUNTER: + ret = alloc_counter_tr(id, extra); + break; + case RES_XRCD: + ret = alloc_xrcdn_tr(id); + break; + case RES_FS_RULE: + ret = alloc_fs_rule_tr(id, extra); + break; + default: + return NULL; + } + if (ret) + ret->owner = slave; + + return ret; +} + +int mlx4_calc_vf_counters(struct mlx4_dev *dev, int slave, int port, + struct mlx4_counter *data) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_common *tmp; + struct res_counter *counter; + int *counters_arr; + int i = 0, err = 0; + + memset(data, 0, sizeof(*data)); + + counters_arr = kmalloc_array(dev->caps.max_counters, + sizeof(*counters_arr), GFP_KERNEL); + if (!counters_arr) + return -ENOMEM; + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry(tmp, + &tracker->slave_list[slave].res_list[RES_COUNTER], + list) { + counter = container_of(tmp, struct res_counter, com); + if (counter->port == port) { + counters_arr[i] = (int)tmp->res_id; + i++; + } + } + spin_unlock_irq(mlx4_tlock(dev)); + counters_arr[i] = -1; + + i = 0; + + while (counters_arr[i] != -1) { + err = mlx4_get_counter_stats(dev, counters_arr[i], data, + 0); + if (err) { + memset(data, 0, sizeof(*data)); + goto table_changed; + } + i++; + } + +table_changed: + kfree(counters_arr); + return 0; +} + +static int add_res_range(struct mlx4_dev *dev, int slave, u64 base, int count, + enum mlx4_resource type, int extra) +{ + int i; + int err; + struct mlx4_priv *priv = mlx4_priv(dev); + struct res_common **res_arr; + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct rb_root *root = &tracker->res_tree[type]; + + res_arr = kzalloc(count * sizeof *res_arr, GFP_KERNEL); + if (!res_arr) + return -ENOMEM; + + for (i = 0; i < count; ++i) { + res_arr[i] = alloc_tr(base + i, type, slave, extra); + if (!res_arr[i]) { + for (--i; i >= 0; --i) + kfree(res_arr[i]); + + kfree(res_arr); + return -ENOMEM; + } + } + + spin_lock_irq(mlx4_tlock(dev)); + for (i = 0; i < count; ++i) { + if (find_res(dev, base + i, type)) { + err = -EEXIST; + goto undo; + } + err = res_tracker_insert(root, res_arr[i]); + if (err) + goto undo; + list_add_tail(&res_arr[i]->list, + &tracker->slave_list[slave].res_list[type]); + } + spin_unlock_irq(mlx4_tlock(dev)); + kfree(res_arr); + + return 0; + +undo: + for (--i; i >= 0; --i) { + rb_erase(&res_arr[i]->node, root); + list_del_init(&res_arr[i]->list); + } + + spin_unlock_irq(mlx4_tlock(dev)); + + for (i = 0; i < count; ++i) + kfree(res_arr[i]); + + kfree(res_arr); + + return err; +} + +static int remove_qp_ok(struct res_qp *res) +{ + if (res->com.state == RES_QP_BUSY || atomic_read(&res->ref_count) || + !list_empty(&res->mcg_list)) { + pr_err("resource tracker: fail to remove qp, state %d, ref_count %d\n", + res->com.state, atomic_read(&res->ref_count)); + return -EBUSY; + } else if (res->com.state != RES_QP_RESERVED) { + return -EPERM; + } + + return 0; +} + +static int remove_mtt_ok(struct res_mtt *res, int order) +{ + if (res->com.state == RES_MTT_BUSY || + atomic_read(&res->ref_count)) { + pr_devel("%s-%d: state %s, ref_count %d\n", + __func__, __LINE__, + mtt_states_str(res->com.state), + atomic_read(&res->ref_count)); + return -EBUSY; + } else if (res->com.state != RES_MTT_ALLOCATED) + return -EPERM; + else if (res->order != order) + return -EINVAL; + + return 0; +} + +static int remove_mpt_ok(struct res_mpt *res) +{ + if (res->com.state == RES_MPT_BUSY) + return -EBUSY; + else if (res->com.state != RES_MPT_RESERVED) + return -EPERM; + + return 0; +} + +static int remove_eq_ok(struct res_eq *res) +{ + if (res->com.state == RES_MPT_BUSY) + return -EBUSY; + else if (res->com.state != RES_MPT_RESERVED) + return -EPERM; + + return 0; +} + +static int remove_counter_ok(struct res_counter *res) +{ + if (res->com.state == RES_COUNTER_BUSY) + return -EBUSY; + else if (res->com.state != RES_COUNTER_ALLOCATED) + return -EPERM; + + return 0; +} + +static int remove_xrcdn_ok(struct res_xrcdn *res) +{ + if (res->com.state == RES_XRCD_BUSY) + return -EBUSY; + else if (res->com.state != RES_XRCD_ALLOCATED) + return -EPERM; + + return 0; +} + +static int remove_fs_rule_ok(struct res_fs_rule *res) +{ + if (res->com.state == RES_FS_RULE_BUSY) + return -EBUSY; + else if (res->com.state != RES_FS_RULE_ALLOCATED) + return -EPERM; + + return 0; +} + +static int remove_cq_ok(struct res_cq *res) +{ + if (res->com.state == RES_CQ_BUSY) + return -EBUSY; + else if (res->com.state != RES_CQ_ALLOCATED) + return -EPERM; + + return 0; +} + +static int remove_srq_ok(struct res_srq *res) +{ + if (res->com.state == RES_SRQ_BUSY) + return -EBUSY; + else if (res->com.state != RES_SRQ_ALLOCATED) + return -EPERM; + + return 0; +} + +static int remove_ok(struct res_common *res, enum mlx4_resource type, int extra) +{ + switch (type) { + case RES_QP: + return remove_qp_ok((struct res_qp *)res); + case RES_CQ: + return remove_cq_ok((struct res_cq *)res); + case RES_SRQ: + return remove_srq_ok((struct res_srq *)res); + case RES_MPT: + return remove_mpt_ok((struct res_mpt *)res); + case RES_MTT: + return remove_mtt_ok((struct res_mtt *)res, extra); + case RES_MAC: + return -ENOSYS; + case RES_EQ: + return remove_eq_ok((struct res_eq *)res); + case RES_COUNTER: + return remove_counter_ok((struct res_counter *)res); + case RES_XRCD: + return remove_xrcdn_ok((struct res_xrcdn *)res); + case RES_FS_RULE: + return remove_fs_rule_ok((struct res_fs_rule *)res); + default: + return -EINVAL; + } +} + +static int rem_res_range(struct mlx4_dev *dev, int slave, u64 base, int count, + enum mlx4_resource type, int extra) +{ + u64 i; + int err; + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_common *r; + + spin_lock_irq(mlx4_tlock(dev)); + for (i = base; i < base + count; ++i) { + r = res_tracker_lookup(&tracker->res_tree[type], i); + if (!r) { + err = -ENOENT; + goto out; + } + if (r->owner != slave) { + err = -EPERM; + goto out; + } + err = remove_ok(r, type, extra); + if (err) + goto out; + } + + for (i = base; i < base + count; ++i) { + r = res_tracker_lookup(&tracker->res_tree[type], i); + rb_erase(&r->node, &tracker->res_tree[type]); + list_del(&r->list); + kfree(r); + } + err = 0; + +out: + spin_unlock_irq(mlx4_tlock(dev)); + + return err; +} + +static int qp_res_start_move_to(struct mlx4_dev *dev, int slave, int qpn, + enum res_qp_states state, struct res_qp **qp, + int alloc) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_qp *r; + int err = 0; + + spin_lock_irq(mlx4_tlock(dev)); + r = res_tracker_lookup(&tracker->res_tree[RES_QP], qpn); + if (!r) + err = -ENOENT; + else if (r->com.owner != slave) + err = -EPERM; + else { + switch (state) { + case RES_QP_BUSY: + mlx4_dbg(dev, "%s: failed RES_QP, 0x%llx\n", + __func__, (unsigned long long)r->com.res_id); + err = -EBUSY; + break; + + case RES_QP_RESERVED: + if (r->com.state == RES_QP_MAPPED && !alloc) + break; + + mlx4_dbg(dev, "failed RES_QP, 0x%llx\n", (unsigned long long)r->com.res_id); + err = -EINVAL; + break; + + case RES_QP_MAPPED: + if ((r->com.state == RES_QP_RESERVED && alloc) || + r->com.state == RES_QP_HW) + break; + else { + mlx4_dbg(dev, "failed RES_QP, 0x%llx\n", + (unsigned long long)r->com.res_id); + err = -EINVAL; + } + + break; + + case RES_QP_HW: + if (r->com.state != RES_QP_MAPPED) + err = -EINVAL; + break; + default: + err = -EINVAL; + } + + if (!err) { + r->com.from_state = r->com.state; + r->com.to_state = state; + r->com.state = RES_QP_BUSY; + if (qp) + *qp = r; + } + } + + spin_unlock_irq(mlx4_tlock(dev)); + + return err; +} + +static int mr_res_start_move_to(struct mlx4_dev *dev, int slave, int index, + enum res_mpt_states state, struct res_mpt **mpt) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_mpt *r; + int err = 0; + + spin_lock_irq(mlx4_tlock(dev)); + r = res_tracker_lookup(&tracker->res_tree[RES_MPT], index); + if (!r) + err = -ENOENT; + else if (r->com.owner != slave) + err = -EPERM; + else { + switch (state) { + case RES_MPT_BUSY: + err = -EINVAL; + break; + + case RES_MPT_RESERVED: + if (r->com.state != RES_MPT_MAPPED) + err = -EINVAL; + break; + + case RES_MPT_MAPPED: + if (r->com.state != RES_MPT_RESERVED && + r->com.state != RES_MPT_HW) + err = -EINVAL; + break; + + case RES_MPT_HW: + if (r->com.state != RES_MPT_MAPPED) + err = -EINVAL; + break; + default: + err = -EINVAL; + } + + if (!err) { + r->com.from_state = r->com.state; + r->com.to_state = state; + r->com.state = RES_MPT_BUSY; + if (mpt) + *mpt = r; + } + } + + spin_unlock_irq(mlx4_tlock(dev)); + + return err; +} + +static int eq_res_start_move_to(struct mlx4_dev *dev, int slave, int index, + enum res_eq_states state, struct res_eq **eq) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_eq *r; + int err = 0; + + spin_lock_irq(mlx4_tlock(dev)); + r = res_tracker_lookup(&tracker->res_tree[RES_EQ], index); + if (!r) + err = -ENOENT; + else if (r->com.owner != slave) + err = -EPERM; + else { + switch (state) { + case RES_EQ_BUSY: + err = -EINVAL; + break; + + case RES_EQ_RESERVED: + if (r->com.state != RES_EQ_HW) + err = -EINVAL; + break; + + case RES_EQ_HW: + if (r->com.state != RES_EQ_RESERVED) + err = -EINVAL; + break; + + default: + err = -EINVAL; + } + + if (!err) { + r->com.from_state = r->com.state; + r->com.to_state = state; + r->com.state = RES_EQ_BUSY; + } + } + + spin_unlock_irq(mlx4_tlock(dev)); + + if (!err && eq) + *eq = r; + + return err; +} + +static int cq_res_start_move_to(struct mlx4_dev *dev, int slave, int cqn, + enum res_cq_states state, struct res_cq **cq) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_cq *r; + int err; + + spin_lock_irq(mlx4_tlock(dev)); + r = res_tracker_lookup(&tracker->res_tree[RES_CQ], cqn); + if (!r) { + err = -ENOENT; + } else if (r->com.owner != slave) { + err = -EPERM; + } else if (state == RES_CQ_ALLOCATED) { + if (r->com.state != RES_CQ_HW) + err = -EINVAL; + else if (atomic_read(&r->ref_count)) + err = -EBUSY; + else + err = 0; + } else if (state != RES_CQ_HW || r->com.state != RES_CQ_ALLOCATED) { + err = -EINVAL; + } else { + err = 0; + } + + if (!err) { + r->com.from_state = r->com.state; + r->com.to_state = state; + r->com.state = RES_CQ_BUSY; + if (cq) + *cq = r; + } + + spin_unlock_irq(mlx4_tlock(dev)); + + return err; +} + +static int srq_res_start_move_to(struct mlx4_dev *dev, int slave, int index, + enum res_srq_states state, struct res_srq **srq) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_srq *r; + int err = 0; + + spin_lock_irq(mlx4_tlock(dev)); + r = res_tracker_lookup(&tracker->res_tree[RES_SRQ], index); + if (!r) { + err = -ENOENT; + } else if (r->com.owner != slave) { + err = -EPERM; + } else if (state == RES_SRQ_ALLOCATED) { + if (r->com.state != RES_SRQ_HW) + err = -EINVAL; + else if (atomic_read(&r->ref_count)) + err = -EBUSY; + } else if (state != RES_SRQ_HW || r->com.state != RES_SRQ_ALLOCATED) { + err = -EINVAL; + } + + if (!err) { + r->com.from_state = r->com.state; + r->com.to_state = state; + r->com.state = RES_SRQ_BUSY; + if (srq) + *srq = r; + } + + spin_unlock_irq(mlx4_tlock(dev)); + + return err; +} + +static void res_abort_move(struct mlx4_dev *dev, int slave, + enum mlx4_resource type, int id) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_common *r; + + spin_lock_irq(mlx4_tlock(dev)); + r = res_tracker_lookup(&tracker->res_tree[type], id); + if (r && (r->owner == slave)) + r->state = r->from_state; + spin_unlock_irq(mlx4_tlock(dev)); +} + +static void res_end_move(struct mlx4_dev *dev, int slave, + enum mlx4_resource type, int id) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_common *r; + + spin_lock_irq(mlx4_tlock(dev)); + r = res_tracker_lookup(&tracker->res_tree[type], id); + if (r && (r->owner == slave)) + r->state = r->to_state; + spin_unlock_irq(mlx4_tlock(dev)); +} + +static int valid_reserved(struct mlx4_dev *dev, int slave, int qpn) +{ + return mlx4_is_qp_reserved(dev, qpn) && + (mlx4_is_master(dev) || mlx4_is_guest_proxy(dev, slave, qpn)); +} + +static int fw_reserved(struct mlx4_dev *dev, int qpn) +{ + return qpn < dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW]; +} + +static int qp_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int err; + int count; + int align; + int base; + int qpn; + u8 flags; + + switch (op) { + case RES_OP_RESERVE: + count = get_param_l(&in_param) & 0xffffff; + /* Turn off all unsupported QP allocation flags that the + * slave tries to set. + */ + flags = (get_param_l(&in_param) >> 24) & dev->caps.alloc_res_qp_mask; + align = get_param_h(&in_param); + err = mlx4_grant_resource(dev, slave, RES_QP, count, 0); + if (err) + return err; + + err = __mlx4_qp_reserve_range(dev, count, align, &base, flags); + if (err) { + mlx4_release_resource(dev, slave, RES_QP, count, 0); + return err; + } + + err = add_res_range(dev, slave, base, count, RES_QP, 0); + if (err) { + mlx4_release_resource(dev, slave, RES_QP, count, 0); + __mlx4_qp_release_range(dev, base, count); + return err; + } + set_param_l(out_param, base); + break; + case RES_OP_MAP_ICM: + qpn = get_param_l(&in_param) & 0x7fffff; + if (valid_reserved(dev, slave, qpn)) { + err = add_res_range(dev, slave, qpn, 1, RES_QP, 0); + if (err) + return err; + } + + err = qp_res_start_move_to(dev, slave, qpn, RES_QP_MAPPED, + NULL, 1); + if (err) + return err; + + if (!fw_reserved(dev, qpn)) { + err = __mlx4_qp_alloc_icm(dev, qpn, GFP_KERNEL); + if (err) { + res_abort_move(dev, slave, RES_QP, qpn); + return err; + } + } + + res_end_move(dev, slave, RES_QP, qpn); + break; + + default: + err = -EINVAL; + break; + } + return err; +} + +static int mtt_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int err = -EINVAL; + int base; + int order; + + if (op != RES_OP_RESERVE_AND_MAP) + return err; + + order = get_param_l(&in_param); + + err = mlx4_grant_resource(dev, slave, RES_MTT, 1 << order, 0); + if (err) + return err; + + base = __mlx4_alloc_mtt_range(dev, order); + if (base == -1) { + mlx4_release_resource(dev, slave, RES_MTT, 1 << order, 0); + return -ENOMEM; + } + + err = add_res_range(dev, slave, base, 1, RES_MTT, order); + if (err) { + mlx4_release_resource(dev, slave, RES_MTT, 1 << order, 0); + __mlx4_free_mtt_range(dev, base, order); + } else { + set_param_l(out_param, base); + } + + return err; +} + +static int mpt_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int err = -EINVAL; + int index; + int id; + struct res_mpt *mpt; + + switch (op) { + case RES_OP_RESERVE: + err = mlx4_grant_resource(dev, slave, RES_MPT, 1, 0); + if (err) + break; + + index = __mlx4_mpt_reserve(dev); + if (index == -1) { + mlx4_release_resource(dev, slave, RES_MPT, 1, 0); + break; + } + id = index & mpt_mask(dev); + + err = add_res_range(dev, slave, id, 1, RES_MPT, index); + if (err) { + mlx4_release_resource(dev, slave, RES_MPT, 1, 0); + __mlx4_mpt_release(dev, index); + break; + } + set_param_l(out_param, index); + break; + case RES_OP_MAP_ICM: + index = get_param_l(&in_param); + id = index & mpt_mask(dev); + err = mr_res_start_move_to(dev, slave, id, + RES_MPT_MAPPED, &mpt); + if (err) + return err; + + err = __mlx4_mpt_alloc_icm(dev, mpt->key, GFP_KERNEL); + if (err) { + res_abort_move(dev, slave, RES_MPT, id); + return err; + } + + res_end_move(dev, slave, RES_MPT, id); + break; + } + return err; +} + +static int cq_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int cqn; + int err; + + switch (op) { + case RES_OP_RESERVE_AND_MAP: + err = mlx4_grant_resource(dev, slave, RES_CQ, 1, 0); + if (err) + break; + + err = __mlx4_cq_alloc_icm(dev, &cqn); + if (err) { + mlx4_release_resource(dev, slave, RES_CQ, 1, 0); + break; + } + + err = add_res_range(dev, slave, cqn, 1, RES_CQ, 0); + if (err) { + mlx4_release_resource(dev, slave, RES_CQ, 1, 0); + __mlx4_cq_free_icm(dev, cqn); + break; + } + + set_param_l(out_param, cqn); + break; + + default: + err = -EINVAL; + } + + return err; +} + +static int srq_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int srqn; + int err; + + switch (op) { + case RES_OP_RESERVE_AND_MAP: + err = mlx4_grant_resource(dev, slave, RES_SRQ, 1, 0); + if (err) + break; + + err = __mlx4_srq_alloc_icm(dev, &srqn); + if (err) { + mlx4_release_resource(dev, slave, RES_SRQ, 1, 0); + break; + } + + err = add_res_range(dev, slave, srqn, 1, RES_SRQ, 0); + if (err) { + mlx4_release_resource(dev, slave, RES_SRQ, 1, 0); + __mlx4_srq_free_icm(dev, srqn); + break; + } + + set_param_l(out_param, srqn); + break; + + default: + err = -EINVAL; + } + + return err; +} + +static int mac_find_smac_ix_in_slave(struct mlx4_dev *dev, int slave, int port, + u8 smac_index, u64 *mac) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *mac_list = + &tracker->slave_list[slave].res_list[RES_MAC]; + struct mac_res *res, *tmp; + + list_for_each_entry_safe(res, tmp, mac_list, list) { + if (res->smac_index == smac_index && res->port == (u8) port) { + *mac = res->mac; + return 0; + } + } + return -ENOENT; +} + +static int mac_add_to_slave(struct mlx4_dev *dev, int slave, u64 mac, int port, u8 smac_index) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *mac_list = + &tracker->slave_list[slave].res_list[RES_MAC]; + struct mac_res *res, *tmp; + + list_for_each_entry_safe(res, tmp, mac_list, list) { + if (res->mac == mac && res->port == (u8) port) { + /* mac found. update ref count */ + ++res->ref_count; + return 0; + } + } + + if (mlx4_grant_resource(dev, slave, RES_MAC, 1, port)) + return -EINVAL; + res = kzalloc(sizeof *res, GFP_KERNEL); + if (!res) { + mlx4_release_resource(dev, slave, RES_MAC, 1, port); + return -ENOMEM; + } + res->mac = mac; + res->port = (u8) port; + res->smac_index = smac_index; + res->ref_count = 1; + list_add_tail(&res->list, + &tracker->slave_list[slave].res_list[RES_MAC]); + return 0; +} + +static void mac_del_from_slave(struct mlx4_dev *dev, int slave, u64 mac, + int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *mac_list = + &tracker->slave_list[slave].res_list[RES_MAC]; + struct mac_res *res, *tmp; + + list_for_each_entry_safe(res, tmp, mac_list, list) { + if (res->mac == mac && res->port == (u8) port) { + if (!--res->ref_count) { + list_del(&res->list); + mlx4_release_resource(dev, slave, RES_MAC, 1, port); + kfree(res); + } + break; + } + } +} + +static void rem_slave_macs(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *mac_list = + &tracker->slave_list[slave].res_list[RES_MAC]; + struct mac_res *res, *tmp; + int i; + + list_for_each_entry_safe(res, tmp, mac_list, list) { + list_del(&res->list); + /* dereference the mac the num times the slave referenced it */ + for (i = 0; i < res->ref_count; i++) + __mlx4_unregister_mac(dev, res->port, res->mac); + mlx4_release_resource(dev, slave, RES_MAC, 1, res->port); + kfree(res); + } +} + +static int mac_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param, int in_port) +{ + int err = -EINVAL; + int port; + u64 mac; + u8 smac_index = 0; + + if (op != RES_OP_RESERVE_AND_MAP) + return err; + + port = !in_port ? get_param_l(out_param) : in_port; + port = mlx4_slave_convert_port( + dev, slave, port); + + if (port < 0) + return -EINVAL; + mac = in_param; + + err = __mlx4_register_mac(dev, port, mac); + if (err >= 0) { + smac_index = err; + set_param_l(out_param, err); + err = 0; + } + + if (!err) { + err = mac_add_to_slave(dev, slave, mac, port, smac_index); + if (err) + __mlx4_unregister_mac(dev, port, mac); + } + return err; +} + +static int vlan_add_to_slave(struct mlx4_dev *dev, int slave, u16 vlan, + int port, int vlan_index) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *vlan_list = + &tracker->slave_list[slave].res_list[RES_VLAN]; + struct vlan_res *res, *tmp; + + list_for_each_entry_safe(res, tmp, vlan_list, list) { + if (res->vlan == vlan && res->port == (u8) port) { + /* vlan found. update ref count */ + ++res->ref_count; + return 0; + } + } + + if (mlx4_grant_resource(dev, slave, RES_VLAN, 1, port)) + return -EINVAL; + res = kzalloc(sizeof(*res), GFP_KERNEL); + if (!res) { + mlx4_release_resource(dev, slave, RES_VLAN, 1, port); + return -ENOMEM; + } + res->vlan = vlan; + res->port = (u8) port; + res->vlan_index = vlan_index; + res->ref_count = 1; + list_add_tail(&res->list, + &tracker->slave_list[slave].res_list[RES_VLAN]); + return 0; +} + + +static void vlan_del_from_slave(struct mlx4_dev *dev, int slave, u16 vlan, + int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *vlan_list = + &tracker->slave_list[slave].res_list[RES_VLAN]; + struct vlan_res *res, *tmp; + + list_for_each_entry_safe(res, tmp, vlan_list, list) { + if (res->vlan == vlan && res->port == (u8) port) { + if (!--res->ref_count) { + list_del(&res->list); + mlx4_release_resource(dev, slave, RES_VLAN, + 1, port); + kfree(res); + } + break; + } + } +} + +static void rem_slave_vlans(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *vlan_list = + &tracker->slave_list[slave].res_list[RES_VLAN]; + struct vlan_res *res, *tmp; + int i; + + list_for_each_entry_safe(res, tmp, vlan_list, list) { + list_del(&res->list); + /* dereference the vlan the num times the slave referenced it */ + for (i = 0; i < res->ref_count; i++) + __mlx4_unregister_vlan(dev, res->port, res->vlan); + mlx4_release_resource(dev, slave, RES_VLAN, 1, res->port); + kfree(res); + } +} + +static int vlan_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param, int in_port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state; + int err; + u16 vlan; + int vlan_index; + int port; + + port = !in_port ? get_param_l(out_param) : in_port; + + if (!port || op != RES_OP_RESERVE_AND_MAP) + return -EINVAL; + + port = mlx4_slave_convert_port( + dev, slave, port); + + if (port < 0) + return -EINVAL; + /* upstream kernels had NOP for reg/unreg vlan. Continue this. */ + if (!in_port && port > 0 && port <= dev->caps.num_ports) { + slave_state[slave].old_vlan_api = true; + return 0; + } + + vlan = (u16) in_param; + + err = __mlx4_register_vlan(dev, port, vlan, &vlan_index); + if (!err) { + set_param_l(out_param, (u32) vlan_index); + err = vlan_add_to_slave(dev, slave, vlan, port, vlan_index); + if (err) + __mlx4_unregister_vlan(dev, port, vlan); + } + return err; +} + +static int counter_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param, int port) +{ + u32 index; + int err; + + if (op != RES_OP_RESERVE) + return -EINVAL; + + err = mlx4_grant_resource(dev, slave, RES_COUNTER, 1, 0); + if (err) + return err; + + err = __mlx4_counter_alloc(dev, &index); + if (err) { + mlx4_release_resource(dev, slave, RES_COUNTER, 1, 0); + return err; + } + + err = add_res_range(dev, slave, index, 1, RES_COUNTER, port); + if (err) { + __mlx4_counter_free(dev, index); + mlx4_release_resource(dev, slave, RES_COUNTER, 1, 0); + } else { + set_param_l(out_param, index); + } + + return err; +} + +static int xrcdn_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + u32 xrcdn; + int err; + + if (op != RES_OP_RESERVE) + return -EINVAL; + + err = __mlx4_xrcd_alloc(dev, &xrcdn); + if (err) + return err; + + err = add_res_range(dev, slave, xrcdn, 1, RES_XRCD, 0); + if (err) + __mlx4_xrcd_free(dev, xrcdn); + else + set_param_l(out_param, xrcdn); + + return err; +} + +int mlx4_ALLOC_RES_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int alop = vhcr->op_modifier; + + switch (vhcr->in_modifier & 0xFF) { + case RES_QP: + err = qp_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_MTT: + err = mtt_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_MPT: + err = mpt_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_CQ: + err = cq_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_SRQ: + err = srq_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_MAC: + err = mac_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param, + (vhcr->in_modifier >> 8) & 0xFF); + break; + + case RES_VLAN: + err = vlan_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param, + (vhcr->in_modifier >> 8) & 0xFF); + break; + + case RES_COUNTER: + err = counter_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param, 0); + break; + + case RES_XRCD: + err = xrcdn_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + default: + err = -EINVAL; + break; + } + + return err; +} + +static int qp_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param) +{ + int err; + int count; + int base; + int qpn; + + switch (op) { + case RES_OP_RESERVE: + base = get_param_l(&in_param) & 0x7fffff; + count = get_param_h(&in_param); + err = rem_res_range(dev, slave, base, count, RES_QP, 0); + if (err) + break; + mlx4_release_resource(dev, slave, RES_QP, count, 0); + __mlx4_qp_release_range(dev, base, count); + break; + case RES_OP_MAP_ICM: + qpn = get_param_l(&in_param) & 0x7fffff; + err = qp_res_start_move_to(dev, slave, qpn, RES_QP_RESERVED, + NULL, 0); + if (err) + return err; + + if (!fw_reserved(dev, qpn)) + __mlx4_qp_free_icm(dev, qpn); + + res_end_move(dev, slave, RES_QP, qpn); + + if (valid_reserved(dev, slave, qpn)) + err = rem_res_range(dev, slave, qpn, 1, RES_QP, 0); + break; + default: + err = -EINVAL; + break; + } + return err; +} + +static int mtt_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int err = -EINVAL; + int base; + int order; + + if (op != RES_OP_RESERVE_AND_MAP) + return err; + + base = get_param_l(&in_param); + order = get_param_h(&in_param); + err = rem_res_range(dev, slave, base, 1, RES_MTT, order); + if (!err) { + mlx4_release_resource(dev, slave, RES_MTT, 1 << order, 0); + __mlx4_free_mtt_range(dev, base, order); + } + return err; +} + +static int mpt_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param) +{ + int err = -EINVAL; + int index; + int id; + struct res_mpt *mpt; + + switch (op) { + case RES_OP_RESERVE: + index = get_param_l(&in_param); + id = index & mpt_mask(dev); + err = get_res(dev, slave, id, RES_MPT, &mpt); + if (err) + break; + index = mpt->key; + put_res(dev, slave, id, RES_MPT); + + err = rem_res_range(dev, slave, id, 1, RES_MPT, 0); + if (err) + break; + mlx4_release_resource(dev, slave, RES_MPT, 1, 0); + __mlx4_mpt_release(dev, index); + break; + case RES_OP_MAP_ICM: + index = get_param_l(&in_param); + id = index & mpt_mask(dev); + err = mr_res_start_move_to(dev, slave, id, + RES_MPT_RESERVED, &mpt); + if (err) + return err; + + __mlx4_mpt_free_icm(dev, mpt->key); + res_end_move(dev, slave, RES_MPT, id); + break; + default: + err = -EINVAL; + break; + } + return err; +} + +static int cq_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int cqn; + int err; + + switch (op) { + case RES_OP_RESERVE_AND_MAP: + cqn = get_param_l(&in_param); + err = rem_res_range(dev, slave, cqn, 1, RES_CQ, 0); + if (err) + break; + + mlx4_release_resource(dev, slave, RES_CQ, 1, 0); + __mlx4_cq_free_icm(dev, cqn); + break; + + default: + err = -EINVAL; + break; + } + + return err; +} + +static int srq_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int srqn; + int err; + + switch (op) { + case RES_OP_RESERVE_AND_MAP: + srqn = get_param_l(&in_param); + err = rem_res_range(dev, slave, srqn, 1, RES_SRQ, 0); + if (err) + break; + + mlx4_release_resource(dev, slave, RES_SRQ, 1, 0); + __mlx4_srq_free_icm(dev, srqn); + break; + + default: + err = -EINVAL; + break; + } + + return err; +} + +static int mac_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param, int in_port) +{ + int port; + int err = 0; + + switch (op) { + case RES_OP_RESERVE_AND_MAP: + port = !in_port ? get_param_l(out_param) : in_port; + port = mlx4_slave_convert_port( + dev, slave, port); + + if (port < 0) + return -EINVAL; + mac_del_from_slave(dev, slave, in_param, port); + __mlx4_unregister_mac(dev, port, in_param); + break; + default: + err = -EINVAL; + break; + } + + return err; + +} + +static int vlan_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param, int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state; + int err = 0; + + port = mlx4_slave_convert_port( + dev, slave, port); + + if (port < 0) + return -EINVAL; + switch (op) { + case RES_OP_RESERVE_AND_MAP: + if (slave_state[slave].old_vlan_api) + return 0; + if (!port) + return -EINVAL; + vlan_del_from_slave(dev, slave, in_param, port); + __mlx4_unregister_vlan(dev, port, in_param); + break; + default: + err = -EINVAL; + break; + } + + return err; +} + +static int counter_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int index; + int err; + + if (op != RES_OP_RESERVE) + return -EINVAL; + + index = get_param_l(&in_param); + if (index == MLX4_SINK_COUNTER_INDEX(dev)) + return 0; + + err = rem_res_range(dev, slave, index, 1, RES_COUNTER, 0); + if (err) + return err; + + __mlx4_counter_free(dev, index); + mlx4_release_resource(dev, slave, RES_COUNTER, 1, 0); + + return err; +} + +static int xrcdn_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int xrcdn; + int err; + + if (op != RES_OP_RESERVE) + return -EINVAL; + + xrcdn = get_param_l(&in_param); + err = rem_res_range(dev, slave, xrcdn, 1, RES_XRCD, 0); + if (err) + return err; + + __mlx4_xrcd_free(dev, xrcdn); + + return err; +} + +int mlx4_FREE_RES_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err = -EINVAL; + int alop = vhcr->op_modifier; + + switch (vhcr->in_modifier & 0xFF) { + case RES_QP: + err = qp_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param); + break; + + case RES_MTT: + err = mtt_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_MPT: + err = mpt_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param); + break; + + case RES_CQ: + err = cq_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_SRQ: + err = srq_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_MAC: + err = mac_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param, + (vhcr->in_modifier >> 8) & 0xFF); + break; + + case RES_VLAN: + err = vlan_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param, + (vhcr->in_modifier >> 8) & 0xFF); + break; + + case RES_COUNTER: + err = counter_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_XRCD: + err = xrcdn_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + + default: + break; + } + return err; +} + +/* ugly but other choices are uglier */ +static int mr_phys_mpt(struct mlx4_mpt_entry *mpt) +{ + return (be32_to_cpu(mpt->flags) >> 9) & 1; +} + +static int mr_get_mtt_addr(struct mlx4_mpt_entry *mpt) +{ + return (int)be64_to_cpu(mpt->mtt_addr) & 0xfffffff8; +} + +static int mr_get_mtt_size(struct mlx4_mpt_entry *mpt) +{ + return be32_to_cpu(mpt->mtt_sz); +} + +static u32 mr_get_pd(struct mlx4_mpt_entry *mpt) +{ + return be32_to_cpu(mpt->pd_flags) & 0x00ffffff; +} + +static int mr_is_fmr(struct mlx4_mpt_entry *mpt) +{ + return be32_to_cpu(mpt->pd_flags) & MLX4_MPT_PD_FLAG_FAST_REG; +} + +static int mr_is_bind_enabled(struct mlx4_mpt_entry *mpt) +{ + return be32_to_cpu(mpt->flags) & MLX4_MPT_FLAG_BIND_ENABLE; +} + +static int mr_is_region(struct mlx4_mpt_entry *mpt) +{ + return be32_to_cpu(mpt->flags) & MLX4_MPT_FLAG_REGION; +} + +static int qp_get_mtt_addr(struct mlx4_qp_context *qpc) +{ + return be32_to_cpu(qpc->mtt_base_addr_l) & 0xfffffff8; +} + +static int srq_get_mtt_addr(struct mlx4_srq_context *srqc) +{ + return be32_to_cpu(srqc->mtt_base_addr_l) & 0xfffffff8; +} + +static int qp_get_mtt_size(struct mlx4_qp_context *qpc) +{ + int page_shift = (qpc->log_page_size & 0x3f) + 12; + int log_sq_size = (qpc->sq_size_stride >> 3) & 0xf; + int log_sq_sride = qpc->sq_size_stride & 7; + int log_rq_size = (qpc->rq_size_stride >> 3) & 0xf; + int log_rq_stride = qpc->rq_size_stride & 7; + int srq = (be32_to_cpu(qpc->srqn) >> 24) & 1; + int rss = (be32_to_cpu(qpc->flags) >> 13) & 1; + u32 ts = (be32_to_cpu(qpc->flags) >> 16) & 0xff; + int xrc = (ts == MLX4_QP_ST_XRC) ? 1 : 0; + int sq_size; + int rq_size; + int total_pages; + int total_mem; + int page_offset = (be32_to_cpu(qpc->params2) >> 6) & 0x3f; + + sq_size = 1 << (log_sq_size + log_sq_sride + 4); + rq_size = (srq|rss|xrc) ? 0 : (1 << (log_rq_size + log_rq_stride + 4)); + total_mem = sq_size + rq_size; + total_pages = + roundup_pow_of_two((total_mem + (page_offset << 6)) >> + page_shift); + + return total_pages; +} + +static int check_mtt_range(struct mlx4_dev *dev, int slave, int start, + int size, struct res_mtt *mtt) +{ + int res_start = mtt->com.res_id; + int res_size = (1 << mtt->order); + + if (start < res_start || start + size > res_start + res_size) + return -EPERM; + return 0; +} + +int mlx4_SW2HW_MPT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int index = vhcr->in_modifier; + struct res_mtt *mtt; + struct res_mpt *mpt; + int mtt_base = mr_get_mtt_addr(inbox->buf) / dev->caps.mtt_entry_sz; + int phys; + int id; + u32 pd; + int pd_slave; + + id = index & mpt_mask(dev); + err = mr_res_start_move_to(dev, slave, id, RES_MPT_HW, &mpt); + if (err) + return err; + + /* Disable memory windows for VFs. */ + if (!mr_is_region(inbox->buf)) { + err = -EPERM; + goto ex_abort; + } + + /* Make sure that the PD bits related to the slave id are zeros. */ + pd = mr_get_pd(inbox->buf); + pd_slave = (pd >> 17) & 0x7f; + if (pd_slave != 0 && --pd_slave != slave) { + err = -EPERM; + goto ex_abort; + } + + if (mr_is_fmr(inbox->buf)) { + /* FMR and Bind Enable are forbidden in slave devices. */ + if (mr_is_bind_enabled(inbox->buf)) { + err = -EPERM; + goto ex_abort; + } + /* FMR and Memory Windows are also forbidden. */ + if (!mr_is_region(inbox->buf)) { + err = -EPERM; + goto ex_abort; + } + } + + phys = mr_phys_mpt(inbox->buf); + if (!phys) { + err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); + if (err) + goto ex_abort; + + err = check_mtt_range(dev, slave, mtt_base, + mr_get_mtt_size(inbox->buf), mtt); + if (err) + goto ex_put; + + mpt->mtt = mtt; + } + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto ex_put; + + if (!phys) { + atomic_inc(&mtt->ref_count); + put_res(dev, slave, mtt->com.res_id, RES_MTT); + } + + res_end_move(dev, slave, RES_MPT, id); + return 0; + +ex_put: + if (!phys) + put_res(dev, slave, mtt->com.res_id, RES_MTT); +ex_abort: + res_abort_move(dev, slave, RES_MPT, id); + + return err; +} + +int mlx4_HW2SW_MPT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int index = vhcr->in_modifier; + struct res_mpt *mpt; + int id; + + id = index & mpt_mask(dev); + err = mr_res_start_move_to(dev, slave, id, RES_MPT_MAPPED, &mpt); + if (err) + return err; + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto ex_abort; + + if (mpt->mtt) + atomic_dec(&mpt->mtt->ref_count); + + res_end_move(dev, slave, RES_MPT, id); + return 0; + +ex_abort: + res_abort_move(dev, slave, RES_MPT, id); + + return err; +} + +int mlx4_QUERY_MPT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int index = vhcr->in_modifier; + struct res_mpt *mpt; + int id; + + id = index & mpt_mask(dev); + err = get_res(dev, slave, id, RES_MPT, &mpt); + if (err) + return err; + + if (mpt->com.from_state == RES_MPT_MAPPED) { + /* In order to allow rereg in SRIOV, we need to alter the MPT entry. To do + * that, the VF must read the MPT. But since the MPT entry memory is not + * in the VF's virtual memory space, it must use QUERY_MPT to obtain the + * entry contents. To guarantee that the MPT cannot be changed, the driver + * must perform HW2SW_MPT before this query and return the MPT entry to HW + * ownership fofollowing the change. The change here allows the VF to + * perform QUERY_MPT also when the entry is in SW ownership. + */ + struct mlx4_mpt_entry *mpt_entry = mlx4_table_find( + &mlx4_priv(dev)->mr_table.dmpt_table, + mpt->key, NULL); + + if (NULL == mpt_entry || NULL == outbox->buf) { + err = -EINVAL; + goto out; + } + + memcpy(outbox->buf, mpt_entry, sizeof(*mpt_entry)); + + err = 0; + } else if (mpt->com.from_state == RES_MPT_HW) { + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + } else { + err = -EBUSY; + goto out; + } + + +out: + put_res(dev, slave, id, RES_MPT); + return err; +} + +static int qp_get_rcqn(struct mlx4_qp_context *qpc) +{ + return be32_to_cpu(qpc->cqn_recv) & 0xffffff; +} + +static int qp_get_scqn(struct mlx4_qp_context *qpc) +{ + return be32_to_cpu(qpc->cqn_send) & 0xffffff; +} + +static u32 qp_get_srqn(struct mlx4_qp_context *qpc) +{ + return be32_to_cpu(qpc->srqn) & 0x1ffffff; +} + +static void adjust_proxy_tun_qkey(struct mlx4_dev *dev, struct mlx4_vhcr *vhcr, + struct mlx4_qp_context *context) +{ + u32 qpn = vhcr->in_modifier & 0xffffff; + u32 qkey = 0; + + if (mlx4_get_parav_qkey(dev, qpn, &qkey)) + return; + + /* adjust qkey in qp context */ + context->qkey = cpu_to_be32(qkey); +} + +static int adjust_qp_sched_queue(struct mlx4_dev *dev, int slave, + struct mlx4_qp_context *qpc, + struct mlx4_cmd_mailbox *inbox); + +int mlx4_RST2INIT_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int qpn = vhcr->in_modifier & 0x7fffff; + struct res_mtt *mtt; + struct res_qp *qp; + struct mlx4_qp_context *qpc = inbox->buf + 8; + int mtt_base = qp_get_mtt_addr(qpc) / dev->caps.mtt_entry_sz; + int mtt_size = qp_get_mtt_size(qpc); + struct res_cq *rcq; + struct res_cq *scq; + int rcqn = qp_get_rcqn(qpc); + int scqn = qp_get_scqn(qpc); + u32 srqn = qp_get_srqn(qpc) & 0xffffff; + int use_srq = (qp_get_srqn(qpc) >> 24) & 1; + struct res_srq *srq; + int local_qpn = be32_to_cpu(qpc->local_qpn) & 0xffffff; + + err = adjust_qp_sched_queue(dev, slave, qpc, inbox); + if (err) + return err; + + err = qp_res_start_move_to(dev, slave, qpn, RES_QP_HW, &qp, 0); + if (err) + return err; + qp->local_qpn = local_qpn; + qp->sched_queue = 0; + qp->param3 = 0; + qp->vlan_control = 0; + qp->fvl_rx = 0; + qp->pri_path_fl = 0; + qp->vlan_index = 0; + qp->feup = 0; + qp->qpc_flags = be32_to_cpu(qpc->flags); + + err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); + if (err) + goto ex_abort; + + err = check_mtt_range(dev, slave, mtt_base, mtt_size, mtt); + if (err) + goto ex_put_mtt; + + err = get_res(dev, slave, rcqn, RES_CQ, &rcq); + if (err) + goto ex_put_mtt; + + if (scqn != rcqn) { + err = get_res(dev, slave, scqn, RES_CQ, &scq); + if (err) + goto ex_put_rcq; + } else + scq = rcq; + + if (use_srq) { + err = get_res(dev, slave, srqn, RES_SRQ, &srq); + if (err) + goto ex_put_scq; + } + + adjust_proxy_tun_qkey(dev, vhcr, qpc); + update_pkey_index(dev, slave, inbox); + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto ex_put_srq; + atomic_inc(&mtt->ref_count); + qp->mtt = mtt; + atomic_inc(&rcq->ref_count); + qp->rcq = rcq; + atomic_inc(&scq->ref_count); + qp->scq = scq; + + if (scqn != rcqn) + put_res(dev, slave, scqn, RES_CQ); + + if (use_srq) { + atomic_inc(&srq->ref_count); + put_res(dev, slave, srqn, RES_SRQ); + qp->srq = srq; + } + put_res(dev, slave, rcqn, RES_CQ); + put_res(dev, slave, mtt_base, RES_MTT); + res_end_move(dev, slave, RES_QP, qpn); + + return 0; + +ex_put_srq: + if (use_srq) + put_res(dev, slave, srqn, RES_SRQ); +ex_put_scq: + if (scqn != rcqn) + put_res(dev, slave, scqn, RES_CQ); +ex_put_rcq: + put_res(dev, slave, rcqn, RES_CQ); +ex_put_mtt: + put_res(dev, slave, mtt_base, RES_MTT); +ex_abort: + res_abort_move(dev, slave, RES_QP, qpn); + + return err; +} + +static int eq_get_mtt_addr(struct mlx4_eq_context *eqc) +{ + return be32_to_cpu(eqc->mtt_base_addr_l) & 0xfffffff8; +} + +static int eq_get_mtt_size(struct mlx4_eq_context *eqc) +{ + int log_eq_size = eqc->log_eq_size & 0x1f; + int page_shift = (eqc->log_page_size & 0x3f) + 12; + + if (log_eq_size + 5 < page_shift) + return 1; + + return 1 << (log_eq_size + 5 - page_shift); +} + +static int cq_get_mtt_addr(struct mlx4_cq_context *cqc) +{ + return be32_to_cpu(cqc->mtt_base_addr_l) & 0xfffffff8; +} + +static int cq_get_mtt_size(struct mlx4_cq_context *cqc) +{ + int log_cq_size = (be32_to_cpu(cqc->logsize_usrpage) >> 24) & 0x1f; + int page_shift = (cqc->log_page_size & 0x3f) + 12; + + if (log_cq_size + 5 < page_shift) + return 1; + + return 1 << (log_cq_size + 5 - page_shift); +} + +int mlx4_SW2HW_EQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int eqn = vhcr->in_modifier; + int res_id = (slave << 10) | eqn; + struct mlx4_eq_context *eqc = inbox->buf; + int mtt_base = eq_get_mtt_addr(eqc) / dev->caps.mtt_entry_sz; + int mtt_size = eq_get_mtt_size(eqc); + struct res_eq *eq; + struct res_mtt *mtt; + + err = add_res_range(dev, slave, res_id, 1, RES_EQ, 0); + if (err) + return err; + err = eq_res_start_move_to(dev, slave, res_id, RES_EQ_HW, &eq); + if (err) + goto out_add; + + err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); + if (err) + goto out_move; + + err = check_mtt_range(dev, slave, mtt_base, mtt_size, mtt); + if (err) + goto out_put; + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto out_put; + + atomic_inc(&mtt->ref_count); + eq->mtt = mtt; + put_res(dev, slave, mtt->com.res_id, RES_MTT); + res_end_move(dev, slave, RES_EQ, res_id); + return 0; + +out_put: + put_res(dev, slave, mtt->com.res_id, RES_MTT); +out_move: + res_abort_move(dev, slave, RES_EQ, res_id); +out_add: + rem_res_range(dev, slave, res_id, 1, RES_EQ, 0); + return err; +} + +int mlx4_CONFIG_DEV_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + u8 get = vhcr->op_modifier; + + if (get != 1) + return -EPERM; + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + + return err; +} + +static int get_containing_mtt(struct mlx4_dev *dev, int slave, int start, + int len, struct res_mtt **res) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_mtt *mtt; + int err = -EINVAL; + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry(mtt, &tracker->slave_list[slave].res_list[RES_MTT], + com.list) { + if (!check_mtt_range(dev, slave, start, len, mtt)) { + *res = mtt; + mtt->com.from_state = mtt->com.state; + mtt->com.state = RES_MTT_BUSY; + err = 0; + break; + } + } + spin_unlock_irq(mlx4_tlock(dev)); + + return err; +} + +static int verify_qp_parameters(struct mlx4_dev *dev, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + enum qp_transition transition, u8 slave) +{ + u32 qp_type; + u32 qpn; + struct mlx4_qp_context *qp_ctx; + enum mlx4_qp_optpar optpar; + int port; + int num_gids; + + qp_ctx = inbox->buf + 8; + qp_type = (be32_to_cpu(qp_ctx->flags) >> 16) & 0xff; + optpar = be32_to_cpu(*(__be32 *) inbox->buf); + + if (slave != mlx4_master_func_num(dev)) { + qp_ctx->params2 &= ~MLX4_QP_BIT_FPP; + /* setting QP rate-limit is disallowed for VFs */ + if (qp_ctx->rate_limit_params) + return -EPERM; + } + + switch (qp_type) { + case MLX4_QP_ST_RC: + case MLX4_QP_ST_XRC: + case MLX4_QP_ST_UC: + switch (transition) { + case QP_TRANS_INIT2RTR: + case QP_TRANS_RTR2RTS: + case QP_TRANS_RTS2RTS: + case QP_TRANS_SQD2SQD: + case QP_TRANS_SQD2RTS: + if (slave != mlx4_master_func_num(dev)) { + if (optpar & MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH) { + port = (qp_ctx->pri_path.sched_queue >> 6 & 1) + 1; + if (dev->caps.port_mask[port] != MLX4_PORT_TYPE_IB) + num_gids = mlx4_get_slave_num_gids(dev, slave, port); + else + num_gids = 1; + if (qp_ctx->pri_path.mgid_index >= num_gids) + return -EINVAL; + } + if (optpar & MLX4_QP_OPTPAR_ALT_ADDR_PATH) { + port = (qp_ctx->alt_path.sched_queue >> 6 & 1) + 1; + if (dev->caps.port_mask[port] != MLX4_PORT_TYPE_IB) + num_gids = mlx4_get_slave_num_gids(dev, slave, port); + else + num_gids = 1; + if (qp_ctx->alt_path.mgid_index >= num_gids) + return -EINVAL; + } + } + break; + default: + break; + } + break; + + case MLX4_QP_ST_MLX: + qpn = vhcr->in_modifier & 0x7fffff; + port = (qp_ctx->pri_path.sched_queue >> 6 & 1) + 1; + if (transition == QP_TRANS_INIT2RTR && + slave != mlx4_master_func_num(dev) && + mlx4_is_qp_reserved(dev, qpn) && + !mlx4_vf_smi_enabled(dev, slave, port)) { + /* only enabled VFs may create MLX proxy QPs */ + mlx4_err(dev, "%s: unprivileged slave %d attempting to create an MLX proxy special QP on port %d\n", + __func__, slave, port); + return -EPERM; + } + break; + + default: + break; + } + + return 0; +} + +int mlx4_WRITE_MTT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_mtt mtt; + __be64 *page_list = inbox->buf; + u64 *pg_list = (u64 *)page_list; + int i; + struct res_mtt *rmtt = NULL; + int start = be64_to_cpu(page_list[0]); + int npages = vhcr->in_modifier; + int err; + + err = get_containing_mtt(dev, slave, start, npages, &rmtt); + if (err) + return err; + + /* Call the SW implementation of write_mtt: + * - Prepare a dummy mtt struct + * - Translate inbox contents to simple addresses in host endianness */ + mtt.offset = 0; /* TBD this is broken but I don't handle it since + we don't really use it */ + mtt.order = 0; + mtt.page_shift = 0; + for (i = 0; i < npages; ++i) + pg_list[i + 2] = (be64_to_cpu(page_list[i + 2]) & ~1ULL); + + err = __mlx4_write_mtt(dev, &mtt, be64_to_cpu(page_list[0]), npages, + ((u64 *)page_list + 2)); + + if (rmtt) + put_res(dev, slave, rmtt->com.res_id, RES_MTT); + + return err; +} + +int mlx4_HW2SW_EQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int eqn = vhcr->in_modifier; + int res_id = eqn | (slave << 10); + struct res_eq *eq; + int err; + + err = eq_res_start_move_to(dev, slave, res_id, RES_EQ_RESERVED, &eq); + if (err) + return err; + + err = get_res(dev, slave, eq->mtt->com.res_id, RES_MTT, NULL); + if (err) + goto ex_abort; + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto ex_put; + + atomic_dec(&eq->mtt->ref_count); + put_res(dev, slave, eq->mtt->com.res_id, RES_MTT); + res_end_move(dev, slave, RES_EQ, res_id); + rem_res_range(dev, slave, res_id, 1, RES_EQ, 0); + + return 0; + +ex_put: + put_res(dev, slave, eq->mtt->com.res_id, RES_MTT); +ex_abort: + res_abort_move(dev, slave, RES_EQ, res_id); + + return err; +} + +int mlx4_GEN_EQE(struct mlx4_dev *dev, int slave, struct mlx4_eqe *eqe) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_event_eq_info *event_eq; + struct mlx4_cmd_mailbox *mailbox; + u32 in_modifier = 0; + int err; + int res_id; + struct res_eq *req; + + if (!priv->mfunc.master.slave_state) + return -EINVAL; + + /* check for slave valid, slave not PF, and slave active */ + if (slave < 0 || slave > dev->persist->num_vfs || + slave == dev->caps.function || + !priv->mfunc.master.slave_state[slave].active) + return 0; + + event_eq = &priv->mfunc.master.slave_state[slave].event_eq[eqe->type]; + + /* Create the event only if the slave is registered */ + if (event_eq->eqn < 0) + return 0; + + mutex_lock(&priv->mfunc.master.gen_eqe_mutex[slave]); + res_id = (slave << 10) | event_eq->eqn; + err = get_res(dev, slave, res_id, RES_EQ, &req); + if (err) + goto unlock; + + if (req->com.from_state != RES_EQ_HW) { + err = -EINVAL; + goto put; + } + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + err = PTR_ERR(mailbox); + goto put; + } + + if (eqe->type == MLX4_EVENT_TYPE_CMD) { + ++event_eq->token; + eqe->event.cmd.token = cpu_to_be16(event_eq->token); + } + + memcpy(mailbox->buf, (u8 *) eqe, 28); + + in_modifier = (slave & 0xff) | ((event_eq->eqn & 0x3ff) << 16); + + err = mlx4_cmd(dev, mailbox->dma, in_modifier, 0, + MLX4_CMD_GEN_EQE, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); + + put_res(dev, slave, res_id, RES_EQ); + mutex_unlock(&priv->mfunc.master.gen_eqe_mutex[slave]); + mlx4_free_cmd_mailbox(dev, mailbox); + return err; + +put: + put_res(dev, slave, res_id, RES_EQ); + +unlock: + mutex_unlock(&priv->mfunc.master.gen_eqe_mutex[slave]); + return err; +} + +int mlx4_QUERY_EQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int eqn = vhcr->in_modifier; + int res_id = eqn | (slave << 10); + struct res_eq *eq; + int err; + + err = get_res(dev, slave, res_id, RES_EQ, &eq); + if (err) + return err; + + if (eq->com.from_state != RES_EQ_HW) { + err = -EINVAL; + goto ex_put; + } + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + +ex_put: + put_res(dev, slave, res_id, RES_EQ); + return err; +} + +int mlx4_SW2HW_CQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int cqn = vhcr->in_modifier; + struct mlx4_cq_context *cqc = inbox->buf; + int mtt_base = cq_get_mtt_addr(cqc) / dev->caps.mtt_entry_sz; + struct res_cq *cq = NULL; + struct res_mtt *mtt; + + err = cq_res_start_move_to(dev, slave, cqn, RES_CQ_HW, &cq); + if (err) + return err; + err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); + if (err) + goto out_move; + err = check_mtt_range(dev, slave, mtt_base, cq_get_mtt_size(cqc), mtt); + if (err) + goto out_put; + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto out_put; + atomic_inc(&mtt->ref_count); + cq->mtt = mtt; + put_res(dev, slave, mtt->com.res_id, RES_MTT); + res_end_move(dev, slave, RES_CQ, cqn); + return 0; + +out_put: + put_res(dev, slave, mtt->com.res_id, RES_MTT); +out_move: + res_abort_move(dev, slave, RES_CQ, cqn); + return err; +} + +int mlx4_HW2SW_CQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int cqn = vhcr->in_modifier; + struct res_cq *cq = NULL; + + err = cq_res_start_move_to(dev, slave, cqn, RES_CQ_ALLOCATED, &cq); + if (err) + return err; + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto out_move; + atomic_dec(&cq->mtt->ref_count); + res_end_move(dev, slave, RES_CQ, cqn); + return 0; + +out_move: + res_abort_move(dev, slave, RES_CQ, cqn); + return err; +} + +int mlx4_QUERY_CQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int cqn = vhcr->in_modifier; + struct res_cq *cq; + int err; + + err = get_res(dev, slave, cqn, RES_CQ, &cq); + if (err) + return err; + + if (cq->com.from_state != RES_CQ_HW) + goto ex_put; + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +ex_put: + put_res(dev, slave, cqn, RES_CQ); + + return err; +} + +static int handle_resize(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd, + struct res_cq *cq) +{ + int err; + struct res_mtt *orig_mtt; + struct res_mtt *mtt; + struct mlx4_cq_context *cqc = inbox->buf; + int mtt_base = cq_get_mtt_addr(cqc) / dev->caps.mtt_entry_sz; + + err = get_res(dev, slave, cq->mtt->com.res_id, RES_MTT, &orig_mtt); + if (err) + return err; + + if (orig_mtt != cq->mtt) { + err = -EINVAL; + goto ex_put; + } + + err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); + if (err) + goto ex_put; + + err = check_mtt_range(dev, slave, mtt_base, cq_get_mtt_size(cqc), mtt); + if (err) + goto ex_put1; + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto ex_put1; + atomic_dec(&orig_mtt->ref_count); + put_res(dev, slave, orig_mtt->com.res_id, RES_MTT); + atomic_inc(&mtt->ref_count); + cq->mtt = mtt; + put_res(dev, slave, mtt->com.res_id, RES_MTT); + return 0; + +ex_put1: + put_res(dev, slave, mtt->com.res_id, RES_MTT); +ex_put: + put_res(dev, slave, orig_mtt->com.res_id, RES_MTT); + + return err; + +} + +int mlx4_MODIFY_CQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int cqn = vhcr->in_modifier; + struct res_cq *cq; + int err; + + err = get_res(dev, slave, cqn, RES_CQ, &cq); + if (err) + return err; + + if (cq->com.from_state != RES_CQ_HW) + goto ex_put; + + if (vhcr->op_modifier == 0) { + err = handle_resize(dev, slave, vhcr, inbox, outbox, cmd, cq); + goto ex_put; + } + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +ex_put: + put_res(dev, slave, cqn, RES_CQ); + + return err; +} + +static int srq_get_mtt_size(struct mlx4_srq_context *srqc) +{ + int log_srq_size = (be32_to_cpu(srqc->state_logsize_srqn) >> 24) & 0xf; + int log_rq_stride = srqc->logstride & 7; + int page_shift = (srqc->log_page_size & 0x3f) + 12; + + if (log_srq_size + log_rq_stride + 4 < page_shift) + return 1; + + return 1 << (log_srq_size + log_rq_stride + 4 - page_shift); +} + +int mlx4_SW2HW_SRQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int srqn = vhcr->in_modifier; + struct res_mtt *mtt; + struct res_srq *srq = NULL; + struct mlx4_srq_context *srqc = inbox->buf; + int mtt_base = srq_get_mtt_addr(srqc) / dev->caps.mtt_entry_sz; + + if (srqn != (be32_to_cpu(srqc->state_logsize_srqn) & 0xffffff)) + return -EINVAL; + + err = srq_res_start_move_to(dev, slave, srqn, RES_SRQ_HW, &srq); + if (err) + return err; + err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); + if (err) + goto ex_abort; + err = check_mtt_range(dev, slave, mtt_base, srq_get_mtt_size(srqc), + mtt); + if (err) + goto ex_put_mtt; + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto ex_put_mtt; + + atomic_inc(&mtt->ref_count); + srq->mtt = mtt; + put_res(dev, slave, mtt->com.res_id, RES_MTT); + res_end_move(dev, slave, RES_SRQ, srqn); + return 0; + +ex_put_mtt: + put_res(dev, slave, mtt->com.res_id, RES_MTT); +ex_abort: + res_abort_move(dev, slave, RES_SRQ, srqn); + + return err; +} + +int mlx4_HW2SW_SRQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int srqn = vhcr->in_modifier; + struct res_srq *srq = NULL; + + err = srq_res_start_move_to(dev, slave, srqn, RES_SRQ_ALLOCATED, &srq); + if (err) + return err; + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto ex_abort; + atomic_dec(&srq->mtt->ref_count); + if (srq->cq) + atomic_dec(&srq->cq->ref_count); + res_end_move(dev, slave, RES_SRQ, srqn); + + return 0; + +ex_abort: + res_abort_move(dev, slave, RES_SRQ, srqn); + + return err; +} + +int mlx4_QUERY_SRQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int srqn = vhcr->in_modifier; + struct res_srq *srq; + + err = get_res(dev, slave, srqn, RES_SRQ, &srq); + if (err) + return err; + if (srq->com.from_state != RES_SRQ_HW) { + err = -EBUSY; + goto out; + } + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +out: + put_res(dev, slave, srqn, RES_SRQ); + return err; +} + +int mlx4_ARM_SRQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int srqn = vhcr->in_modifier; + struct res_srq *srq; + + err = get_res(dev, slave, srqn, RES_SRQ, &srq); + if (err) + return err; + + if (srq->com.from_state != RES_SRQ_HW) { + err = -EBUSY; + goto out; + } + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +out: + put_res(dev, slave, srqn, RES_SRQ); + return err; +} + +int mlx4_GEN_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int qpn = vhcr->in_modifier & 0x7fffff; + struct res_qp *qp; + + err = get_res(dev, slave, qpn, RES_QP, &qp); + if (err) + return err; + if (qp->com.from_state != RES_QP_HW) { + err = -EBUSY; + goto out; + } + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +out: + put_res(dev, slave, qpn, RES_QP); + return err; +} + +int mlx4_INIT2INIT_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_qp_context *context = inbox->buf + 8; + adjust_proxy_tun_qkey(dev, vhcr, context); + update_pkey_index(dev, slave, inbox); + return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +} + +static int adjust_qp_sched_queue(struct mlx4_dev *dev, int slave, + struct mlx4_qp_context *qpc, + struct mlx4_cmd_mailbox *inbox) +{ + enum mlx4_qp_optpar optpar = be32_to_cpu(*(__be32 *)inbox->buf); + u8 pri_sched_queue; + int port = mlx4_slave_convert_port( + dev, slave, (qpc->pri_path.sched_queue >> 6 & 1) + 1) - 1; + + if (port < 0) + return -EINVAL; + + pri_sched_queue = (qpc->pri_path.sched_queue & ~(1 << 6)) | + ((port & 1) << 6); + + if (optpar & (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH | MLX4_QP_OPTPAR_SCHED_QUEUE) || + qpc->pri_path.sched_queue || mlx4_is_eth(dev, port + 1)) { + qpc->pri_path.sched_queue = pri_sched_queue; + } + + if (optpar & MLX4_QP_OPTPAR_ALT_ADDR_PATH) { + port = mlx4_slave_convert_port( + dev, slave, (qpc->alt_path.sched_queue >> 6 & 1) + + 1) - 1; + if (port < 0) + return -EINVAL; + qpc->alt_path.sched_queue = + (qpc->alt_path.sched_queue & ~(1 << 6)) | + (port & 1) << 6; + } + return 0; +} + +static int roce_verify_mac(struct mlx4_dev *dev, int slave, + struct mlx4_qp_context *qpc, + struct mlx4_cmd_mailbox *inbox) +{ + u64 mac; + int port; + u32 ts = (be32_to_cpu(qpc->flags) >> 16) & 0xff; + u8 sched = *(u8 *)(inbox->buf + 64); + u8 smac_ix; + + port = (sched >> 6 & 1) + 1; + if (mlx4_is_eth(dev, port) && (ts != MLX4_QP_ST_MLX)) { + smac_ix = qpc->pri_path.grh_mylmc & 0x7f; + if (mac_find_smac_ix_in_slave(dev, slave, port, smac_ix, &mac)) + return -ENOENT; + } + return 0; +} + +int mlx4_INIT2RTR_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + struct mlx4_qp_context *qpc = inbox->buf + 8; + int qpn = vhcr->in_modifier & 0x7fffff; + struct res_qp *qp; + u8 orig_sched_queue; + __be32 orig_param3 = qpc->param3; + u8 orig_vlan_control = qpc->pri_path.vlan_control; + u8 orig_fvl_rx = qpc->pri_path.fvl_rx; + u8 orig_pri_path_fl = qpc->pri_path.fl; + u8 orig_vlan_index = qpc->pri_path.vlan_index; + u8 orig_feup = qpc->pri_path.feup; + + err = adjust_qp_sched_queue(dev, slave, qpc, inbox); + if (err) + return err; + err = verify_qp_parameters(dev, vhcr, inbox, QP_TRANS_INIT2RTR, slave); + if (err) + return err; + + if (roce_verify_mac(dev, slave, qpc, inbox)) + return -EINVAL; + + update_pkey_index(dev, slave, inbox); + update_gid(dev, inbox, (u8)slave); + adjust_proxy_tun_qkey(dev, vhcr, qpc); + orig_sched_queue = qpc->pri_path.sched_queue; + + err = get_res(dev, slave, qpn, RES_QP, &qp); + if (err) + return err; + if (qp->com.from_state != RES_QP_HW) { + err = -EBUSY; + goto out; + } + + err = update_vport_qp_param(dev, inbox, slave, qpn); + if (err) + goto out; + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +out: + /* if no error, save sched queue value passed in by VF. This is + * essentially the QOS value provided by the VF. This will be useful + * if we allow dynamic changes from VST back to VGT + */ + if (!err) { + qp->sched_queue = orig_sched_queue; + qp->param3 = orig_param3; + qp->vlan_control = orig_vlan_control; + qp->fvl_rx = orig_fvl_rx; + qp->pri_path_fl = orig_pri_path_fl; + qp->vlan_index = orig_vlan_index; + qp->feup = orig_feup; + } + put_res(dev, slave, qpn, RES_QP); + return err; +} + +int mlx4_RTR2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + struct mlx4_qp_context *context = inbox->buf + 8; + + err = adjust_qp_sched_queue(dev, slave, context, inbox); + if (err) + return err; + err = verify_qp_parameters(dev, vhcr, inbox, QP_TRANS_RTR2RTS, slave); + if (err) + return err; + + update_pkey_index(dev, slave, inbox); + update_gid(dev, inbox, (u8)slave); + adjust_proxy_tun_qkey(dev, vhcr, context); + return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +} + +int mlx4_RTS2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + struct mlx4_qp_context *context = inbox->buf + 8; + + err = adjust_qp_sched_queue(dev, slave, context, inbox); + if (err) + return err; + err = verify_qp_parameters(dev, vhcr, inbox, QP_TRANS_RTS2RTS, slave); + if (err) + return err; + + update_pkey_index(dev, slave, inbox); + update_gid(dev, inbox, (u8)slave); + adjust_proxy_tun_qkey(dev, vhcr, context); + return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +} + + +int mlx4_SQERR2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_qp_context *context = inbox->buf + 8; + int err = adjust_qp_sched_queue(dev, slave, context, inbox); + if (err) + return err; + adjust_proxy_tun_qkey(dev, vhcr, context); + return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +} + +int mlx4_SQD2SQD_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + struct mlx4_qp_context *context = inbox->buf + 8; + + err = adjust_qp_sched_queue(dev, slave, context, inbox); + if (err) + return err; + err = verify_qp_parameters(dev, vhcr, inbox, QP_TRANS_SQD2SQD, slave); + if (err) + return err; + + adjust_proxy_tun_qkey(dev, vhcr, context); + update_gid(dev, inbox, (u8)slave); + update_pkey_index(dev, slave, inbox); + return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +} + +int mlx4_SQD2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + struct mlx4_qp_context *context = inbox->buf + 8; + + err = adjust_qp_sched_queue(dev, slave, context, inbox); + if (err) + return err; + err = verify_qp_parameters(dev, vhcr, inbox, QP_TRANS_SQD2RTS, slave); + if (err) + return err; + + adjust_proxy_tun_qkey(dev, vhcr, context); + update_gid(dev, inbox, (u8)slave); + update_pkey_index(dev, slave, inbox); + return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +} + +int mlx4_2RST_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int qpn = vhcr->in_modifier & 0x7fffff; + struct res_qp *qp; + + err = qp_res_start_move_to(dev, slave, qpn, RES_QP_MAPPED, &qp, 0); + if (err) + return err; + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto ex_abort; + + atomic_dec(&qp->mtt->ref_count); + atomic_dec(&qp->rcq->ref_count); + atomic_dec(&qp->scq->ref_count); + if (qp->srq) + atomic_dec(&qp->srq->ref_count); + res_end_move(dev, slave, RES_QP, qpn); + return 0; + +ex_abort: + res_abort_move(dev, slave, RES_QP, qpn); + + return err; +} + +static struct res_gid *find_gid(struct mlx4_dev *dev, int slave, + struct res_qp *rqp, u8 *gid) +{ + struct res_gid *res; + + list_for_each_entry(res, &rqp->mcg_list, list) { + if (!memcmp(res->gid, gid, 16)) + return res; + } + return NULL; +} + +static int add_mcg_res(struct mlx4_dev *dev, int slave, struct res_qp *rqp, + u8 *gid, enum mlx4_protocol prot, + enum mlx4_steer_type steer, u64 reg_id) +{ + struct res_gid *res; + int err; + + res = kzalloc(sizeof *res, GFP_KERNEL); + if (!res) + return -ENOMEM; + + spin_lock_irq(&rqp->mcg_spl); + if (find_gid(dev, slave, rqp, gid)) { + kfree(res); + err = -EEXIST; + } else { + memcpy(res->gid, gid, 16); + res->prot = prot; + res->steer = steer; + res->reg_id = reg_id; + list_add_tail(&res->list, &rqp->mcg_list); + err = 0; + } + spin_unlock_irq(&rqp->mcg_spl); + + return err; +} + +static int rem_mcg_res(struct mlx4_dev *dev, int slave, struct res_qp *rqp, + u8 *gid, enum mlx4_protocol prot, + enum mlx4_steer_type steer, u64 *reg_id) +{ + struct res_gid *res; + int err; + + spin_lock_irq(&rqp->mcg_spl); + res = find_gid(dev, slave, rqp, gid); + if (!res || res->prot != prot || res->steer != steer) + err = -EINVAL; + else { + *reg_id = res->reg_id; + list_del(&res->list); + kfree(res); + err = 0; + } + spin_unlock_irq(&rqp->mcg_spl); + + return err; +} + +static int qp_attach(struct mlx4_dev *dev, int slave, struct mlx4_qp *qp, + u8 gid[16], int block_loopback, enum mlx4_protocol prot, + enum mlx4_steer_type type, u64 *reg_id) +{ + switch (dev->caps.steering_mode) { + case MLX4_STEERING_MODE_DEVICE_MANAGED: { + int port = mlx4_slave_convert_port(dev, slave, gid[5]); + if (port < 0) + return port; + return mlx4_trans_to_dmfs_attach(dev, qp, gid, port, + block_loopback, prot, + reg_id); + } + case MLX4_STEERING_MODE_B0: + if (prot == MLX4_PROT_ETH) { + int port = mlx4_slave_convert_port(dev, slave, gid[5]); + if (port < 0) + return port; + gid[5] = port; + } + return mlx4_qp_attach_common(dev, qp, gid, + block_loopback, prot, type); + default: + return -EINVAL; + } +} + +static int qp_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, + u8 gid[16], enum mlx4_protocol prot, + enum mlx4_steer_type type, u64 reg_id) +{ + switch (dev->caps.steering_mode) { + case MLX4_STEERING_MODE_DEVICE_MANAGED: + return mlx4_flow_detach(dev, reg_id); + case MLX4_STEERING_MODE_B0: + return mlx4_qp_detach_common(dev, qp, gid, prot, type); + default: + return -EINVAL; + } +} + +static int mlx4_adjust_port(struct mlx4_dev *dev, int slave, + u8 *gid, enum mlx4_protocol prot) +{ + int real_port; + + if (prot != MLX4_PROT_ETH) + return 0; + + if (dev->caps.steering_mode == MLX4_STEERING_MODE_B0 || + dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) { + real_port = mlx4_slave_convert_port(dev, slave, gid[5]); + if (real_port < 0) + return -EINVAL; + gid[5] = real_port; + } + + return 0; +} + +int mlx4_QP_ATTACH_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_qp qp; /* dummy for calling attach/detach */ + u8 *gid = inbox->buf; + enum mlx4_protocol prot = (vhcr->in_modifier >> 28) & 0x7; + int err; + int qpn; + struct res_qp *rqp; + u64 reg_id = 0; + int attach = vhcr->op_modifier; + int block_loopback = vhcr->in_modifier >> 31; + u8 steer_type_mask = 2; + enum mlx4_steer_type type = (gid[7] & steer_type_mask) >> 1; + + qpn = vhcr->in_modifier & 0xffffff; + err = get_res(dev, slave, qpn, RES_QP, &rqp); + if (err) + return err; + + qp.qpn = qpn; + if (attach) { + err = qp_attach(dev, slave, &qp, gid, block_loopback, prot, + type, ®_id); + if (err) { + pr_err("Fail to attach rule to qp 0x%x\n", qpn); + goto ex_put; + } + err = add_mcg_res(dev, slave, rqp, gid, prot, type, reg_id); + if (err) + goto ex_detach; + } else { + err = mlx4_adjust_port(dev, slave, gid, prot); + if (err) + goto ex_put; + + err = rem_mcg_res(dev, slave, rqp, gid, prot, type, ®_id); + if (err) + goto ex_put; + + err = qp_detach(dev, &qp, gid, prot, type, reg_id); + if (err) + pr_err("Fail to detach rule from qp 0x%x reg_id = 0x%llx\n", + qpn, (unsigned long long)reg_id); + } + put_res(dev, slave, qpn, RES_QP); + return err; + +ex_detach: + qp_detach(dev, &qp, gid, prot, type, reg_id); +ex_put: + put_res(dev, slave, qpn, RES_QP); + return err; +} + +/* + * MAC validation for Flow Steering rules. + * VF can attach rules only with a mac address which is assigned to it. + */ +static int validate_eth_header_mac(int slave, struct _rule_hw *eth_header, + struct list_head *rlist) +{ + struct mac_res *res, *tmp; + __be64 be_mac; + + /* make sure it isn't multicast or broadcast mac*/ + if (!is_multicast_ether_addr(eth_header->eth.dst_mac) && + !is_broadcast_ether_addr(eth_header->eth.dst_mac)) { + list_for_each_entry_safe(res, tmp, rlist, list) { + be_mac = cpu_to_be64(res->mac << 16); + if (ether_addr_equal((u8 *)&be_mac, eth_header->eth.dst_mac)) + return 0; + } + pr_err("MAC %pM doesn't belong to VF %d, Steering rule rejected\n", + eth_header->eth.dst_mac, slave); + return -EINVAL; + } + return 0; +} + +static void handle_eth_header_mcast_prio(struct mlx4_net_trans_rule_hw_ctrl *ctrl, + struct _rule_hw *eth_header) +{ + if (is_multicast_ether_addr(eth_header->eth.dst_mac) || + is_broadcast_ether_addr(eth_header->eth.dst_mac)) { + struct mlx4_net_trans_rule_hw_eth *eth = + (struct mlx4_net_trans_rule_hw_eth *)eth_header; + struct _rule_hw *next_rule = (struct _rule_hw *)(eth + 1); + bool last_rule = next_rule->size == 0 && next_rule->id == 0 && + next_rule->rsvd == 0; + + if (last_rule) + ctrl->prio = cpu_to_be16(MLX4_DOMAIN_NIC); + } +} + +/* + * In case of missing eth header, append eth header with a MAC address + * assigned to the VF. + */ +static int add_eth_header(struct mlx4_dev *dev, int slave, + struct mlx4_cmd_mailbox *inbox, + struct list_head *rlist, int header_id) +{ + struct mac_res *res, *tmp; + u8 port; + struct mlx4_net_trans_rule_hw_ctrl *ctrl; + struct mlx4_net_trans_rule_hw_eth *eth_header; + struct mlx4_net_trans_rule_hw_ipv4 *ip_header; + struct mlx4_net_trans_rule_hw_tcp_udp *l4_header; + __be64 be_mac = 0; + __be64 mac_msk = cpu_to_be64(MLX4_MAC_MASK << 16); + + ctrl = (struct mlx4_net_trans_rule_hw_ctrl *)inbox->buf; + port = ctrl->port; + eth_header = (struct mlx4_net_trans_rule_hw_eth *)(ctrl + 1); + + /* Clear a space in the inbox for eth header */ + switch (header_id) { + case MLX4_NET_TRANS_RULE_ID_IPV4: + ip_header = + (struct mlx4_net_trans_rule_hw_ipv4 *)(eth_header + 1); + memmove(ip_header, eth_header, + sizeof(*ip_header) + sizeof(*l4_header)); + break; + case MLX4_NET_TRANS_RULE_ID_TCP: + case MLX4_NET_TRANS_RULE_ID_UDP: + l4_header = (struct mlx4_net_trans_rule_hw_tcp_udp *) + (eth_header + 1); + memmove(l4_header, eth_header, sizeof(*l4_header)); + break; + default: + return -EINVAL; + } + list_for_each_entry_safe(res, tmp, rlist, list) { + if (port == res->port) { + be_mac = cpu_to_be64(res->mac << 16); + break; + } + } + if (!be_mac) { + pr_err("Failed adding eth header to FS rule, Can't find matching MAC for port %d\n", + port); + return -EINVAL; + } + + memset(eth_header, 0, sizeof(*eth_header)); + eth_header->size = sizeof(*eth_header) >> 2; + eth_header->id = cpu_to_be16(__sw_id_hw[MLX4_NET_TRANS_RULE_ID_ETH]); + memcpy(eth_header->dst_mac, &be_mac, ETH_ALEN); + memcpy(eth_header->dst_mac_msk, &mac_msk, ETH_ALEN); + + return 0; +} + +#define MLX4_UPD_QP_PATH_MASK_SUPPORTED ( \ + 1ULL << MLX4_UPD_QP_PATH_MASK_MAC_INDEX |\ + 1ULL << MLX4_UPD_QP_PATH_MASK_ETH_SRC_CHECK_MC_LB) +int mlx4_UPDATE_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd_info) +{ + int err; + u32 qpn = vhcr->in_modifier & 0xffffff; + struct res_qp *rqp; + u64 mac; + unsigned port; + u64 pri_addr_path_mask; + struct mlx4_update_qp_context *cmd; + int smac_index; + + cmd = (struct mlx4_update_qp_context *)inbox->buf; + + pri_addr_path_mask = be64_to_cpu(cmd->primary_addr_path_mask); + if (cmd->qp_mask || cmd->secondary_addr_path_mask || + (pri_addr_path_mask & ~MLX4_UPD_QP_PATH_MASK_SUPPORTED)) + return -EPERM; + + if ((pri_addr_path_mask & + (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_SRC_CHECK_MC_LB)) && + !(dev->caps.flags2 & + MLX4_DEV_CAP_FLAG2_UPDATE_QP_SRC_CHECK_LB)) { + mlx4_warn(dev, "Src check LB for slave %d isn't supported\n", + slave); + return -ENOTSUPP; + } + + /* Just change the smac for the QP */ + err = get_res(dev, slave, qpn, RES_QP, &rqp); + if (err) { + mlx4_err(dev, "Updating qpn 0x%x for slave %d rejected\n", qpn, slave); + return err; + } + + port = (rqp->sched_queue >> 6 & 1) + 1; + + if (pri_addr_path_mask & (1ULL << MLX4_UPD_QP_PATH_MASK_MAC_INDEX)) { + smac_index = cmd->qp_context.pri_path.grh_mylmc; + err = mac_find_smac_ix_in_slave(dev, slave, port, + smac_index, &mac); + + if (err) { + mlx4_err(dev, "Failed to update qpn 0x%x, MAC is invalid. smac_ix: %d\n", + qpn, smac_index); + goto err_mac; + } + } + + err = mlx4_cmd(dev, inbox->dma, + vhcr->in_modifier, 0, + MLX4_CMD_UPDATE_QP, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) { + mlx4_err(dev, "Failed to update qpn on qpn 0x%x, command failed\n", qpn); + goto err_mac; + } + +err_mac: + put_res(dev, slave, qpn, RES_QP); + return err; +} + +static u32 qp_attach_mbox_size(void *mbox) +{ + u32 size = sizeof(struct mlx4_net_trans_rule_hw_ctrl); + struct _rule_hw *rule_header; + + rule_header = (struct _rule_hw *)(mbox + size); + + while (rule_header->size) { + size += rule_header->size * sizeof(u32); + rule_header += 1; + } + return size; +} + +static int mlx4_do_mirror_rule(struct mlx4_dev *dev, struct res_fs_rule *fs_rule); + +int mlx4_QP_FLOW_STEERING_ATTACH_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *rlist = &tracker->slave_list[slave].res_list[RES_MAC]; + int err; + int qpn; + struct res_qp *rqp; + struct mlx4_net_trans_rule_hw_ctrl *ctrl; + struct _rule_hw *rule_header; + int header_id; + struct res_fs_rule *rrule; + u32 mbox_size; + + if (dev->caps.steering_mode != + MLX4_STEERING_MODE_DEVICE_MANAGED) + return -EOPNOTSUPP; + + ctrl = (struct mlx4_net_trans_rule_hw_ctrl *)inbox->buf; + err = mlx4_slave_convert_port(dev, slave, ctrl->port); + if (err <= 0) + return -EINVAL; + ctrl->port = err; + qpn = be32_to_cpu(ctrl->qpn) & 0xffffff; + err = get_res(dev, slave, qpn, RES_QP, &rqp); + if (err) { + pr_err("Steering rule with qpn 0x%x rejected\n", qpn); + return err; + } + rule_header = (struct _rule_hw *)(ctrl + 1); + header_id = map_hw_to_sw_id(be16_to_cpu(rule_header->id)); + + if (header_id == MLX4_NET_TRANS_RULE_ID_ETH) + handle_eth_header_mcast_prio(ctrl, rule_header); + + if (slave == dev->caps.function) + goto execute; + + switch (header_id) { + case MLX4_NET_TRANS_RULE_ID_ETH: + if (validate_eth_header_mac(slave, rule_header, rlist)) { + err = -EINVAL; + goto err_put_qp; + } + break; + case MLX4_NET_TRANS_RULE_ID_IB: + break; + case MLX4_NET_TRANS_RULE_ID_IPV4: + case MLX4_NET_TRANS_RULE_ID_TCP: + case MLX4_NET_TRANS_RULE_ID_UDP: + pr_warn("Can't attach FS rule without L2 headers, adding L2 header\n"); + if (add_eth_header(dev, slave, inbox, rlist, header_id)) { + err = -EINVAL; + goto err_put_qp; + } + vhcr->in_modifier += + sizeof(struct mlx4_net_trans_rule_hw_eth) >> 2; + break; + default: + pr_err("Corrupted mailbox\n"); + err = -EINVAL; + goto err_put_qp; + } + +execute: + err = mlx4_cmd_imm(dev, inbox->dma, &vhcr->out_param, + vhcr->in_modifier, 0, + MLX4_QP_FLOW_STEERING_ATTACH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + goto err_put_qp; + + + err = add_res_range(dev, slave, vhcr->out_param, 1, RES_FS_RULE, qpn); + if (err) { + mlx4_err(dev, "Fail to add flow steering resources\n"); + goto err_detach; + } + + err = get_res(dev, slave, vhcr->out_param, RES_FS_RULE, &rrule); + if (err) + goto err_detach; + + mbox_size = qp_attach_mbox_size(inbox->buf); + rrule->mirr_mbox = kmalloc(mbox_size, GFP_KERNEL); + if (!rrule->mirr_mbox) { + err = -ENOMEM; + goto err_put_rule; + } + rrule->mirr_mbox_size = mbox_size; + rrule->mirr_rule_id = 0; + memcpy(rrule->mirr_mbox, inbox->buf, mbox_size); + + /* set different port */ + ctrl = (struct mlx4_net_trans_rule_hw_ctrl *)rrule->mirr_mbox; + if (ctrl->port == 1) + ctrl->port = 2; + else + ctrl->port = 1; + + if (mlx4_is_bonded(dev)) + mlx4_do_mirror_rule(dev, rrule); + + atomic_inc(&rqp->ref_count); + +err_put_rule: + put_res(dev, slave, vhcr->out_param, RES_FS_RULE); +err_detach: + /* detach rule on error */ + if (err) + mlx4_cmd(dev, vhcr->out_param, 0, 0, + MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); +err_put_qp: + put_res(dev, slave, qpn, RES_QP); + return err; +} + +static int mlx4_undo_mirror_rule(struct mlx4_dev *dev, struct res_fs_rule *fs_rule) +{ + int err; + + err = rem_res_range(dev, fs_rule->com.owner, fs_rule->com.res_id, 1, RES_FS_RULE, 0); + if (err) { + mlx4_err(dev, "Fail to remove flow steering resources\n"); + return err; + } + + mlx4_cmd(dev, fs_rule->com.res_id, 0, 0, MLX4_QP_FLOW_STEERING_DETACH, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + return 0; +} + +int mlx4_QP_FLOW_STEERING_DETACH_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + struct res_qp *rqp; + struct res_fs_rule *rrule; + u64 mirr_reg_id; + + if (dev->caps.steering_mode != + MLX4_STEERING_MODE_DEVICE_MANAGED) + return -EOPNOTSUPP; + + err = get_res(dev, slave, vhcr->in_param, RES_FS_RULE, &rrule); + if (err) + return err; + + if (!rrule->mirr_mbox) { + mlx4_err(dev, "Mirror rules cannot be removed explicitly\n"); + put_res(dev, slave, vhcr->in_param, RES_FS_RULE); + return -EINVAL; + } + mirr_reg_id = rrule->mirr_rule_id; + kfree(rrule->mirr_mbox); + + /* Release the rule form busy state before removal */ + put_res(dev, slave, vhcr->in_param, RES_FS_RULE); + err = get_res(dev, slave, rrule->qpn, RES_QP, &rqp); + if (err) + return err; + + if (mirr_reg_id && mlx4_is_bonded(dev)) { + err = get_res(dev, slave, mirr_reg_id, RES_FS_RULE, &rrule); + if (err) { + mlx4_err(dev, "Fail to get resource of mirror rule\n"); + } else { + put_res(dev, slave, mirr_reg_id, RES_FS_RULE); + mlx4_undo_mirror_rule(dev, rrule); + } + } + err = rem_res_range(dev, slave, vhcr->in_param, 1, RES_FS_RULE, 0); + if (err) { + mlx4_err(dev, "Fail to remove flow steering resources\n"); + goto out; + } + + err = mlx4_cmd(dev, vhcr->in_param, 0, 0, + MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (!err) + atomic_dec(&rqp->ref_count); +out: + put_res(dev, slave, rrule->qpn, RES_QP); + return err; +} + +enum { + BUSY_MAX_RETRIES = 10 +}; + +int mlx4_QUERY_IF_STAT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int index = vhcr->in_modifier & 0xffff; + + err = get_res(dev, slave, index, RES_COUNTER, NULL); + if (err) + return err; + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + put_res(dev, slave, index, RES_COUNTER); + return err; +} + +static void detach_qp(struct mlx4_dev *dev, int slave, struct res_qp *rqp) +{ + struct res_gid *rgid; + struct res_gid *tmp; + struct mlx4_qp qp; /* dummy for calling attach/detach */ + + list_for_each_entry_safe(rgid, tmp, &rqp->mcg_list, list) { + switch (dev->caps.steering_mode) { + case MLX4_STEERING_MODE_DEVICE_MANAGED: + mlx4_flow_detach(dev, rgid->reg_id); + break; + case MLX4_STEERING_MODE_B0: + qp.qpn = rqp->local_qpn; + (void) mlx4_qp_detach_common(dev, &qp, rgid->gid, + rgid->prot, rgid->steer); + break; + } + list_del(&rgid->list); + kfree(rgid); + } +} + +static int _move_all_busy(struct mlx4_dev *dev, int slave, + enum mlx4_resource type, int print) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = + &priv->mfunc.master.res_tracker; + struct list_head *rlist = &tracker->slave_list[slave].res_list[type]; + struct res_common *r; + struct res_common *tmp; + int busy; + + busy = 0; + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(r, tmp, rlist, list) { + if (r->owner == slave) { + if (!r->removing) { + if (r->state == RES_ANY_BUSY) { + if (print) + mlx4_dbg(dev, + "%s id 0x%llx is busy\n", + resource_str(type), + (long long)r->res_id); + ++busy; + } else { + r->from_state = r->state; + r->state = RES_ANY_BUSY; + r->removing = 1; + } + } + } + } + spin_unlock_irq(mlx4_tlock(dev)); + + return busy; +} + +static int move_all_busy(struct mlx4_dev *dev, int slave, + enum mlx4_resource type) +{ + unsigned long begin; + int busy; + + begin = jiffies; + do { + busy = _move_all_busy(dev, slave, type, 0); + if (time_after(jiffies, begin + 5 * HZ)) + break; + if (busy) + cond_resched(); + } while (busy); + + if (busy) + busy = _move_all_busy(dev, slave, type, 1); + + return busy; +} +static void rem_slave_qps(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *qp_list = + &tracker->slave_list[slave].res_list[RES_QP]; + struct res_qp *qp; + struct res_qp *tmp; + int state; + u64 in_param; + int qpn; + int err; + + err = move_all_busy(dev, slave, RES_QP); + if (err) + mlx4_warn(dev, "rem_slave_qps: Could not move all qps to busy for slave %d\n", + slave); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(qp, tmp, qp_list, com.list) { + spin_unlock_irq(mlx4_tlock(dev)); + if (qp->com.owner == slave) { + qpn = qp->com.res_id; + detach_qp(dev, slave, qp); + state = qp->com.from_state; + while (state != 0) { + switch (state) { + case RES_QP_RESERVED: + spin_lock_irq(mlx4_tlock(dev)); + rb_erase(&qp->com.node, + &tracker->res_tree[RES_QP]); + list_del(&qp->com.list); + spin_unlock_irq(mlx4_tlock(dev)); + if (!valid_reserved(dev, slave, qpn)) { + __mlx4_qp_release_range(dev, qpn, 1); + mlx4_release_resource(dev, slave, + RES_QP, 1, 0); + } + kfree(qp); + state = 0; + break; + case RES_QP_MAPPED: + if (!valid_reserved(dev, slave, qpn)) + __mlx4_qp_free_icm(dev, qpn); + state = RES_QP_RESERVED; + break; + case RES_QP_HW: + in_param = slave; + err = mlx4_cmd(dev, in_param, + qp->local_qpn, 2, + MLX4_CMD_2RST_QP, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + mlx4_dbg(dev, "rem_slave_qps: failed to move slave %d qpn %d to reset\n", + slave, qp->local_qpn); + atomic_dec(&qp->rcq->ref_count); + atomic_dec(&qp->scq->ref_count); + atomic_dec(&qp->mtt->ref_count); + if (qp->srq) + atomic_dec(&qp->srq->ref_count); + state = RES_QP_MAPPED; + break; + default: + state = 0; + } + } + } + spin_lock_irq(mlx4_tlock(dev)); + } + spin_unlock_irq(mlx4_tlock(dev)); +} + +static void rem_slave_srqs(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *srq_list = + &tracker->slave_list[slave].res_list[RES_SRQ]; + struct res_srq *srq; + struct res_srq *tmp; + int state; + u64 in_param; + LIST_HEAD(tlist); + int srqn; + int err; + + err = move_all_busy(dev, slave, RES_SRQ); + if (err) + mlx4_warn(dev, "rem_slave_srqs: Could not move all srqs - too busy for slave %d\n", + slave); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(srq, tmp, srq_list, com.list) { + spin_unlock_irq(mlx4_tlock(dev)); + if (srq->com.owner == slave) { + srqn = srq->com.res_id; + state = srq->com.from_state; + while (state != 0) { + switch (state) { + case RES_SRQ_ALLOCATED: + __mlx4_srq_free_icm(dev, srqn); + spin_lock_irq(mlx4_tlock(dev)); + rb_erase(&srq->com.node, + &tracker->res_tree[RES_SRQ]); + list_del(&srq->com.list); + spin_unlock_irq(mlx4_tlock(dev)); + mlx4_release_resource(dev, slave, + RES_SRQ, 1, 0); + kfree(srq); + state = 0; + break; + + case RES_SRQ_HW: + in_param = slave; + err = mlx4_cmd(dev, in_param, srqn, 1, + MLX4_CMD_HW2SW_SRQ, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + mlx4_dbg(dev, "rem_slave_srqs: failed to move slave %d srq %d to SW ownership\n", + slave, srqn); + + atomic_dec(&srq->mtt->ref_count); + if (srq->cq) + atomic_dec(&srq->cq->ref_count); + state = RES_SRQ_ALLOCATED; + break; + + default: + state = 0; + } + } + } + spin_lock_irq(mlx4_tlock(dev)); + } + spin_unlock_irq(mlx4_tlock(dev)); +} + +static void rem_slave_cqs(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *cq_list = + &tracker->slave_list[slave].res_list[RES_CQ]; + struct res_cq *cq; + struct res_cq *tmp; + int state; + u64 in_param; + LIST_HEAD(tlist); + int cqn; + int err; + + err = move_all_busy(dev, slave, RES_CQ); + if (err) + mlx4_warn(dev, "rem_slave_cqs: Could not move all cqs - too busy for slave %d\n", + slave); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(cq, tmp, cq_list, com.list) { + spin_unlock_irq(mlx4_tlock(dev)); + if (cq->com.owner == slave && !atomic_read(&cq->ref_count)) { + cqn = cq->com.res_id; + state = cq->com.from_state; + while (state != 0) { + switch (state) { + case RES_CQ_ALLOCATED: + __mlx4_cq_free_icm(dev, cqn); + spin_lock_irq(mlx4_tlock(dev)); + rb_erase(&cq->com.node, + &tracker->res_tree[RES_CQ]); + list_del(&cq->com.list); + spin_unlock_irq(mlx4_tlock(dev)); + mlx4_release_resource(dev, slave, + RES_CQ, 1, 0); + kfree(cq); + state = 0; + break; + + case RES_CQ_HW: + in_param = slave; + err = mlx4_cmd(dev, in_param, cqn, 1, + MLX4_CMD_HW2SW_CQ, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + mlx4_dbg(dev, "rem_slave_cqs: failed to move slave %d cq %d to SW ownership\n", + slave, cqn); + atomic_dec(&cq->mtt->ref_count); + state = RES_CQ_ALLOCATED; + break; + + default: + state = 0; + } + } + } + spin_lock_irq(mlx4_tlock(dev)); + } + spin_unlock_irq(mlx4_tlock(dev)); +} + +static void rem_slave_mrs(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *mpt_list = + &tracker->slave_list[slave].res_list[RES_MPT]; + struct res_mpt *mpt; + struct res_mpt *tmp; + int state; + u64 in_param; + LIST_HEAD(tlist); + int mptn; + int err; + + err = move_all_busy(dev, slave, RES_MPT); + if (err) + mlx4_warn(dev, "rem_slave_mrs: Could not move all mpts - too busy for slave %d\n", + slave); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(mpt, tmp, mpt_list, com.list) { + spin_unlock_irq(mlx4_tlock(dev)); + if (mpt->com.owner == slave) { + mptn = mpt->com.res_id; + state = mpt->com.from_state; + while (state != 0) { + switch (state) { + case RES_MPT_RESERVED: + __mlx4_mpt_release(dev, mpt->key); + spin_lock_irq(mlx4_tlock(dev)); + rb_erase(&mpt->com.node, + &tracker->res_tree[RES_MPT]); + list_del(&mpt->com.list); + spin_unlock_irq(mlx4_tlock(dev)); + mlx4_release_resource(dev, slave, + RES_MPT, 1, 0); + kfree(mpt); + state = 0; + break; + + case RES_MPT_MAPPED: + __mlx4_mpt_free_icm(dev, mpt->key); + state = RES_MPT_RESERVED; + break; + + case RES_MPT_HW: + in_param = slave; + err = mlx4_cmd(dev, in_param, mptn, 0, + MLX4_CMD_HW2SW_MPT, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + mlx4_dbg(dev, "rem_slave_mrs: failed to move slave %d mpt %d to SW ownership\n", + slave, mptn); + if (mpt->mtt) + atomic_dec(&mpt->mtt->ref_count); + state = RES_MPT_MAPPED; + break; + default: + state = 0; + } + } + } + spin_lock_irq(mlx4_tlock(dev)); + } + spin_unlock_irq(mlx4_tlock(dev)); +} + +static void rem_slave_mtts(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = + &priv->mfunc.master.res_tracker; + struct list_head *mtt_list = + &tracker->slave_list[slave].res_list[RES_MTT]; + struct res_mtt *mtt; + struct res_mtt *tmp; + int state; + LIST_HEAD(tlist); + int base; + int err; + + err = move_all_busy(dev, slave, RES_MTT); + if (err) + mlx4_warn(dev, "rem_slave_mtts: Could not move all mtts - too busy for slave %d\n", + slave); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(mtt, tmp, mtt_list, com.list) { + spin_unlock_irq(mlx4_tlock(dev)); + if (mtt->com.owner == slave) { + base = mtt->com.res_id; + state = mtt->com.from_state; + while (state != 0) { + switch (state) { + case RES_MTT_ALLOCATED: + __mlx4_free_mtt_range(dev, base, + mtt->order); + spin_lock_irq(mlx4_tlock(dev)); + rb_erase(&mtt->com.node, + &tracker->res_tree[RES_MTT]); + list_del(&mtt->com.list); + spin_unlock_irq(mlx4_tlock(dev)); + mlx4_release_resource(dev, slave, RES_MTT, + 1 << mtt->order, 0); + kfree(mtt); + state = 0; + break; + + default: + state = 0; + } + } + } + spin_lock_irq(mlx4_tlock(dev)); + } + spin_unlock_irq(mlx4_tlock(dev)); +} + +static int mlx4_do_mirror_rule(struct mlx4_dev *dev, struct res_fs_rule *fs_rule) +{ + struct mlx4_cmd_mailbox *mailbox; + int err; + struct res_fs_rule *mirr_rule; + u64 reg_id; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + if (!fs_rule->mirr_mbox) { + mlx4_err(dev, "rule mirroring mailbox is null\n"); + return -EINVAL; + } + memcpy(mailbox->buf, fs_rule->mirr_mbox, fs_rule->mirr_mbox_size); + err = mlx4_cmd_imm(dev, mailbox->dma, ®_id, fs_rule->mirr_mbox_size >> 2, 0, + MLX4_QP_FLOW_STEERING_ATTACH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + mlx4_free_cmd_mailbox(dev, mailbox); + + if (err) + goto err; + + err = add_res_range(dev, fs_rule->com.owner, reg_id, 1, RES_FS_RULE, fs_rule->qpn); + if (err) + goto err_detach; + + err = get_res(dev, fs_rule->com.owner, reg_id, RES_FS_RULE, &mirr_rule); + if (err) + goto err_rem; + + fs_rule->mirr_rule_id = reg_id; + mirr_rule->mirr_rule_id = 0; + mirr_rule->mirr_mbox_size = 0; + mirr_rule->mirr_mbox = NULL; + put_res(dev, fs_rule->com.owner, reg_id, RES_FS_RULE); + + return 0; +err_rem: + rem_res_range(dev, fs_rule->com.owner, reg_id, 1, RES_FS_RULE, 0); +err_detach: + mlx4_cmd(dev, reg_id, 0, 0, MLX4_QP_FLOW_STEERING_DETACH, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); +err: + return err; +} + +static int mlx4_mirror_fs_rules(struct mlx4_dev *dev, bool bond) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = + &priv->mfunc.master.res_tracker; + struct rb_root *root = &tracker->res_tree[RES_FS_RULE]; + struct rb_node *p; + struct res_fs_rule *fs_rule; + int err = 0; + LIST_HEAD(mirr_list); + + for (p = rb_first(root); p; p = rb_next(p)) { + fs_rule = rb_entry(p, struct res_fs_rule, com.node); + if ((bond && fs_rule->mirr_mbox_size) || + (!bond && !fs_rule->mirr_mbox_size)) + list_add_tail(&fs_rule->mirr_list, &mirr_list); + } + + list_for_each_entry(fs_rule, &mirr_list, mirr_list) { + if (bond) + err += mlx4_do_mirror_rule(dev, fs_rule); + else + err += mlx4_undo_mirror_rule(dev, fs_rule); + } + return err; +} + +int mlx4_bond_fs_rules(struct mlx4_dev *dev) +{ + return mlx4_mirror_fs_rules(dev, true); +} + +int mlx4_unbond_fs_rules(struct mlx4_dev *dev) +{ + return mlx4_mirror_fs_rules(dev, false); +} + +static void rem_slave_fs_rule(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = + &priv->mfunc.master.res_tracker; + struct list_head *fs_rule_list = + &tracker->slave_list[slave].res_list[RES_FS_RULE]; + struct res_fs_rule *fs_rule; + struct res_fs_rule *tmp; + int state; + u64 base; + int err; + + err = move_all_busy(dev, slave, RES_FS_RULE); + if (err) + mlx4_warn(dev, "rem_slave_fs_rule: Could not move all mtts to busy for slave %d\n", + slave); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(fs_rule, tmp, fs_rule_list, com.list) { + spin_unlock_irq(mlx4_tlock(dev)); + if (fs_rule->com.owner == slave) { + base = fs_rule->com.res_id; + state = fs_rule->com.from_state; + while (state != 0) { + switch (state) { + case RES_FS_RULE_ALLOCATED: + /* detach rule */ + err = mlx4_cmd(dev, base, 0, 0, + MLX4_QP_FLOW_STEERING_DETACH, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + + spin_lock_irq(mlx4_tlock(dev)); + rb_erase(&fs_rule->com.node, + &tracker->res_tree[RES_FS_RULE]); + list_del(&fs_rule->com.list); + spin_unlock_irq(mlx4_tlock(dev)); + kfree(fs_rule); + state = 0; + break; + + default: + state = 0; + } + } + } + spin_lock_irq(mlx4_tlock(dev)); + } + spin_unlock_irq(mlx4_tlock(dev)); +} + +static void rem_slave_eqs(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *eq_list = + &tracker->slave_list[slave].res_list[RES_EQ]; + struct res_eq *eq; + struct res_eq *tmp; + int err; + int state; + LIST_HEAD(tlist); + int eqn; + + err = move_all_busy(dev, slave, RES_EQ); + if (err) + mlx4_warn(dev, "rem_slave_eqs: Could not move all eqs - too busy for slave %d\n", + slave); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(eq, tmp, eq_list, com.list) { + spin_unlock_irq(mlx4_tlock(dev)); + if (eq->com.owner == slave) { + eqn = eq->com.res_id; + state = eq->com.from_state; + while (state != 0) { + switch (state) { + case RES_EQ_RESERVED: + spin_lock_irq(mlx4_tlock(dev)); + rb_erase(&eq->com.node, + &tracker->res_tree[RES_EQ]); + list_del(&eq->com.list); + spin_unlock_irq(mlx4_tlock(dev)); + kfree(eq); + state = 0; + break; + + case RES_EQ_HW: + err = mlx4_cmd(dev, slave, eqn & 0x3ff, + 1, MLX4_CMD_HW2SW_EQ, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + mlx4_dbg(dev, "rem_slave_eqs: failed to move slave %d eqs %d to SW ownership\n", + slave, eqn & 0x3ff); + atomic_dec(&eq->mtt->ref_count); + state = RES_EQ_RESERVED; + break; + + default: + state = 0; + } + } + } + spin_lock_irq(mlx4_tlock(dev)); + } + spin_unlock_irq(mlx4_tlock(dev)); +} + +static void rem_slave_counters(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *counter_list = + &tracker->slave_list[slave].res_list[RES_COUNTER]; + struct res_counter *counter; + struct res_counter *tmp; + int err; + int *counters_arr = NULL; + int i, j; + + err = move_all_busy(dev, slave, RES_COUNTER); + if (err) + mlx4_warn(dev, "rem_slave_counters: Could not move all counters - too busy for slave %d\n", + slave); + + counters_arr = kmalloc_array(dev->caps.max_counters, + sizeof(*counters_arr), GFP_KERNEL); + if (!counters_arr) + return; + + do { + i = 0; + j = 0; + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(counter, tmp, counter_list, com.list) { + if (counter->com.owner == slave) { + counters_arr[i++] = counter->com.res_id; + rb_erase(&counter->com.node, + &tracker->res_tree[RES_COUNTER]); + list_del(&counter->com.list); + kfree(counter); + } + } + spin_unlock_irq(mlx4_tlock(dev)); + + while (j < i) { + __mlx4_counter_free(dev, counters_arr[j++]); + mlx4_release_resource(dev, slave, RES_COUNTER, 1, 0); + } + } while (i); + + kfree(counters_arr); +} + +static void rem_slave_xrcdns(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *xrcdn_list = + &tracker->slave_list[slave].res_list[RES_XRCD]; + struct res_xrcdn *xrcd; + struct res_xrcdn *tmp; + int err; + int xrcdn; + + err = move_all_busy(dev, slave, RES_XRCD); + if (err) + mlx4_warn(dev, "rem_slave_xrcdns: Could not move all xrcdns - too busy for slave %d\n", + slave); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(xrcd, tmp, xrcdn_list, com.list) { + if (xrcd->com.owner == slave) { + xrcdn = xrcd->com.res_id; + rb_erase(&xrcd->com.node, &tracker->res_tree[RES_XRCD]); + list_del(&xrcd->com.list); + kfree(xrcd); + __mlx4_xrcd_free(dev, xrcdn); + } + } + spin_unlock_irq(mlx4_tlock(dev)); +} + +void mlx4_delete_all_resources_for_slave(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + mlx4_reset_roce_gids(dev, slave); + mutex_lock(&priv->mfunc.master.res_tracker.slave_list[slave].mutex); + rem_slave_vlans(dev, slave); + rem_slave_macs(dev, slave); + rem_slave_fs_rule(dev, slave); + rem_slave_qps(dev, slave); + rem_slave_srqs(dev, slave); + rem_slave_cqs(dev, slave); + rem_slave_mrs(dev, slave); + rem_slave_eqs(dev, slave); + rem_slave_mtts(dev, slave); + rem_slave_counters(dev, slave); + rem_slave_xrcdns(dev, slave); + mutex_unlock(&priv->mfunc.master.res_tracker.slave_list[slave].mutex); +} + +void mlx4_vf_immed_vlan_work_handler(struct work_struct *_work) +{ + struct mlx4_vf_immed_vlan_work *work = + container_of(_work, struct mlx4_vf_immed_vlan_work, work); + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_update_qp_context *upd_context; + struct mlx4_dev *dev = &work->priv->dev; + struct mlx4_resource_tracker *tracker = + &work->priv->mfunc.master.res_tracker; + struct list_head *qp_list = + &tracker->slave_list[work->slave].res_list[RES_QP]; + struct res_qp *qp; + struct res_qp *tmp; + u64 qp_path_mask_vlan_ctrl = + ((1ULL << MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_UNTAGGED) | + (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_1P) | + (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_TAGGED) | + (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_UNTAGGED) | + (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_1P) | + (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_TAGGED)); + + u64 qp_path_mask = ((1ULL << MLX4_UPD_QP_PATH_MASK_VLAN_INDEX) | + (1ULL << MLX4_UPD_QP_PATH_MASK_FVL) | + (1ULL << MLX4_UPD_QP_PATH_MASK_CV) | + (1ULL << MLX4_UPD_QP_PATH_MASK_SV) | + (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_HIDE_CQE_VLAN) | + (1ULL << MLX4_UPD_QP_PATH_MASK_FEUP) | + (1ULL << MLX4_UPD_QP_PATH_MASK_FVL_RX) | + (1ULL << MLX4_UPD_QP_PATH_MASK_SCHED_QUEUE)); + + int err; + int port, errors = 0; + u8 vlan_control; + + if (mlx4_is_slave(dev)) { + mlx4_warn(dev, "Trying to update-qp in slave %d\n", + work->slave); + goto out; + } + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + goto out; + if (work->flags & MLX4_VF_IMMED_VLAN_FLAG_LINK_DISABLE) /* block all */ + vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED | + MLX4_VLAN_CTRL_ETH_TX_BLOCK_PRIO_TAGGED | + MLX4_VLAN_CTRL_ETH_TX_BLOCK_UNTAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED; + else if (!work->vlan_id) + vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED; + else if (work->vlan_proto == htons(ETH_P_8021AD)) + vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_PRIO_TAGGED | + MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED; + else /* vst 802.1Q */ + vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED; + + upd_context = mailbox->buf; + upd_context->qp_mask = cpu_to_be64(1ULL << MLX4_UPD_QP_MASK_VSD); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(qp, tmp, qp_list, com.list) { + spin_unlock_irq(mlx4_tlock(dev)); + if (qp->com.owner == work->slave) { + if (qp->com.from_state != RES_QP_HW || + !qp->sched_queue || /* no INIT2RTR trans yet */ + mlx4_is_qp_reserved(dev, qp->local_qpn) || + qp->qpc_flags & (1 << MLX4_RSS_QPC_FLAG_OFFSET)) { + spin_lock_irq(mlx4_tlock(dev)); + continue; + } + port = (qp->sched_queue >> 6 & 1) + 1; + if (port != work->port) { + spin_lock_irq(mlx4_tlock(dev)); + continue; + } + if (MLX4_QP_ST_RC == ((qp->qpc_flags >> 16) & 0xff)) + upd_context->primary_addr_path_mask = cpu_to_be64(qp_path_mask); + else + upd_context->primary_addr_path_mask = + cpu_to_be64(qp_path_mask | qp_path_mask_vlan_ctrl); + if (work->vlan_id == MLX4_VGT) { + upd_context->qp_context.param3 = qp->param3; + upd_context->qp_context.pri_path.vlan_control = qp->vlan_control; + upd_context->qp_context.pri_path.fvl_rx = qp->fvl_rx; + upd_context->qp_context.pri_path.vlan_index = qp->vlan_index; + upd_context->qp_context.pri_path.fl = qp->pri_path_fl; + upd_context->qp_context.pri_path.feup = qp->feup; + upd_context->qp_context.pri_path.sched_queue = + qp->sched_queue; + } else { + upd_context->qp_context.param3 = qp->param3 & ~cpu_to_be32(MLX4_STRIP_VLAN); + upd_context->qp_context.pri_path.vlan_control = vlan_control; + upd_context->qp_context.pri_path.vlan_index = work->vlan_ix; + upd_context->qp_context.pri_path.fvl_rx = + qp->fvl_rx | MLX4_FVL_RX_FORCE_ETH_VLAN; + upd_context->qp_context.pri_path.fl = + qp->pri_path_fl | MLX4_FL_ETH_HIDE_CQE_VLAN; + if (work->vlan_proto == htons(ETH_P_8021AD)) + upd_context->qp_context.pri_path.fl |= MLX4_FL_SV; + else + upd_context->qp_context.pri_path.fl |= MLX4_FL_CV; + upd_context->qp_context.pri_path.feup = + qp->feup | MLX4_FEUP_FORCE_ETH_UP | MLX4_FVL_FORCE_ETH_VLAN; + upd_context->qp_context.pri_path.sched_queue = + qp->sched_queue & 0xC7; + upd_context->qp_context.pri_path.sched_queue |= + ((work->qos & 0x7) << 3); + upd_context->qp_mask |= + cpu_to_be64(1ULL << + MLX4_UPD_QP_MASK_QOS_VPP); + upd_context->qp_context.qos_vport = + work->qos_vport; + } + + err = mlx4_cmd(dev, mailbox->dma, + qp->local_qpn & 0xffffff, + 0, MLX4_CMD_UPDATE_QP, + MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); + if (err) { + mlx4_info(dev, "UPDATE_QP failed for slave %d, port %d, qpn %d (%d)\n", + work->slave, port, qp->local_qpn, err); + errors++; + } + } + spin_lock_irq(mlx4_tlock(dev)); + } + spin_unlock_irq(mlx4_tlock(dev)); + mlx4_free_cmd_mailbox(dev, mailbox); + + if (errors) + mlx4_err(dev, "%d UPDATE_QP failures for slave %d, port %d\n", + errors, work->slave, work->port); + + /* unregister previous vlan_id if needed and we had no errors + * while updating the QPs + */ + if (work->flags & MLX4_VF_IMMED_VLAN_FLAG_VLAN && !errors && + NO_INDX != work->orig_vlan_ix) + __mlx4_unregister_vlan(&work->priv->dev, work->port, + work->orig_vlan_id); +out: + kfree(work); + return; +} + Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_resource_tracker.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_sense.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_sense.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_sense.c (revision 329159) @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#include +#include + +#include + +#include "mlx4.h" + +int mlx4_SENSE_PORT(struct mlx4_dev *dev, int port, + enum mlx4_port_type *type) +{ + u64 out_param; + int err = 0; + + err = mlx4_cmd_imm(dev, 0, &out_param, port, 0, + MLX4_CMD_SENSE_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); + if (err) { + mlx4_err(dev, "Sense command failed for port: %d\n", port); + return err; + } + + if (out_param > 2) { + mlx4_err(dev, "Sense returned illegal value: 0x%llx\n", (long long)out_param); + return -EINVAL; + } + + *type = out_param; + return 0; +} + +void mlx4_do_sense_ports(struct mlx4_dev *dev, + enum mlx4_port_type *stype, + enum mlx4_port_type *defaults) +{ + struct mlx4_sense *sense = &mlx4_priv(dev)->sense; + int err; + int i; + + for (i = 1; i <= dev->caps.num_ports; i++) { + stype[i - 1] = 0; + if (sense->do_sense_port[i] && sense->sense_allowed[i] && + dev->caps.possible_type[i] == MLX4_PORT_TYPE_AUTO) { + err = mlx4_SENSE_PORT(dev, i, &stype[i - 1]); + if (err) + stype[i - 1] = defaults[i - 1]; + } else + stype[i - 1] = defaults[i - 1]; + } + + /* + * If sensed nothing, remain in current configuration. + */ + for (i = 0; i < dev->caps.num_ports; i++) + stype[i] = stype[i] ? stype[i] : defaults[i]; + +} + +static void mlx4_sense_port(struct work_struct *work) +{ + struct delayed_work *delay = to_delayed_work(work); + struct mlx4_sense *sense = container_of(delay, struct mlx4_sense, + sense_poll); + struct mlx4_dev *dev = sense->dev; + struct mlx4_priv *priv = mlx4_priv(dev); + enum mlx4_port_type stype[MLX4_MAX_PORTS]; + + mutex_lock(&priv->port_mutex); + if (sense->gone != 0) { + mutex_unlock(&priv->port_mutex); + return; + } + mlx4_do_sense_ports(dev, stype, &dev->caps.port_type[1]); + + if (mlx4_check_port_params(dev, stype)) + goto sense_again; + + if (mlx4_change_port_types(dev, stype)) + mlx4_err(dev, "Failed to change port_types\n"); + +sense_again: + queue_delayed_work(mlx4_wq , &sense->sense_poll, + round_jiffies_relative(MLX4_SENSE_RANGE)); + mutex_unlock(&priv->port_mutex); +} + +void mlx4_start_sense(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_sense *sense = &priv->sense; + + if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) + return; + + mutex_lock(&priv->port_mutex); + sense->gone = 0; + queue_delayed_work(mlx4_wq , &sense->sense_poll, + round_jiffies_relative(MLX4_SENSE_RANGE)); + mutex_unlock(&priv->port_mutex); +} + +void mlx4_stop_sense(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_sense *sense = &priv->sense; + + mutex_lock(&priv->port_mutex); + sense->gone = 1; + mutex_unlock(&priv->port_mutex); + + cancel_delayed_work_sync(&mlx4_priv(dev)->sense.sense_poll); +} + +void mlx4_sense_init(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_sense *sense = &priv->sense; + int port; + + sense->dev = dev; + for (port = 1; port <= dev->caps.num_ports; port++) + sense->do_sense_port[port] = 1; + + INIT_DEFERRABLE_WORK(&sense->sense_poll, mlx4_sense_port); +} Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_sense.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_core/mlx4_srq.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_core/mlx4_srq.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_core/mlx4_srq.c (revision 329159) @@ -0,0 +1,309 @@ +/* + * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include + +#include "mlx4.h" +#include "icm.h" + +void mlx4_srq_event(struct mlx4_dev *dev, u32 srqn, int event_type) +{ + struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; + struct mlx4_srq *srq; + + rcu_read_lock(); + srq = radix_tree_lookup(&srq_table->tree, srqn & (dev->caps.num_srqs - 1)); + rcu_read_unlock(); + if (srq) + atomic_inc(&srq->refcount); + else { + mlx4_warn(dev, "Async event for bogus SRQ %08x\n", srqn); + return; + } + + srq->event(srq, event_type); + + if (atomic_dec_and_test(&srq->refcount)) + complete(&srq->free); +} + +static int mlx4_SW2HW_SRQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, + int srq_num) +{ + return mlx4_cmd(dev, mailbox->dma, srq_num, 0, + MLX4_CMD_SW2HW_SRQ, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); +} + +static int mlx4_HW2SW_SRQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, + int srq_num) +{ + return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, srq_num, + mailbox ? 0 : 1, MLX4_CMD_HW2SW_SRQ, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); +} + +static int mlx4_ARM_SRQ(struct mlx4_dev *dev, int srq_num, int limit_watermark) +{ + return mlx4_cmd(dev, limit_watermark, srq_num, 0, MLX4_CMD_ARM_SRQ, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); +} + +static int mlx4_QUERY_SRQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, + int srq_num) +{ + return mlx4_cmd_box(dev, 0, mailbox->dma, srq_num, 0, MLX4_CMD_QUERY_SRQ, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); +} + +int __mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn) +{ + struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; + int err; + + + *srqn = mlx4_bitmap_alloc(&srq_table->bitmap); + if (*srqn == -1) + return -ENOMEM; + + err = mlx4_table_get(dev, &srq_table->table, *srqn, GFP_KERNEL); + if (err) + goto err_out; + + err = mlx4_table_get(dev, &srq_table->cmpt_table, *srqn, GFP_KERNEL); + if (err) + goto err_put; + return 0; + +err_put: + mlx4_table_put(dev, &srq_table->table, *srqn); + +err_out: + mlx4_bitmap_free(&srq_table->bitmap, *srqn, MLX4_NO_RR); + return err; +} + +static int mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn) +{ + u64 out_param; + int err; + + if (mlx4_is_mfunc(dev)) { + err = mlx4_cmd_imm(dev, 0, &out_param, RES_SRQ, + RES_OP_RESERVE_AND_MAP, + MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (!err) + *srqn = get_param_l(&out_param); + + return err; + } + return __mlx4_srq_alloc_icm(dev, srqn); +} + +void __mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn) +{ + struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; + + mlx4_table_put(dev, &srq_table->cmpt_table, srqn); + mlx4_table_put(dev, &srq_table->table, srqn); + mlx4_bitmap_free(&srq_table->bitmap, srqn, MLX4_NO_RR); +} + +static void mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn) +{ + u64 in_param = 0; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, srqn); + if (mlx4_cmd(dev, in_param, RES_SRQ, RES_OP_RESERVE_AND_MAP, + MLX4_CMD_FREE_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) + mlx4_warn(dev, "Failed freeing cq:%d\n", srqn); + return; + } + __mlx4_srq_free_icm(dev, srqn); +} + +int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, u32 cqn, u16 xrcd, + struct mlx4_mtt *mtt, u64 db_rec, struct mlx4_srq *srq) +{ + struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_srq_context *srq_context; + u64 mtt_addr; + int err; + + err = mlx4_srq_alloc_icm(dev, &srq->srqn); + if (err) + return err; + + spin_lock_irq(&srq_table->lock); + err = radix_tree_insert(&srq_table->tree, srq->srqn, srq); + spin_unlock_irq(&srq_table->lock); + if (err) + goto err_icm; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + err = PTR_ERR(mailbox); + goto err_radix; + } + + srq_context = mailbox->buf; + srq_context->state_logsize_srqn = cpu_to_be32((ilog2(srq->max) << 24) | + srq->srqn); + srq_context->logstride = srq->wqe_shift - 4; + srq_context->xrcd = cpu_to_be16(xrcd); + srq_context->pg_offset_cqn = cpu_to_be32(cqn & 0xffffff); + srq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT; + + mtt_addr = mlx4_mtt_addr(dev, mtt); + srq_context->mtt_base_addr_h = mtt_addr >> 32; + srq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); + srq_context->pd = cpu_to_be32(pdn); + srq_context->db_rec_addr = cpu_to_be64(db_rec); + + err = mlx4_SW2HW_SRQ(dev, mailbox, srq->srqn); + mlx4_free_cmd_mailbox(dev, mailbox); + if (err) + goto err_radix; + + atomic_set(&srq->refcount, 1); + init_completion(&srq->free); + + return 0; + +err_radix: + spin_lock_irq(&srq_table->lock); + radix_tree_delete(&srq_table->tree, srq->srqn); + spin_unlock_irq(&srq_table->lock); + +err_icm: + mlx4_srq_free_icm(dev, srq->srqn); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_srq_alloc); + +void mlx4_srq_free(struct mlx4_dev *dev, struct mlx4_srq *srq) +{ + struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; + int err; + + err = mlx4_HW2SW_SRQ(dev, NULL, srq->srqn); + if (err) + mlx4_warn(dev, "HW2SW_SRQ failed (%d) for SRQN %06x\n", err, srq->srqn); + + spin_lock_irq(&srq_table->lock); + radix_tree_delete(&srq_table->tree, srq->srqn); + spin_unlock_irq(&srq_table->lock); + + if (atomic_dec_and_test(&srq->refcount)) + complete(&srq->free); + wait_for_completion(&srq->free); + + mlx4_srq_free_icm(dev, srq->srqn); +} +EXPORT_SYMBOL_GPL(mlx4_srq_free); + +int mlx4_srq_arm(struct mlx4_dev *dev, struct mlx4_srq *srq, int limit_watermark) +{ + return mlx4_ARM_SRQ(dev, srq->srqn, limit_watermark); +} +EXPORT_SYMBOL_GPL(mlx4_srq_arm); + +int mlx4_srq_query(struct mlx4_dev *dev, struct mlx4_srq *srq, int *limit_watermark) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_srq_context *srq_context; + int err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + srq_context = mailbox->buf; + + err = mlx4_QUERY_SRQ(dev, mailbox, srq->srqn); + if (err) + goto err_out; + *limit_watermark = be16_to_cpu(srq_context->limit_watermark); + +err_out: + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_srq_query); + +int mlx4_init_srq_table(struct mlx4_dev *dev) +{ + struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; + int err; + + spin_lock_init(&srq_table->lock); + INIT_RADIX_TREE(&srq_table->tree, GFP_ATOMIC); + if (mlx4_is_slave(dev)) + return 0; + + err = mlx4_bitmap_init(&srq_table->bitmap, dev->caps.num_srqs, + dev->caps.num_srqs - 1, dev->caps.reserved_srqs, 0); + if (err) + return err; + + return 0; +} + +void mlx4_cleanup_srq_table(struct mlx4_dev *dev) +{ + if (mlx4_is_slave(dev)) + return; + mlx4_bitmap_cleanup(&mlx4_priv(dev)->srq_table.bitmap); +} + +struct mlx4_srq *mlx4_srq_lookup(struct mlx4_dev *dev, u32 srqn) +{ + struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; + struct mlx4_srq *srq; + + rcu_read_lock(); + srq = radix_tree_lookup(&srq_table->tree, + srqn & (dev->caps.num_srqs - 1)); + rcu_read_unlock(); + + return srq; +} +EXPORT_SYMBOL_GPL(mlx4_srq_lookup); Property changes on: stable/11/sys/dev/mlx4/mlx4_core/mlx4_srq.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_en/en.h =================================================================== --- stable/11/sys/dev/mlx4/mlx4_en/en.h (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_en/en.h (revision 329159) @@ -0,0 +1,944 @@ +/* + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#ifndef _MLX4_EN_H_ +#define _MLX4_EN_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_MLX4_EN_DCB +#include +#endif + +#include +#include +#include +#include +#include +#include + +#include + +#include "en_port.h" +#include + +#define DRV_NAME "mlx4_en" + +#define MLX4_EN_MSG_LEVEL (NETIF_MSG_LINK | NETIF_MSG_IFDOWN) + +/* + * Device constants + */ + + +#define MLX4_EN_PAGE_SHIFT 12 +#define MLX4_EN_PAGE_SIZE (1 << MLX4_EN_PAGE_SHIFT) +#define MLX4_NET_IP_ALIGN 2 /* bytes */ +#define DEF_RX_RINGS 16 +#define MAX_RX_RINGS 128 +#define MIN_RX_RINGS 4 +#define TXBB_SIZE 64 +#define HEADROOM (2048 / TXBB_SIZE + 1) +#define INIT_OWNER_BIT 0xffffffff +#define STAMP_STRIDE 64 +#define STAMP_DWORDS (STAMP_STRIDE / 4) +#define STAMP_SHIFT 31 +#define STAMP_VAL 0x7fffffff +#define STATS_DELAY (HZ / 4) +#define SERVICE_TASK_DELAY (HZ / 4) +#define MAX_NUM_OF_FS_RULES 256 + +#define MLX4_EN_FILTER_HASH_SHIFT 4 +#define MLX4_EN_FILTER_EXPIRY_QUOTA 60 + +#ifdef CONFIG_NET_RX_BUSY_POLL +#define LL_EXTENDED_STATS +#endif + +/* vlan valid range */ +#define VLAN_MIN_VALUE 1 +#define VLAN_MAX_VALUE 4094 + +/* + * OS related constants and tunables + */ + +#define MLX4_EN_WATCHDOG_TIMEOUT (15 * HZ) + +#define MLX4_EN_ALLOC_SIZE PAGE_ALIGN(PAGE_SIZE) +#define MLX4_EN_ALLOC_ORDER get_order(MLX4_EN_ALLOC_SIZE) + +enum mlx4_en_alloc_type { + MLX4_EN_ALLOC_NEW = 0, + MLX4_EN_ALLOC_REPLACEMENT = 1, +}; + +/* Maximum ring sizes */ +#define MLX4_EN_DEF_TX_QUEUE_SIZE 4096 + +/* Minimum packet number till arming the CQ */ +#define MLX4_EN_MIN_RX_ARM 2048 +#define MLX4_EN_MIN_TX_ARM 2048 + +/* Maximum ring sizes */ +#define MLX4_EN_MAX_TX_SIZE 8192 +#define MLX4_EN_MAX_RX_SIZE 8192 + +/* Minimum ring sizes */ +#define MLX4_EN_MIN_RX_SIZE (4096 / TXBB_SIZE) +#define MLX4_EN_MIN_TX_SIZE (4096 / TXBB_SIZE) + +#define MLX4_EN_SMALL_PKT_SIZE 64 + +#define MLX4_EN_MAX_TX_RING_P_UP 32 +#define MLX4_EN_NUM_UP 1 + +#define MAX_TX_RINGS (MLX4_EN_MAX_TX_RING_P_UP * \ + MLX4_EN_NUM_UP) + +#define MLX4_EN_NO_VLAN 0xffff + +#define MLX4_EN_DEF_TX_RING_SIZE 1024 +#define MLX4_EN_DEF_RX_RING_SIZE 1024 + +/* Target number of bytes to coalesce with interrupt moderation */ +#define MLX4_EN_RX_COAL_TARGET 44 +#define MLX4_EN_RX_COAL_TIME 0x10 + +#define MLX4_EN_TX_COAL_PKTS 64 +#define MLX4_EN_TX_COAL_TIME 64 + +#define MLX4_EN_RX_RATE_LOW 400000 +#define MLX4_EN_RX_COAL_TIME_LOW 0 +#define MLX4_EN_RX_RATE_HIGH 450000 +#define MLX4_EN_RX_COAL_TIME_HIGH 128 +#define MLX4_EN_RX_SIZE_THRESH 1024 +#define MLX4_EN_RX_RATE_THRESH (1000000 / MLX4_EN_RX_COAL_TIME_HIGH) +#define MLX4_EN_SAMPLE_INTERVAL 0 +#define MLX4_EN_AVG_PKT_SMALL 256 + +#define MLX4_EN_AUTO_CONF 0xffff + +#define MLX4_EN_DEF_RX_PAUSE 1 +#define MLX4_EN_DEF_TX_PAUSE 1 + +/* Interval between successive polls in the Tx routine when polling is used + instead of interrupts (in per-core Tx rings) - should be power of 2 */ +#define MLX4_EN_TX_POLL_MODER 16 +#define MLX4_EN_TX_POLL_TIMEOUT (HZ / 4) + +#define MLX4_EN_64_ALIGN (64 - NET_SKB_PAD) +#define SMALL_PACKET_SIZE (256 - NET_IP_ALIGN) +#define HEADER_COPY_SIZE (128) +#define MLX4_LOOPBACK_TEST_PAYLOAD (HEADER_COPY_SIZE - ETHER_HDR_LEN) + +#define MLX4_EN_MIN_MTU 46 +#define ETH_BCAST 0xffffffffffffULL + +#define MLX4_EN_LOOPBACK_RETRIES 5 +#define MLX4_EN_LOOPBACK_TIMEOUT 100 + +#ifdef MLX4_EN_PERF_STAT +/* Number of samples to 'average' */ +#define AVG_SIZE 128 +#define AVG_FACTOR 1024 + +#define INC_PERF_COUNTER(cnt) (++(cnt)) +#define ADD_PERF_COUNTER(cnt, add) ((cnt) += (add)) +#define AVG_PERF_COUNTER(cnt, sample) \ + ((cnt) = ((cnt) * (AVG_SIZE - 1) + (sample) * AVG_FACTOR) / AVG_SIZE) +#define GET_PERF_COUNTER(cnt) (cnt) +#define GET_AVG_PERF_COUNTER(cnt) ((cnt) / AVG_FACTOR) + +#else + +#define INC_PERF_COUNTER(cnt) do {} while (0) +#define ADD_PERF_COUNTER(cnt, add) do {} while (0) +#define AVG_PERF_COUNTER(cnt, sample) do {} while (0) +#define GET_PERF_COUNTER(cnt) (0) +#define GET_AVG_PERF_COUNTER(cnt) (0) +#endif /* MLX4_EN_PERF_STAT */ + +/* Constants for TX flow */ +enum { + MAX_INLINE = 104, /* 128 - 16 - 4 - 4 */ + MAX_BF = 256, + MIN_PKT_LEN = 17, +}; + +/* + * Configurables + */ + +enum cq_type { + RX = 0, + TX = 1, +}; + + +/* + * Useful macros + */ +#define ROUNDUP_LOG2(x) ilog2(roundup_pow_of_two(x)) +#define XNOR(x, y) (!(x) == !(y)) +#define ILLEGAL_MAC(addr) (addr == 0xffffffffffffULL || addr == 0x0) + +struct mlx4_en_tx_info { + bus_dmamap_t dma_map; + struct mbuf *mb; + u32 nr_txbb; + u32 nr_bytes; +}; + + +#define MLX4_EN_BIT_DESC_OWN 0x80000000 +#define CTRL_SIZE sizeof(struct mlx4_wqe_ctrl_seg) +#define MLX4_EN_MEMTYPE_PAD 0x100 +#define DS_SIZE sizeof(struct mlx4_wqe_data_seg) + + +struct mlx4_en_tx_desc { + struct mlx4_wqe_ctrl_seg ctrl; + union { + struct mlx4_wqe_data_seg data; /* at least one data segment */ + struct mlx4_wqe_lso_seg lso; + struct mlx4_wqe_inline_seg inl; + }; +}; + +#define MLX4_EN_USE_SRQ 0x01000000 + +#define MLX4_EN_RX_BUDGET 64 + +#define MLX4_EN_TX_MAX_DESC_SIZE 512 /* bytes */ +#define MLX4_EN_TX_MAX_MBUF_SIZE 65536 /* bytes */ +#define MLX4_EN_TX_MAX_PAYLOAD_SIZE 65536 /* bytes */ +#define MLX4_EN_TX_MAX_MBUF_FRAGS \ + ((MLX4_EN_TX_MAX_DESC_SIZE - 128) / DS_SIZE_ALIGNMENT) /* units */ +#define MLX4_EN_TX_WQE_MAX_WQEBBS \ + (MLX4_EN_TX_MAX_DESC_SIZE / TXBB_SIZE) /* units */ + +#define MLX4_EN_CX3_LOW_ID 0x1000 +#define MLX4_EN_CX3_HIGH_ID 0x1005 + +struct mlx4_en_tx_ring { + spinlock_t tx_lock; + bus_dma_tag_t dma_tag; + struct mlx4_hwq_resources wqres; + u32 size ; /* number of TXBBs */ + u32 size_mask; + u16 stride; + u16 cqn; /* index of port CQ associated with this ring */ + u32 prod; + u32 cons; + u32 buf_size; + u32 doorbell_qpn; + u8 *buf; + u16 poll_cnt; + int blocked; + struct mlx4_en_tx_info *tx_info; + u8 queue_index; + struct buf_ring *br; + u32 last_nr_txbb; + struct mlx4_qp qp; + struct mlx4_qp_context context; + int qpn; + enum mlx4_qp_state qp_state; + struct mlx4_srq dummy; + u64 bytes; + u64 packets; + u64 tx_csum; + u64 queue_stopped; + u64 oversized_packets; + u64 wake_queue; + u64 tso_packets; + u64 defrag_attempts; + struct mlx4_bf bf; + bool bf_enabled; + int hwtstamp_tx_type; + spinlock_t comp_lock; + int inline_thold; + u64 watchdog_time; +}; + +struct mlx4_en_rx_desc { + /* actual number of entries depends on rx ring stride */ + struct mlx4_wqe_data_seg data[0]; +}; + +struct mlx4_en_rx_mbuf { + bus_dmamap_t dma_map; + struct mbuf *mbuf; +}; + +struct mlx4_en_rx_spare { + bus_dmamap_t dma_map; + struct mbuf *mbuf; + u64 paddr_be; +}; + +struct mlx4_en_rx_ring { + struct mlx4_hwq_resources wqres; + bus_dma_tag_t dma_tag; + struct mlx4_en_rx_spare spare; + u32 size ; /* number of Rx descs*/ + u32 actual_size; + u32 size_mask; + u16 stride; + u16 log_stride; + u16 cqn; /* index of port CQ associated with this ring */ + u32 prod; + u32 cons; + u32 buf_size; + u8 fcs_del; + u32 rx_mb_size; + int qpn; + u8 *buf; + struct mlx4_en_rx_mbuf *mbuf; + u64 errors; + u64 bytes; + u64 packets; +#ifdef LL_EXTENDED_STATS + u64 yields; + u64 misses; + u64 cleaned; +#endif + u64 csum_ok; + u64 csum_none; + int hwtstamp_rx_filter; + int numa_node; + struct lro_ctrl lro; +}; + +static inline int mlx4_en_can_lro(__be16 status) +{ + const __be16 status_all = cpu_to_be16( + MLX4_CQE_STATUS_IPV4 | + MLX4_CQE_STATUS_IPV4F | + MLX4_CQE_STATUS_IPV6 | + MLX4_CQE_STATUS_IPV4OPT | + MLX4_CQE_STATUS_TCP | + MLX4_CQE_STATUS_UDP | + MLX4_CQE_STATUS_IPOK); + const __be16 status_ipv4_ipok_tcp = cpu_to_be16( + MLX4_CQE_STATUS_IPV4 | + MLX4_CQE_STATUS_IPOK | + MLX4_CQE_STATUS_TCP); + const __be16 status_ipv6_ipok_tcp = cpu_to_be16( + MLX4_CQE_STATUS_IPV6 | + MLX4_CQE_STATUS_IPOK | + MLX4_CQE_STATUS_TCP); + + status &= status_all; + return (status == status_ipv4_ipok_tcp || + status == status_ipv6_ipok_tcp); +} + +struct mlx4_en_cq { + struct mlx4_cq mcq; + struct mlx4_hwq_resources wqres; + int ring; + spinlock_t lock; + struct net_device *dev; + /* Per-core Tx cq processing support */ + struct timer_list timer; + int size; + int buf_size; + unsigned vector; + enum cq_type is_tx; + u16 moder_time; + u16 moder_cnt; + struct mlx4_cqe *buf; + struct task cq_task; + struct taskqueue *tq; +#define MLX4_EN_OPCODE_ERROR 0x1e + u32 tot_rx; + u32 tot_tx; + u32 curr_poll_rx_cpu_id; + +#ifdef CONFIG_NET_RX_BUSY_POLL + unsigned int state; +#define MLX4_EN_CQ_STATE_IDLE 0 +#define MLX4_EN_CQ_STATE_NAPI 1 /* NAPI owns this CQ */ +#define MLX4_EN_CQ_STATE_POLL 2 /* poll owns this CQ */ +#define MLX4_CQ_LOCKED (MLX4_EN_CQ_STATE_NAPI | MLX4_EN_CQ_STATE_POLL) +#define MLX4_EN_CQ_STATE_NAPI_YIELD 4 /* NAPI yielded this CQ */ +#define MLX4_EN_CQ_STATE_POLL_YIELD 8 /* poll yielded this CQ */ +#define CQ_YIELD (MLX4_EN_CQ_STATE_NAPI_YIELD | MLX4_EN_CQ_STATE_POLL_YIELD) +#define CQ_USER_PEND (MLX4_EN_CQ_STATE_POLL | MLX4_EN_CQ_STATE_POLL_YIELD) + spinlock_t poll_lock; /* protects from LLS/napi conflicts */ +#endif /* CONFIG_NET_RX_BUSY_POLL */ +}; + +struct mlx4_en_port_profile { + u32 flags; + u32 tx_ring_num; + u32 rx_ring_num; + u32 tx_ring_size; + u32 rx_ring_size; + u8 rx_pause; + u8 rx_ppp; + u8 tx_pause; + u8 tx_ppp; + int rss_rings; + int inline_thold; +}; + +struct mlx4_en_profile { + int rss_xor; + int udp_rss; + u8 rss_mask; + u32 active_ports; + u32 small_pkt_int; + u8 no_reset; + u8 num_tx_rings_p_up; + struct mlx4_en_port_profile prof[MLX4_MAX_PORTS + 1]; +}; + +struct mlx4_en_dev { + struct mlx4_dev *dev; + struct pci_dev *pdev; + struct mutex state_lock; + struct net_device *pndev[MLX4_MAX_PORTS + 1]; + u32 port_cnt; + bool device_up; + struct mlx4_en_profile profile; + u32 LSO_support; + struct workqueue_struct *workqueue; + struct device *dma_device; + void __iomem *uar_map; + struct mlx4_uar priv_uar; + struct mlx4_mr mr; + u32 priv_pdn; + spinlock_t uar_lock; + u8 mac_removed[MLX4_MAX_PORTS + 1]; + unsigned long last_overflow_check; + unsigned long overflow_period; +}; + + +struct mlx4_en_rss_map { + int base_qpn; + struct mlx4_qp qps[MAX_RX_RINGS]; + enum mlx4_qp_state state[MAX_RX_RINGS]; + struct mlx4_qp indir_qp; + enum mlx4_qp_state indir_state; +}; + +enum mlx4_en_port_flag { + MLX4_EN_PORT_ANC = 1<<0, /* Auto-negotiation complete */ + MLX4_EN_PORT_ANE = 1<<1, /* Auto-negotiation enabled */ +}; + +struct mlx4_en_port_state { + int link_state; + int link_speed; + int transceiver; + u32 flags; +}; + +enum mlx4_en_addr_list_act { + MLX4_ADDR_LIST_NONE, + MLX4_ADDR_LIST_REM, + MLX4_ADDR_LIST_ADD, +}; + +struct mlx4_en_addr_list { + struct list_head list; + enum mlx4_en_addr_list_act action; + u8 addr[ETH_ALEN]; + u64 reg_id; + u64 tunnel_reg_id; +}; + +#ifdef CONFIG_MLX4_EN_DCB +/* Minimal TC BW - setting to 0 will block traffic */ +#define MLX4_EN_BW_MIN 1 +#define MLX4_EN_BW_MAX 100 /* Utilize 100% of the line */ + +#define MLX4_EN_TC_VENDOR 0 +#define MLX4_EN_TC_ETS 7 + +#endif + + +enum { + MLX4_EN_FLAG_PROMISC = (1 << 0), + MLX4_EN_FLAG_MC_PROMISC = (1 << 1), + /* whether we need to enable hardware loopback by putting dmac + * in Tx WQE + */ + MLX4_EN_FLAG_ENABLE_HW_LOOPBACK = (1 << 2), + /* whether we need to drop packets that hardware loopback-ed */ + MLX4_EN_FLAG_RX_FILTER_NEEDED = (1 << 3), + MLX4_EN_FLAG_FORCE_PROMISC = (1 << 4), +#ifdef CONFIG_MLX4_EN_DCB + MLX4_EN_FLAG_DCB_ENABLED = (1 << 5) +#endif +}; + +#define MLX4_EN_MAC_HASH_SIZE (1 << BITS_PER_BYTE) +#define MLX4_EN_MAC_HASH_IDX 5 + +struct en_port { + struct kobject kobj; + struct mlx4_dev *dev; + u8 port_num; + u8 vport_num; +}; + +struct mlx4_en_priv { + struct mlx4_en_dev *mdev; + struct mlx4_en_port_profile *prof; + struct net_device *dev; + unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; + struct mlx4_en_port_state port_state; + spinlock_t stats_lock; + /* To allow rules removal while port is going down */ + struct list_head ethtool_list; + + unsigned long last_moder_packets[MAX_RX_RINGS]; + unsigned long last_moder_tx_packets; + unsigned long last_moder_bytes[MAX_RX_RINGS]; + unsigned long last_moder_jiffies; + int last_moder_time[MAX_RX_RINGS]; + u16 rx_usecs; + u16 rx_frames; + u16 tx_usecs; + u16 tx_frames; + u32 pkt_rate_low; + u32 rx_usecs_low; + u32 pkt_rate_high; + u32 rx_usecs_high; + u32 sample_interval; + u32 adaptive_rx_coal; + u32 msg_enable; + u32 loopback_ok; + u32 validate_loopback; + + struct mlx4_hwq_resources res; + int link_state; + int last_link_state; + bool port_up; + int port; + int registered; + int gone; + int allocated; + int stride; + unsigned char current_mac[ETH_ALEN + 2]; + u64 mac; + int mac_index; + unsigned max_mtu; + int base_qpn; + int cqe_factor; + + struct mlx4_en_rss_map rss_map; + u32 flags; + u8 num_tx_rings_p_up; + u32 tx_ring_num; + u32 rx_ring_num; + u32 rx_mb_size; + + struct mlx4_en_tx_ring **tx_ring; + struct mlx4_en_rx_ring *rx_ring[MAX_RX_RINGS]; + struct mlx4_en_cq **tx_cq; + struct mlx4_en_cq *rx_cq[MAX_RX_RINGS]; + struct mlx4_qp drop_qp; + struct work_struct rx_mode_task; + struct work_struct watchdog_task; + struct work_struct linkstate_task; + struct delayed_work stats_task; + struct delayed_work service_task; + struct mlx4_en_perf_stats pstats; + struct mlx4_en_pkt_stats pkstats; + struct mlx4_en_pkt_stats pkstats_last; + struct mlx4_en_flow_stats_rx rx_priority_flowstats[MLX4_NUM_PRIORITIES]; + struct mlx4_en_flow_stats_tx tx_priority_flowstats[MLX4_NUM_PRIORITIES]; + struct mlx4_en_flow_stats_rx rx_flowstats; + struct mlx4_en_flow_stats_tx tx_flowstats; + struct mlx4_en_port_stats port_stats; + struct mlx4_en_vport_stats vport_stats; + struct mlx4_en_vf_stats vf_stats; + struct list_head mc_list; + struct list_head uc_list; + struct list_head curr_mc_list; + struct list_head curr_uc_list; + u64 broadcast_id; + struct mlx4_en_stat_out_mbox hw_stats; + int vids[128]; + bool wol; + struct device *ddev; + struct dentry *dev_root; + u32 counter_index; + eventhandler_tag vlan_attach; + eventhandler_tag vlan_detach; + struct callout watchdog_timer; + struct ifmedia media; + volatile int blocked; + struct sysctl_oid *conf_sysctl; + struct sysctl_oid *stat_sysctl; + struct sysctl_ctx_list conf_ctx; + struct sysctl_ctx_list stat_ctx; + +#ifdef CONFIG_MLX4_EN_DCB + struct ieee_ets ets; + u16 maxrate[IEEE_8021QAZ_MAX_TCS]; + u8 dcbx_cap; +#endif +#ifdef CONFIG_RFS_ACCEL + spinlock_t filters_lock; + int last_filter_id; + struct list_head filters; + struct hlist_head filter_hash[1 << MLX4_EN_FILTER_HASH_SHIFT]; +#endif + u64 tunnel_reg_id; + struct en_port *vf_ports[MLX4_MAX_NUM_VF]; + unsigned long last_ifq_jiffies; + u64 if_counters_rx_errors; + u64 if_counters_rx_no_buffer; +}; + +enum mlx4_en_wol { + MLX4_EN_WOL_MAGIC = (1ULL << 61), + MLX4_EN_WOL_ENABLED = (1ULL << 62), +}; + +struct mlx4_mac_entry { + struct hlist_node hlist; + unsigned char mac[ETH_ALEN + 2]; + u64 reg_id; +}; + +static inline struct mlx4_cqe *mlx4_en_get_cqe(u8 *buf, int idx, int cqe_sz) +{ + return (struct mlx4_cqe *)(buf + idx * cqe_sz); +} + +#ifdef CONFIG_NET_RX_BUSY_POLL +static inline void mlx4_en_cq_init_lock(struct mlx4_en_cq *cq) +{ + spin_lock_init(&cq->poll_lock); + cq->state = MLX4_EN_CQ_STATE_IDLE; +} + +/* called from the device poll rutine to get ownership of a cq */ +static inline bool mlx4_en_cq_lock_napi(struct mlx4_en_cq *cq) +{ + int rc = true; + spin_lock(&cq->poll_lock); + if (cq->state & MLX4_CQ_LOCKED) { + WARN_ON(cq->state & MLX4_EN_CQ_STATE_NAPI); + cq->state |= MLX4_EN_CQ_STATE_NAPI_YIELD; + rc = false; + } else + /* we don't care if someone yielded */ + cq->state = MLX4_EN_CQ_STATE_NAPI; + spin_unlock(&cq->poll_lock); + return rc; +} + +/* returns true is someone tried to get the cq while napi had it */ +static inline bool mlx4_en_cq_unlock_napi(struct mlx4_en_cq *cq) +{ + int rc = false; + spin_lock(&cq->poll_lock); + WARN_ON(cq->state & (MLX4_EN_CQ_STATE_POLL | + MLX4_EN_CQ_STATE_NAPI_YIELD)); + + if (cq->state & MLX4_EN_CQ_STATE_POLL_YIELD) + rc = true; + cq->state = MLX4_EN_CQ_STATE_IDLE; + spin_unlock(&cq->poll_lock); + return rc; +} + +/* called from mlx4_en_low_latency_poll() */ +static inline bool mlx4_en_cq_lock_poll(struct mlx4_en_cq *cq) +{ + int rc = true; + spin_lock_bh(&cq->poll_lock); + if ((cq->state & MLX4_CQ_LOCKED)) { + struct net_device *dev = cq->dev; + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_rx_ring *rx_ring = priv->rx_ring[cq->ring]; + + cq->state |= MLX4_EN_CQ_STATE_POLL_YIELD; + rc = false; +#ifdef LL_EXTENDED_STATS + rx_ring->yields++; +#endif + } else + /* preserve yield marks */ + cq->state |= MLX4_EN_CQ_STATE_POLL; + spin_unlock_bh(&cq->poll_lock); + return rc; +} + +/* returns true if someone tried to get the cq while it was locked */ +static inline bool mlx4_en_cq_unlock_poll(struct mlx4_en_cq *cq) +{ + int rc = false; + spin_lock_bh(&cq->poll_lock); + WARN_ON(cq->state & (MLX4_EN_CQ_STATE_NAPI)); + + if (cq->state & MLX4_EN_CQ_STATE_POLL_YIELD) + rc = true; + cq->state = MLX4_EN_CQ_STATE_IDLE; + spin_unlock_bh(&cq->poll_lock); + return rc; +} + +/* true if a socket is polling, even if it did not get the lock */ +static inline bool mlx4_en_cq_busy_polling(struct mlx4_en_cq *cq) +{ + WARN_ON(!(cq->state & MLX4_CQ_LOCKED)); + return cq->state & CQ_USER_PEND; +} +#else +static inline void mlx4_en_cq_init_lock(struct mlx4_en_cq *cq) +{ +} + +static inline bool mlx4_en_cq_lock_napi(struct mlx4_en_cq *cq) +{ + return true; +} + +static inline bool mlx4_en_cq_unlock_napi(struct mlx4_en_cq *cq) +{ + return false; +} + +static inline bool mlx4_en_cq_lock_poll(struct mlx4_en_cq *cq) +{ + return false; +} + +static inline bool mlx4_en_cq_unlock_poll(struct mlx4_en_cq *cq) +{ + return false; +} + +static inline bool mlx4_en_cq_busy_polling(struct mlx4_en_cq *cq) +{ + return false; +} +#endif /* CONFIG_NET_RX_BUSY_POLL */ + +#define MLX4_EN_WOL_DO_MODIFY (1ULL << 63) + +void mlx4_en_destroy_netdev(struct net_device *dev); +int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, + struct mlx4_en_port_profile *prof); + +int mlx4_en_start_port(struct net_device *dev); +void mlx4_en_stop_port(struct net_device *dev); + +void mlx4_en_free_resources(struct mlx4_en_priv *priv); +int mlx4_en_alloc_resources(struct mlx4_en_priv *priv); + +int mlx4_en_pre_config(struct mlx4_en_priv *priv); +int mlx4_en_create_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq, + int entries, int ring, enum cq_type mode, int node); +void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq); +int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq, + int cq_idx); +void mlx4_en_deactivate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); +int mlx4_en_set_cq_moder(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); +int mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); + +void mlx4_en_tx_irq(struct mlx4_cq *mcq); +u16 mlx4_en_select_queue(struct net_device *dev, struct mbuf *mb); + +int mlx4_en_transmit(struct ifnet *dev, struct mbuf *m); +int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring **pring, + u32 size, u16 stride, int node, int queue_idx); +void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring **pring); +int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring *ring, + int cq, int user_prio); +void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring *ring); +void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev); +void mlx4_en_qflush(struct ifnet *dev); + +int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring **pring, + u32 size, int node); +void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring **pring, + u32 size, u16 stride); +void mlx4_en_tx_que(void *context, int pending); +void mlx4_en_rx_que(void *context, int pending); +int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv); +void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring *ring); +int mlx4_en_process_rx_cq(struct net_device *dev, + struct mlx4_en_cq *cq, + int budget); +void mlx4_en_poll_tx_cq(unsigned long data); +void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride, + int is_tx, int rss, int qpn, int cqn, int user_prio, + struct mlx4_qp_context *context); +void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event); +int mlx4_en_map_buffer(struct mlx4_buf *buf); +void mlx4_en_unmap_buffer(struct mlx4_buf *buf); +void mlx4_en_calc_rx_buf(struct net_device *dev); + +const u32 *mlx4_en_get_rss_key(struct mlx4_en_priv *priv, u16 *keylen); +u8 mlx4_en_get_rss_mask(struct mlx4_en_priv *priv); +int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv); +void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv); +int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv); +void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv); +int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring); +void mlx4_en_rx_irq(struct mlx4_cq *mcq); + +int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, u64 mac, u64 clear, u8 mode); +int mlx4_SET_VLAN_FLTR(struct mlx4_dev *dev, struct mlx4_en_priv *priv); + +int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset); +int mlx4_en_QUERY_PORT(struct mlx4_en_dev *mdev, u8 port); +int mlx4_en_get_vport_stats(struct mlx4_en_dev *mdev, u8 port); +void mlx4_en_create_debug_files(struct mlx4_en_priv *priv); +void mlx4_en_delete_debug_files(struct mlx4_en_priv *priv); +int mlx4_en_register_debugfs(void); +void mlx4_en_unregister_debugfs(void); + +#ifdef CONFIG_MLX4_EN_DCB +extern const struct dcbnl_rtnl_ops mlx4_en_dcbnl_ops; +extern const struct dcbnl_rtnl_ops mlx4_en_dcbnl_pfc_ops; +#endif + +int mlx4_en_setup_tc(struct net_device *dev, u8 up); + +#ifdef CONFIG_RFS_ACCEL +void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring *rx_ring); +#endif + +#define MLX4_EN_NUM_SELF_TEST 5 +void mlx4_en_ex_selftest(struct net_device *dev, u32 *flags, u64 *buf); +void mlx4_en_ptp_overflow_check(struct mlx4_en_dev *mdev); + +/* + * Functions for time stamping + */ +#define SKBTX_HW_TSTAMP (1 << 0) +#define SKBTX_IN_PROGRESS (1 << 2) + +u64 mlx4_en_get_cqe_ts(struct mlx4_cqe *cqe); + +/* Functions for caching and restoring statistics */ +int mlx4_en_get_sset_count(struct net_device *dev, int sset); +void mlx4_en_restore_ethtool_stats(struct mlx4_en_priv *priv, + u64 *data); + +/* + * Globals + */ +extern const struct ethtool_ops mlx4_en_ethtool_ops; + +/* + * Defines for link speed - needed by selftest + */ +#define MLX4_EN_LINK_SPEED_1G 1000 +#define MLX4_EN_LINK_SPEED_10G 10000 +#define MLX4_EN_LINK_SPEED_40G 40000 + +enum { + NETIF_MSG_DRV = 0x0001, + NETIF_MSG_PROBE = 0x0002, + NETIF_MSG_LINK = 0x0004, + NETIF_MSG_TIMER = 0x0008, + NETIF_MSG_IFDOWN = 0x0010, + NETIF_MSG_IFUP = 0x0020, + NETIF_MSG_RX_ERR = 0x0040, + NETIF_MSG_TX_ERR = 0x0080, + NETIF_MSG_TX_QUEUED = 0x0100, + NETIF_MSG_INTR = 0x0200, + NETIF_MSG_TX_DONE = 0x0400, + NETIF_MSG_RX_STATUS = 0x0800, + NETIF_MSG_PKTDATA = 0x1000, + NETIF_MSG_HW = 0x2000, + NETIF_MSG_WOL = 0x4000, +}; + + +/* + * printk / logging functions + */ + +#define en_print(level, priv, format, arg...) \ + { \ + if ((priv)->registered) \ + printk(level "%s: %s: " format, DRV_NAME, \ + (priv)->dev->if_xname, ## arg); \ + else \ + printk(level "%s: %s: Port %d: " format, \ + DRV_NAME, dev_name(&(priv)->mdev->pdev->dev), \ + (priv)->port, ## arg); \ + } + + +#define en_dbg(mlevel, priv, format, arg...) \ +do { \ + if (NETIF_MSG_##mlevel & priv->msg_enable) \ + en_print(KERN_DEBUG, priv, format, ##arg); \ +} while (0) +#define en_warn(priv, format, arg...) \ + en_print(KERN_WARNING, priv, format, ##arg) +#define en_err(priv, format, arg...) \ + en_print(KERN_ERR, priv, format, ##arg) +#define en_info(priv, format, arg...) \ + en_print(KERN_INFO, priv, format, ## arg) + +#define mlx4_err(mdev, format, arg...) \ + pr_err("%s %s: " format, DRV_NAME, \ + dev_name(&(mdev)->pdev->dev), ##arg) +#define mlx4_info(mdev, format, arg...) \ + pr_info("%s %s: " format, DRV_NAME, \ + dev_name(&(mdev)->pdev->dev), ##arg) +#define mlx4_warn(mdev, format, arg...) \ + pr_warning("%s %s: " format, DRV_NAME, \ + dev_name(&(mdev)->pdev->dev), ##arg) + +#endif Property changes on: stable/11/sys/dev/mlx4/mlx4_en/en.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_en/en_port.h =================================================================== --- stable/11/sys/dev/mlx4/mlx4_en/en_port.h (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_en/en_port.h (revision 329159) @@ -0,0 +1,586 @@ +/* + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#ifndef _MLX4_EN_PORT_H_ +#define _MLX4_EN_PORT_H_ + + +#define SET_PORT_GEN_ALL_VALID 0x7 +#define SET_PORT_PROMISC_SHIFT 31 +#define SET_PORT_MC_PROMISC_SHIFT 30 + +#define MLX4_EN_NUM_TC 8 + +#define VLAN_FLTR_SIZE 128 +struct mlx4_set_vlan_fltr_mbox { + __be32 entry[VLAN_FLTR_SIZE]; +}; + + +enum { + MLX4_MCAST_CONFIG = 0, + MLX4_MCAST_DISABLE = 1, + MLX4_MCAST_ENABLE = 2, +}; + +enum mlx4_link_mode { + MLX4_1000BASE_CX_SGMII = 0, + MLX4_1000BASE_KX = 1, + MLX4_10GBASE_CX4 = 2, + MLX4_10GBASE_KX4 = 3, + MLX4_10GBASE_KR = 4, + MLX4_20GBASE_KR2 = 5, + MLX4_40GBASE_CR4 = 6, + MLX4_40GBASE_KR4 = 7, + MLX4_56GBASE_KR4 = 8, + MLX4_10GBASE_CR = 12, + MLX4_10GBASE_SR = 13, + MLX4_40GBASE_SR4 = 15, + MLX4_56GBASE_CR4 = 17, + MLX4_56GBASE_SR4 = 18, + MLX4_100BASE_TX = 24, + MLX4_1000BASE_T = 25, + MLX4_10GBASE_T = 26, +}; + +#define MLX4_PROT_MASK(link_mode) (1<<(link_mode)) + +enum { + MLX4_EN_100M_SPEED = 0x04, + MLX4_EN_10G_SPEED_XAUI = 0x00, + MLX4_EN_10G_SPEED_XFI = 0x01, + MLX4_EN_1G_SPEED = 0x02, + MLX4_EN_20G_SPEED = 0x08, + MLX4_EN_40G_SPEED = 0x40, + MLX4_EN_56G_SPEED = 0x20, + MLX4_EN_OTHER_SPEED = 0x0f, +}; + +struct mlx4_en_query_port_context { + u8 link_up; +#define MLX4_EN_LINK_UP_MASK 0x80 +#define MLX4_EN_ANC_MASK 0x40 + u8 autoneg; +#define MLX4_EN_AUTONEG_MASK 0x80 + __be16 mtu; + u8 reserved2; + u8 link_speed; +#define MLX4_EN_SPEED_MASK 0x6f + u16 reserved3[5]; + __be64 mac; + u8 transceiver; +}; + + +struct mlx4_en_stat_out_mbox { + /* Received frames with a length of 64 octets */ + __be64 R64_prio_0; + __be64 R64_prio_1; + __be64 R64_prio_2; + __be64 R64_prio_3; + __be64 R64_prio_4; + __be64 R64_prio_5; + __be64 R64_prio_6; + __be64 R64_prio_7; + __be64 R64_novlan; + /* Received frames with a length of 127 octets */ + __be64 R127_prio_0; + __be64 R127_prio_1; + __be64 R127_prio_2; + __be64 R127_prio_3; + __be64 R127_prio_4; + __be64 R127_prio_5; + __be64 R127_prio_6; + __be64 R127_prio_7; + __be64 R127_novlan; + /* Received frames with a length of 255 octets */ + __be64 R255_prio_0; + __be64 R255_prio_1; + __be64 R255_prio_2; + __be64 R255_prio_3; + __be64 R255_prio_4; + __be64 R255_prio_5; + __be64 R255_prio_6; + __be64 R255_prio_7; + __be64 R255_novlan; + /* Received frames with a length of 511 octets */ + __be64 R511_prio_0; + __be64 R511_prio_1; + __be64 R511_prio_2; + __be64 R511_prio_3; + __be64 R511_prio_4; + __be64 R511_prio_5; + __be64 R511_prio_6; + __be64 R511_prio_7; + __be64 R511_novlan; + /* Received frames with a length of 1023 octets */ + __be64 R1023_prio_0; + __be64 R1023_prio_1; + __be64 R1023_prio_2; + __be64 R1023_prio_3; + __be64 R1023_prio_4; + __be64 R1023_prio_5; + __be64 R1023_prio_6; + __be64 R1023_prio_7; + __be64 R1023_novlan; + /* Received frames with a length of 1518 octets */ + __be64 R1518_prio_0; + __be64 R1518_prio_1; + __be64 R1518_prio_2; + __be64 R1518_prio_3; + __be64 R1518_prio_4; + __be64 R1518_prio_5; + __be64 R1518_prio_6; + __be64 R1518_prio_7; + __be64 R1518_novlan; + /* Received frames with a length of 1522 octets */ + __be64 R1522_prio_0; + __be64 R1522_prio_1; + __be64 R1522_prio_2; + __be64 R1522_prio_3; + __be64 R1522_prio_4; + __be64 R1522_prio_5; + __be64 R1522_prio_6; + __be64 R1522_prio_7; + __be64 R1522_novlan; + /* Received frames with a length of 1548 octets */ + __be64 R1548_prio_0; + __be64 R1548_prio_1; + __be64 R1548_prio_2; + __be64 R1548_prio_3; + __be64 R1548_prio_4; + __be64 R1548_prio_5; + __be64 R1548_prio_6; + __be64 R1548_prio_7; + __be64 R1548_novlan; + /* Received frames with a length of 1548 < octets < MTU */ + __be64 R2MTU_prio_0; + __be64 R2MTU_prio_1; + __be64 R2MTU_prio_2; + __be64 R2MTU_prio_3; + __be64 R2MTU_prio_4; + __be64 R2MTU_prio_5; + __be64 R2MTU_prio_6; + __be64 R2MTU_prio_7; + __be64 R2MTU_novlan; + /* Received frames with a length of MTU< octets and good CRC */ + __be64 RGIANT_prio_0; + __be64 RGIANT_prio_1; + __be64 RGIANT_prio_2; + __be64 RGIANT_prio_3; + __be64 RGIANT_prio_4; + __be64 RGIANT_prio_5; + __be64 RGIANT_prio_6; + __be64 RGIANT_prio_7; + __be64 RGIANT_novlan; + /* Received broadcast frames with good CRC */ + __be64 RBCAST_prio_0; + __be64 RBCAST_prio_1; + __be64 RBCAST_prio_2; + __be64 RBCAST_prio_3; + __be64 RBCAST_prio_4; + __be64 RBCAST_prio_5; + __be64 RBCAST_prio_6; + __be64 RBCAST_prio_7; + __be64 RBCAST_novlan; + /* Received multicast frames with good CRC */ + __be64 MCAST_prio_0; + __be64 MCAST_prio_1; + __be64 MCAST_prio_2; + __be64 MCAST_prio_3; + __be64 MCAST_prio_4; + __be64 MCAST_prio_5; + __be64 MCAST_prio_6; + __be64 MCAST_prio_7; + __be64 MCAST_novlan; + /* Received unicast not short or GIANT frames with good CRC */ + __be64 RTOTG_prio_0; + __be64 RTOTG_prio_1; + __be64 RTOTG_prio_2; + __be64 RTOTG_prio_3; + __be64 RTOTG_prio_4; + __be64 RTOTG_prio_5; + __be64 RTOTG_prio_6; + __be64 RTOTG_prio_7; + __be64 RTOTG_novlan; + + /* Count of total octets of received frames, includes framing characters */ + __be64 RTTLOCT_prio_0; + /* Count of total octets of received frames, not including framing + characters */ + __be64 RTTLOCT_NOFRM_prio_0; + /* Count of Total number of octets received + (only for frames without errors) */ + __be64 ROCT_prio_0; + + __be64 RTTLOCT_prio_1; + __be64 RTTLOCT_NOFRM_prio_1; + __be64 ROCT_prio_1; + + __be64 RTTLOCT_prio_2; + __be64 RTTLOCT_NOFRM_prio_2; + __be64 ROCT_prio_2; + + __be64 RTTLOCT_prio_3; + __be64 RTTLOCT_NOFRM_prio_3; + __be64 ROCT_prio_3; + + __be64 RTTLOCT_prio_4; + __be64 RTTLOCT_NOFRM_prio_4; + __be64 ROCT_prio_4; + + __be64 RTTLOCT_prio_5; + __be64 RTTLOCT_NOFRM_prio_5; + __be64 ROCT_prio_5; + + __be64 RTTLOCT_prio_6; + __be64 RTTLOCT_NOFRM_prio_6; + __be64 ROCT_prio_6; + + __be64 RTTLOCT_prio_7; + __be64 RTTLOCT_NOFRM_prio_7; + __be64 ROCT_prio_7; + + __be64 RTTLOCT_novlan; + __be64 RTTLOCT_NOFRM_novlan; + __be64 ROCT_novlan; + + /* Count of Total received frames including bad frames */ + __be64 RTOT_prio_0; + /* Count of Total number of received frames with 802.1Q encapsulation */ + __be64 R1Q_prio_0; + __be64 reserved1; + + __be64 RTOT_prio_1; + __be64 R1Q_prio_1; + __be64 reserved2; + + __be64 RTOT_prio_2; + __be64 R1Q_prio_2; + __be64 reserved3; + + __be64 RTOT_prio_3; + __be64 R1Q_prio_3; + __be64 reserved4; + + __be64 RTOT_prio_4; + __be64 R1Q_prio_4; + __be64 reserved5; + + __be64 RTOT_prio_5; + __be64 R1Q_prio_5; + __be64 reserved6; + + __be64 RTOT_prio_6; + __be64 R1Q_prio_6; + __be64 reserved7; + + __be64 RTOT_prio_7; + __be64 R1Q_prio_7; + __be64 reserved8; + + __be64 RTOT_novlan; + __be64 R1Q_novlan; + __be64 reserved9; + + /* Total number of Successfully Received Control Frames */ + __be64 RCNTL; + __be64 reserved10; + __be64 reserved11; + __be64 reserved12; + /* Count of received frames with a length/type field value between 46 + (42 for VLANtagged frames) and 1500 (also 1500 for VLAN-tagged frames), + inclusive */ + __be64 RInRangeLengthErr; + /* Count of received frames with length/type field between 1501 and 1535 + decimal, inclusive */ + __be64 ROutRangeLengthErr; + /* Count of received frames that are longer than max allowed size for + 802.3 frames (1518/1522) */ + __be64 RFrmTooLong; + /* Count frames received with PCS error */ + __be64 PCS; + + /* Transmit frames with a length of 64 octets */ + __be64 T64_prio_0; + __be64 T64_prio_1; + __be64 T64_prio_2; + __be64 T64_prio_3; + __be64 T64_prio_4; + __be64 T64_prio_5; + __be64 T64_prio_6; + __be64 T64_prio_7; + __be64 T64_novlan; + __be64 T64_loopbk; + /* Transmit frames with a length of 65 to 127 octets. */ + __be64 T127_prio_0; + __be64 T127_prio_1; + __be64 T127_prio_2; + __be64 T127_prio_3; + __be64 T127_prio_4; + __be64 T127_prio_5; + __be64 T127_prio_6; + __be64 T127_prio_7; + __be64 T127_novlan; + __be64 T127_loopbk; + /* Transmit frames with a length of 128 to 255 octets */ + __be64 T255_prio_0; + __be64 T255_prio_1; + __be64 T255_prio_2; + __be64 T255_prio_3; + __be64 T255_prio_4; + __be64 T255_prio_5; + __be64 T255_prio_6; + __be64 T255_prio_7; + __be64 T255_novlan; + __be64 T255_loopbk; + /* Transmit frames with a length of 256 to 511 octets */ + __be64 T511_prio_0; + __be64 T511_prio_1; + __be64 T511_prio_2; + __be64 T511_prio_3; + __be64 T511_prio_4; + __be64 T511_prio_5; + __be64 T511_prio_6; + __be64 T511_prio_7; + __be64 T511_novlan; + __be64 T511_loopbk; + /* Transmit frames with a length of 512 to 1023 octets */ + __be64 T1023_prio_0; + __be64 T1023_prio_1; + __be64 T1023_prio_2; + __be64 T1023_prio_3; + __be64 T1023_prio_4; + __be64 T1023_prio_5; + __be64 T1023_prio_6; + __be64 T1023_prio_7; + __be64 T1023_novlan; + __be64 T1023_loopbk; + /* Transmit frames with a length of 1024 to 1518 octets */ + __be64 T1518_prio_0; + __be64 T1518_prio_1; + __be64 T1518_prio_2; + __be64 T1518_prio_3; + __be64 T1518_prio_4; + __be64 T1518_prio_5; + __be64 T1518_prio_6; + __be64 T1518_prio_7; + __be64 T1518_novlan; + __be64 T1518_loopbk; + /* Counts transmit frames with a length of 1519 to 1522 bytes */ + __be64 T1522_prio_0; + __be64 T1522_prio_1; + __be64 T1522_prio_2; + __be64 T1522_prio_3; + __be64 T1522_prio_4; + __be64 T1522_prio_5; + __be64 T1522_prio_6; + __be64 T1522_prio_7; + __be64 T1522_novlan; + __be64 T1522_loopbk; + /* Transmit frames with a length of 1523 to 1548 octets */ + __be64 T1548_prio_0; + __be64 T1548_prio_1; + __be64 T1548_prio_2; + __be64 T1548_prio_3; + __be64 T1548_prio_4; + __be64 T1548_prio_5; + __be64 T1548_prio_6; + __be64 T1548_prio_7; + __be64 T1548_novlan; + __be64 T1548_loopbk; + /* Counts transmit frames with a length of 1549 to MTU bytes */ + __be64 T2MTU_prio_0; + __be64 T2MTU_prio_1; + __be64 T2MTU_prio_2; + __be64 T2MTU_prio_3; + __be64 T2MTU_prio_4; + __be64 T2MTU_prio_5; + __be64 T2MTU_prio_6; + __be64 T2MTU_prio_7; + __be64 T2MTU_novlan; + __be64 T2MTU_loopbk; + /* Transmit frames with a length greater than MTU octets and a good CRC. */ + __be64 TGIANT_prio_0; + __be64 TGIANT_prio_1; + __be64 TGIANT_prio_2; + __be64 TGIANT_prio_3; + __be64 TGIANT_prio_4; + __be64 TGIANT_prio_5; + __be64 TGIANT_prio_6; + __be64 TGIANT_prio_7; + __be64 TGIANT_novlan; + __be64 TGIANT_loopbk; + /* Transmit broadcast frames with a good CRC */ + __be64 TBCAST_prio_0; + __be64 TBCAST_prio_1; + __be64 TBCAST_prio_2; + __be64 TBCAST_prio_3; + __be64 TBCAST_prio_4; + __be64 TBCAST_prio_5; + __be64 TBCAST_prio_6; + __be64 TBCAST_prio_7; + __be64 TBCAST_novlan; + __be64 TBCAST_loopbk; + /* Transmit multicast frames with a good CRC */ + __be64 TMCAST_prio_0; + __be64 TMCAST_prio_1; + __be64 TMCAST_prio_2; + __be64 TMCAST_prio_3; + __be64 TMCAST_prio_4; + __be64 TMCAST_prio_5; + __be64 TMCAST_prio_6; + __be64 TMCAST_prio_7; + __be64 TMCAST_novlan; + __be64 TMCAST_loopbk; + /* Transmit good frames that are neither broadcast nor multicast */ + __be64 TTOTG_prio_0; + __be64 TTOTG_prio_1; + __be64 TTOTG_prio_2; + __be64 TTOTG_prio_3; + __be64 TTOTG_prio_4; + __be64 TTOTG_prio_5; + __be64 TTOTG_prio_6; + __be64 TTOTG_prio_7; + __be64 TTOTG_novlan; + __be64 TTOTG_loopbk; + + /* total octets of transmitted frames, including framing characters */ + __be64 TTTLOCT_prio_0; + /* total octets of transmitted frames, not including framing characters */ + __be64 TTTLOCT_NOFRM_prio_0; + /* ifOutOctets */ + __be64 TOCT_prio_0; + + __be64 TTTLOCT_prio_1; + __be64 TTTLOCT_NOFRM_prio_1; + __be64 TOCT_prio_1; + + __be64 TTTLOCT_prio_2; + __be64 TTTLOCT_NOFRM_prio_2; + __be64 TOCT_prio_2; + + __be64 TTTLOCT_prio_3; + __be64 TTTLOCT_NOFRM_prio_3; + __be64 TOCT_prio_3; + + __be64 TTTLOCT_prio_4; + __be64 TTTLOCT_NOFRM_prio_4; + __be64 TOCT_prio_4; + + __be64 TTTLOCT_prio_5; + __be64 TTTLOCT_NOFRM_prio_5; + __be64 TOCT_prio_5; + + __be64 TTTLOCT_prio_6; + __be64 TTTLOCT_NOFRM_prio_6; + __be64 TOCT_prio_6; + + __be64 TTTLOCT_prio_7; + __be64 TTTLOCT_NOFRM_prio_7; + __be64 TOCT_prio_7; + + __be64 TTTLOCT_novlan; + __be64 TTTLOCT_NOFRM_novlan; + __be64 TOCT_novlan; + + __be64 TTTLOCT_loopbk; + __be64 TTTLOCT_NOFRM_loopbk; + __be64 TOCT_loopbk; + + /* Total frames transmitted with a good CRC that are not aborted */ + __be64 TTOT_prio_0; + /* Total number of frames transmitted with 802.1Q encapsulation */ + __be64 T1Q_prio_0; + __be64 reserved13; + + __be64 TTOT_prio_1; + __be64 T1Q_prio_1; + __be64 reserved14; + + __be64 TTOT_prio_2; + __be64 T1Q_prio_2; + __be64 reserved15; + + __be64 TTOT_prio_3; + __be64 T1Q_prio_3; + __be64 reserved16; + + __be64 TTOT_prio_4; + __be64 T1Q_prio_4; + __be64 reserved17; + + __be64 TTOT_prio_5; + __be64 T1Q_prio_5; + __be64 reserved18; + + __be64 TTOT_prio_6; + __be64 T1Q_prio_6; + __be64 reserved19; + + __be64 TTOT_prio_7; + __be64 T1Q_prio_7; + __be64 reserved20; + + __be64 TTOT_novlan; + __be64 T1Q_novlan; + __be64 reserved21; + + __be64 TTOT_loopbk; + __be64 T1Q_loopbk; + __be64 reserved22; + + /* Received frames with a length greater than MTU octets and a bad CRC */ + __be32 RJBBR; + /* Received frames with a bad CRC that are not runts, jabbers, + or alignment errors */ + __be32 RCRC; + /* Received frames with SFD with a length of less than 64 octets and a + bad CRC */ + __be32 RRUNT; + /* Received frames with a length less than 64 octets and a good CRC */ + __be32 RSHORT; + /* Total Number of Received Packets Dropped */ + __be32 RDROP; + /* Drop due to overflow */ + __be32 RdropOvflw; + /* Drop due to overflow */ + __be32 RdropLength; + /* Total of good frames. Does not include frames received with + frame-too-long, FCS, or length errors */ + __be32 RTOTFRMS; + /* Total dropped Xmited packets */ + __be32 TDROP; +}; + +#endif Property changes on: stable/11/sys/dev/mlx4/mlx4_en/en_port.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_cq.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_cq.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_cq.c (revision 329159) @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#include + +#include +#include +#include + +#include "en.h" + +static void mlx4_en_cq_event(struct mlx4_cq *cq, enum mlx4_event event) +{ + return; +} + + +int mlx4_en_create_cq(struct mlx4_en_priv *priv, + struct mlx4_en_cq **pcq, + int entries, int ring, enum cq_type mode, + int node) +{ + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_cq *cq; + int err; + + cq = kzalloc_node(sizeof(*cq), GFP_KERNEL, node); + if (!cq) { + cq = kzalloc(sizeof(*cq), GFP_KERNEL); + if (!cq) { + en_err(priv, "Failed to allocate CQ structure\n"); + return -ENOMEM; + } + } + + cq->size = entries; + cq->buf_size = cq->size * mdev->dev->caps.cqe_size; + + cq->tq = taskqueue_create_fast("mlx4_en_que", M_NOWAIT, + taskqueue_thread_enqueue, &cq->tq); + if (mode == RX) { + TASK_INIT(&cq->cq_task, 0, mlx4_en_rx_que, cq); + taskqueue_start_threads(&cq->tq, 1, PI_NET, "%s rx cq", + if_name(priv->dev)); + + } else { + TASK_INIT(&cq->cq_task, 0, mlx4_en_tx_que, cq); + taskqueue_start_threads(&cq->tq, 1, PI_NET, "%s tx cq", + if_name(priv->dev)); + } + + cq->ring = ring; + cq->is_tx = mode; + cq->vector = mdev->dev->caps.num_comp_vectors; + spin_lock_init(&cq->lock); + + err = mlx4_alloc_hwq_res(mdev->dev, &cq->wqres, + cq->buf_size, 2 * PAGE_SIZE); + if (err) + goto err_cq; + + err = mlx4_en_map_buffer(&cq->wqres.buf); + if (err) + goto err_res; + + cq->buf = (struct mlx4_cqe *)cq->wqres.buf.direct.buf; + *pcq = cq; + + return 0; + +err_res: + mlx4_free_hwq_res(mdev->dev, &cq->wqres, cq->buf_size); +err_cq: + kfree(cq); + *pcq = NULL; + return err; +} + +int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq, + int cq_idx) +{ + struct mlx4_en_dev *mdev = priv->mdev; + int err = 0; + int timestamp_en = 0; + bool assigned_eq = false; + + cq->dev = mdev->pndev[priv->port]; + cq->mcq.set_ci_db = cq->wqres.db.db; + cq->mcq.arm_db = cq->wqres.db.db + 1; + *cq->mcq.set_ci_db = 0; + *cq->mcq.arm_db = 0; + memset(cq->buf, 0, cq->buf_size); + + if (cq->is_tx == RX) { + if (!mlx4_is_eq_vector_valid(mdev->dev, priv->port, + cq->vector)) { + cq->vector = cq_idx % mdev->dev->caps.num_comp_vectors; + + err = mlx4_assign_eq(mdev->dev, priv->port, + &cq->vector); + if (err) { + mlx4_err(mdev, "Failed assigning an EQ to CQ vector %d\n", + cq->vector); + goto free_eq; + } + + assigned_eq = true; + } + } else { + struct mlx4_en_cq *rx_cq; + /* + * For TX we use the same irq per + * ring we assigned for the RX + */ + cq_idx = cq_idx % priv->rx_ring_num; + rx_cq = priv->rx_cq[cq_idx]; + cq->vector = rx_cq->vector; + } + + if (!cq->is_tx) + cq->size = priv->rx_ring[cq->ring]->actual_size; + + err = mlx4_cq_alloc(mdev->dev, cq->size, &cq->wqres.mtt, + &mdev->priv_uar, cq->wqres.db.dma, &cq->mcq, + cq->vector, 0, timestamp_en); + if (err) + goto free_eq; + + cq->mcq.comp = cq->is_tx ? mlx4_en_tx_irq : mlx4_en_rx_irq; + cq->mcq.event = mlx4_en_cq_event; + + if (cq->is_tx) { + init_timer(&cq->timer); + cq->timer.function = mlx4_en_poll_tx_cq; + cq->timer.data = (unsigned long) cq; + } + + + return 0; + +free_eq: + if (assigned_eq) + mlx4_release_eq(mdev->dev, cq->vector); + cq->vector = mdev->dev->caps.num_comp_vectors; + return err; +} + +void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq) +{ + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_cq *cq = *pcq; + + taskqueue_drain(cq->tq, &cq->cq_task); + taskqueue_free(cq->tq); + mlx4_en_unmap_buffer(&cq->wqres.buf); + mlx4_free_hwq_res(mdev->dev, &cq->wqres, cq->buf_size); + if (mlx4_is_eq_vector_valid(mdev->dev, priv->port, cq->vector) && + cq->is_tx == RX) + mlx4_release_eq(priv->mdev->dev, cq->vector); + cq->vector = 0; + cq->buf_size = 0; + cq->buf = NULL; + kfree(cq); + *pcq = NULL; +} + +void mlx4_en_deactivate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq) +{ + taskqueue_drain(cq->tq, &cq->cq_task); + if (!cq->is_tx) { + synchronize_rcu(); + } else { + del_timer_sync(&cq->timer); + } + + mlx4_cq_free(priv->mdev->dev, &cq->mcq); +} + +/* Set rx cq moderation parameters */ +int mlx4_en_set_cq_moder(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq) +{ + return mlx4_cq_modify(priv->mdev->dev, &cq->mcq, + cq->moder_cnt, cq->moder_time); +} + +int mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq) +{ + mlx4_cq_arm(&cq->mcq, MLX4_CQ_DB_REQ_NOT, priv->mdev->uar_map, + &priv->mdev->uar_lock); + + return 0; +} + + Property changes on: stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_cq.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_main.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_main.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_main.c (revision 329159) @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#define LINUXKPI_PARAM_PREFIX mlx4_ + +#include +#include +#include +#include + +#include +#include +#include + +#include "en.h" + +/* Mellanox ConnectX HCA Ethernet driver */ + +#define MLX4_EN_PARM_INT(X, def_val, desc) \ + static unsigned int X = def_val;\ + module_param(X , uint, 0444); \ + MODULE_PARM_DESC(X, desc); + + +/* + * Device scope module parameters + */ + +/* Enable RSS UDP traffic */ +MLX4_EN_PARM_INT(udp_rss, 1, + "Enable RSS for incoming UDP traffic or disabled (0)"); + +/* Priority pausing */ +MLX4_EN_PARM_INT(pfctx, 0, "Priority based Flow Control policy on TX[7:0]." + " Per priority bit mask"); +MLX4_EN_PARM_INT(pfcrx, 0, "Priority based Flow Control policy on RX[7:0]." + " Per priority bit mask"); + +MLX4_EN_PARM_INT(inline_thold, MAX_INLINE, + "Threshold for using inline data (range: 17-104, default: 104)"); + +#define MAX_PFC_TX 0xff +#define MAX_PFC_RX 0xff + +static int mlx4_en_get_profile(struct mlx4_en_dev *mdev) +{ + struct mlx4_en_profile *params = &mdev->profile; + int i; + + params->udp_rss = udp_rss; + params->num_tx_rings_p_up = min_t(int, mp_ncpus, + MLX4_EN_MAX_TX_RING_P_UP); + if (params->udp_rss && !(mdev->dev->caps.flags + & MLX4_DEV_CAP_FLAG_UDP_RSS)) { + mlx4_warn(mdev, "UDP RSS is not supported on this device.\n"); + params->udp_rss = 0; + } + for (i = 1; i <= MLX4_MAX_PORTS; i++) { + params->prof[i].rx_pause = 1; + params->prof[i].rx_ppp = pfcrx; + params->prof[i].tx_pause = 1; + params->prof[i].tx_ppp = pfctx; + params->prof[i].tx_ring_size = MLX4_EN_DEF_TX_RING_SIZE; + params->prof[i].rx_ring_size = MLX4_EN_DEF_RX_RING_SIZE; + params->prof[i].tx_ring_num = params->num_tx_rings_p_up * + MLX4_EN_NUM_UP; + params->prof[i].rss_rings = 0; + params->prof[i].inline_thold = inline_thold; + } + + return 0; +} + +static void *mlx4_en_get_netdev(struct mlx4_dev *dev, void *ctx, u8 port) +{ + struct mlx4_en_dev *endev = ctx; + + return endev->pndev[port]; +} + +static void mlx4_en_event(struct mlx4_dev *dev, void *endev_ptr, + enum mlx4_dev_event event, unsigned long port) +{ + struct mlx4_en_dev *mdev = (struct mlx4_en_dev *) endev_ptr; + struct mlx4_en_priv *priv; + + switch (event) { + case MLX4_DEV_EVENT_PORT_UP: + case MLX4_DEV_EVENT_PORT_DOWN: + if (!mdev->pndev[port]) + return; + priv = netdev_priv(mdev->pndev[port]); + /* To prevent races, we poll the link state in a separate + task rather than changing it here */ + priv->link_state = event; + queue_work(mdev->workqueue, &priv->linkstate_task); + break; + + case MLX4_DEV_EVENT_CATASTROPHIC_ERROR: + mlx4_err(mdev, "Internal error detected, restarting device\n"); + break; + + case MLX4_DEV_EVENT_SLAVE_INIT: + case MLX4_DEV_EVENT_SLAVE_SHUTDOWN: + break; + default: + if (port < 1 || port > dev->caps.num_ports || + !mdev->pndev[port]) + return; + mlx4_warn(mdev, "Unhandled event %d for port %d\n", event, + (int) port); + } +} + +static void mlx4_en_remove(struct mlx4_dev *dev, void *endev_ptr) +{ + struct mlx4_en_dev *mdev = endev_ptr; + int i; + + mutex_lock(&mdev->state_lock); + mdev->device_up = false; + mutex_unlock(&mdev->state_lock); + + mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) + if (mdev->pndev[i]) + mlx4_en_destroy_netdev(mdev->pndev[i]); + + flush_workqueue(mdev->workqueue); + destroy_workqueue(mdev->workqueue); + (void) mlx4_mr_free(dev, &mdev->mr); + iounmap(mdev->uar_map); + mlx4_uar_free(dev, &mdev->priv_uar); + mlx4_pd_free(dev, mdev->priv_pdn); + kfree(mdev); +} + +static void mlx4_en_activate(struct mlx4_dev *dev, void *ctx) +{ + int i; + struct mlx4_en_dev *mdev = ctx; + + /* Create a netdev for each port */ + mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { + mlx4_info(mdev, "Activating port:%d\n", i); + if (mlx4_en_init_netdev(mdev, i, &mdev->profile.prof[i])) + mdev->pndev[i] = NULL; + } +} + +static void *mlx4_en_add(struct mlx4_dev *dev) +{ + struct mlx4_en_dev *mdev; + int i; + + mdev = kzalloc(sizeof(*mdev), GFP_KERNEL); + if (!mdev) + goto err_free_res; + + if (mlx4_pd_alloc(dev, &mdev->priv_pdn)) + goto err_free_dev; + + if (mlx4_uar_alloc(dev, &mdev->priv_uar)) + goto err_pd; + + mdev->uar_map = ioremap((phys_addr_t) mdev->priv_uar.pfn << PAGE_SHIFT, + PAGE_SIZE); + if (!mdev->uar_map) + goto err_uar; + spin_lock_init(&mdev->uar_lock); + + mdev->dev = dev; + mdev->dma_device = &dev->persist->pdev->dev; + mdev->pdev = dev->persist->pdev; + mdev->device_up = false; + + mdev->LSO_support = !!(dev->caps.flags & (1 << 15)); + if (!mdev->LSO_support) + mlx4_warn(mdev, "LSO not supported, please upgrade to later " + "FW version to enable LSO\n"); + + if (mlx4_mr_alloc(mdev->dev, mdev->priv_pdn, 0, ~0ull, + MLX4_PERM_LOCAL_WRITE | MLX4_PERM_LOCAL_READ, + 0, 0, &mdev->mr)) { + mlx4_err(mdev, "Failed allocating memory region\n"); + goto err_map; + } + if (mlx4_mr_enable(mdev->dev, &mdev->mr)) { + mlx4_err(mdev, "Failed enabling memory region\n"); + goto err_mr; + } + + /* Build device profile according to supplied module parameters */ + if (mlx4_en_get_profile(mdev)) { + mlx4_err(mdev, "Bad module parameters, aborting\n"); + goto err_mr; + } + + /* Configure which ports to start according to module parameters */ + mdev->port_cnt = 0; + mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) + mdev->port_cnt++; + + /* Set default number of RX rings*/ + mlx4_en_set_num_rx_rings(mdev); + + /* Create our own workqueue for reset/multicast tasks + * Note: we cannot use the shared workqueue because of deadlocks caused + * by the rtnl lock */ + mdev->workqueue = create_singlethread_workqueue("mlx4_en"); + if (!mdev->workqueue) + goto err_mr; + + /* At this stage all non-port specific tasks are complete: + * mark the card state as up */ + mutex_init(&mdev->state_lock); + mdev->device_up = true; + + return mdev; + +err_mr: + (void) mlx4_mr_free(dev, &mdev->mr); +err_map: + if (mdev->uar_map) + iounmap(mdev->uar_map); +err_uar: + mlx4_uar_free(dev, &mdev->priv_uar); +err_pd: + mlx4_pd_free(dev, mdev->priv_pdn); +err_free_dev: + kfree(mdev); +err_free_res: + return NULL; +} + +static struct mlx4_interface mlx4_en_interface = { + .add = mlx4_en_add, + .remove = mlx4_en_remove, + .event = mlx4_en_event, + .get_dev = mlx4_en_get_netdev, + .protocol = MLX4_PROT_ETH, + .activate = mlx4_en_activate, +}; + +static void mlx4_en_verify_params(void) +{ + if (pfctx > MAX_PFC_TX) { + pr_warn("mlx4_en: WARNING: illegal module parameter pfctx 0x%x - should be in range 0-0x%x, will be changed to default (0)\n", + pfctx, MAX_PFC_TX); + pfctx = 0; + } + + if (pfcrx > MAX_PFC_RX) { + pr_warn("mlx4_en: WARNING: illegal module parameter pfcrx 0x%x - should be in range 0-0x%x, will be changed to default (0)\n", + pfcrx, MAX_PFC_RX); + pfcrx = 0; + } + + if (inline_thold < MIN_PKT_LEN || inline_thold > MAX_INLINE) { + pr_warn("mlx4_en: WARNING: illegal module parameter inline_thold %d - should be in range %d-%d, will be changed to default (%d)\n", + inline_thold, MIN_PKT_LEN, MAX_INLINE, MAX_INLINE); + inline_thold = MAX_INLINE; + } +} + +static int __init mlx4_en_init(void) +{ + mlx4_en_verify_params(); + + return mlx4_register_interface(&mlx4_en_interface); +} + +static void __exit mlx4_en_cleanup(void) +{ + mlx4_unregister_interface(&mlx4_en_interface); +} + +module_init(mlx4_en_init); +module_exit(mlx4_en_cleanup); + +static int +mlx4en_evhand(module_t mod, int event, void *arg) +{ + return (0); +} +static moduledata_t mlx4en_mod = { + .name = "mlx4en", + .evhand = mlx4en_evhand, +}; +DECLARE_MODULE(mlx4en, mlx4en_mod, SI_SUB_OFED_PREINIT, SI_ORDER_ANY); +MODULE_DEPEND(mlx4en, mlx4, 1, 1, 1); +MODULE_DEPEND(mlx4en, linuxkpi, 1, 1, 1); Property changes on: stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_main.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c (revision 329159) @@ -0,0 +1,2881 @@ +/* + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#include +#include +#include +#include +#ifdef CONFIG_NET_RX_BUSY_POLL +#include +#endif + +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include "en.h" +#include "en_port.h" + +static void mlx4_en_sysctl_stat(struct mlx4_en_priv *priv); +static void mlx4_en_sysctl_conf(struct mlx4_en_priv *priv); + +#ifdef CONFIG_NET_RX_BUSY_POLL +/* must be called with local_bh_disable()d */ +static int mlx4_en_low_latency_recv(struct napi_struct *napi) +{ + struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi); + struct net_device *dev = cq->dev; + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_rx_ring *rx_ring = priv->rx_ring[cq->ring]; + int done; + + if (!priv->port_up) + return LL_FLUSH_FAILED; + + if (!mlx4_en_cq_lock_poll(cq)) + return LL_FLUSH_BUSY; + + done = mlx4_en_process_rx_cq(dev, cq, 4); +#ifdef LL_EXTENDED_STATS + if (likely(done)) + rx_ring->cleaned += done; + else + rx_ring->misses++; +#endif + + mlx4_en_cq_unlock_poll(cq); + + return done; +} +#endif /* CONFIG_NET_RX_BUSY_POLL */ + +#ifdef CONFIG_RFS_ACCEL + +struct mlx4_en_filter { + struct list_head next; + struct work_struct work; + + u8 ip_proto; + __be32 src_ip; + __be32 dst_ip; + __be16 src_port; + __be16 dst_port; + + int rxq_index; + struct mlx4_en_priv *priv; + u32 flow_id; /* RFS infrastructure id */ + int id; /* mlx4_en driver id */ + u64 reg_id; /* Flow steering API id */ + u8 activated; /* Used to prevent expiry before filter + * is attached + */ + struct hlist_node filter_chain; +}; + +static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv); + +static enum mlx4_net_trans_rule_id mlx4_ip_proto_to_trans_rule_id(u8 ip_proto) +{ + switch (ip_proto) { + case IPPROTO_UDP: + return MLX4_NET_TRANS_RULE_ID_UDP; + case IPPROTO_TCP: + return MLX4_NET_TRANS_RULE_ID_TCP; + default: + return MLX4_NET_TRANS_RULE_NUM; + } +}; + +static void mlx4_en_filter_work(struct work_struct *work) +{ + struct mlx4_en_filter *filter = container_of(work, + struct mlx4_en_filter, + work); + struct mlx4_en_priv *priv = filter->priv; + struct mlx4_spec_list spec_tcp_udp = { + .id = mlx4_ip_proto_to_trans_rule_id(filter->ip_proto), + { + .tcp_udp = { + .dst_port = filter->dst_port, + .dst_port_msk = (__force __be16)-1, + .src_port = filter->src_port, + .src_port_msk = (__force __be16)-1, + }, + }, + }; + struct mlx4_spec_list spec_ip = { + .id = MLX4_NET_TRANS_RULE_ID_IPV4, + { + .ipv4 = { + .dst_ip = filter->dst_ip, + .dst_ip_msk = (__force __be32)-1, + .src_ip = filter->src_ip, + .src_ip_msk = (__force __be32)-1, + }, + }, + }; + struct mlx4_spec_list spec_eth = { + .id = MLX4_NET_TRANS_RULE_ID_ETH, + }; + struct mlx4_net_trans_rule rule = { + .list = LIST_HEAD_INIT(rule.list), + .queue_mode = MLX4_NET_TRANS_Q_LIFO, + .exclusive = 1, + .allow_loopback = 1, + .promisc_mode = MLX4_FS_REGULAR, + .port = priv->port, + .priority = MLX4_DOMAIN_RFS, + }; + int rc; + __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); + + if (spec_tcp_udp.id >= MLX4_NET_TRANS_RULE_NUM) { + en_warn(priv, "RFS: ignoring unsupported ip protocol (%d)\n", + filter->ip_proto); + goto ignore; + } + list_add_tail(&spec_eth.list, &rule.list); + list_add_tail(&spec_ip.list, &rule.list); + list_add_tail(&spec_tcp_udp.list, &rule.list); + + rule.qpn = priv->rss_map.qps[filter->rxq_index].qpn; + memcpy(spec_eth.eth.dst_mac, priv->dev->dev_addr, ETH_ALEN); + memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN); + + filter->activated = 0; + + if (filter->reg_id) { + rc = mlx4_flow_detach(priv->mdev->dev, filter->reg_id); + if (rc && rc != -ENOENT) + en_err(priv, "Error detaching flow. rc = %d\n", rc); + } + + rc = mlx4_flow_attach(priv->mdev->dev, &rule, &filter->reg_id); + if (rc) + en_err(priv, "Error attaching flow. err = %d\n", rc); + +ignore: + mlx4_en_filter_rfs_expire(priv); + + filter->activated = 1; +} + +static inline struct hlist_head * +filter_hash_bucket(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip, + __be16 src_port, __be16 dst_port) +{ + unsigned long l; + int bucket_idx; + + l = (__force unsigned long)src_port | + ((__force unsigned long)dst_port << 2); + l ^= (__force unsigned long)(src_ip ^ dst_ip); + + bucket_idx = hash_long(l, MLX4_EN_FILTER_HASH_SHIFT); + + return &priv->filter_hash[bucket_idx]; +} + +static struct mlx4_en_filter * +mlx4_en_filter_alloc(struct mlx4_en_priv *priv, int rxq_index, __be32 src_ip, + __be32 dst_ip, u8 ip_proto, __be16 src_port, + __be16 dst_port, u32 flow_id) +{ + struct mlx4_en_filter *filter = NULL; + + filter = kzalloc(sizeof(struct mlx4_en_filter), GFP_ATOMIC); + if (!filter) + return NULL; + + filter->priv = priv; + filter->rxq_index = rxq_index; + INIT_WORK(&filter->work, mlx4_en_filter_work); + + filter->src_ip = src_ip; + filter->dst_ip = dst_ip; + filter->ip_proto = ip_proto; + filter->src_port = src_port; + filter->dst_port = dst_port; + + filter->flow_id = flow_id; + + filter->id = priv->last_filter_id++ % RPS_NO_FILTER; + + list_add_tail(&filter->next, &priv->filters); + hlist_add_head(&filter->filter_chain, + filter_hash_bucket(priv, src_ip, dst_ip, src_port, + dst_port)); + + return filter; +} + +static void mlx4_en_filter_free(struct mlx4_en_filter *filter) +{ + struct mlx4_en_priv *priv = filter->priv; + int rc; + + list_del(&filter->next); + + rc = mlx4_flow_detach(priv->mdev->dev, filter->reg_id); + if (rc && rc != -ENOENT) + en_err(priv, "Error detaching flow. rc = %d\n", rc); + + kfree(filter); +} + +static inline struct mlx4_en_filter * +mlx4_en_filter_find(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip, + u8 ip_proto, __be16 src_port, __be16 dst_port) +{ + struct mlx4_en_filter *filter; + struct mlx4_en_filter *ret = NULL; + + hlist_for_each_entry(filter, + filter_hash_bucket(priv, src_ip, dst_ip, + src_port, dst_port), + filter_chain) { + if (filter->src_ip == src_ip && + filter->dst_ip == dst_ip && + filter->ip_proto == ip_proto && + filter->src_port == src_port && + filter->dst_port == dst_port) { + ret = filter; + break; + } + } + + return ret; +} + +static int +mlx4_en_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, + u16 rxq_index, u32 flow_id) +{ + struct mlx4_en_priv *priv = netdev_priv(net_dev); + struct mlx4_en_filter *filter; + const struct iphdr *ip; + const __be16 *ports; + u8 ip_proto; + __be32 src_ip; + __be32 dst_ip; + __be16 src_port; + __be16 dst_port; + int nhoff = skb_network_offset(skb); + int ret = 0; + + if (skb->protocol != htons(ETH_P_IP)) + return -EPROTONOSUPPORT; + + ip = (const struct iphdr *)(skb->data + nhoff); + if (ip_is_fragment(ip)) + return -EPROTONOSUPPORT; + + if ((ip->protocol != IPPROTO_TCP) && (ip->protocol != IPPROTO_UDP)) + return -EPROTONOSUPPORT; + ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl); + + ip_proto = ip->protocol; + src_ip = ip->saddr; + dst_ip = ip->daddr; + src_port = ports[0]; + dst_port = ports[1]; + + spin_lock_bh(&priv->filters_lock); + filter = mlx4_en_filter_find(priv, src_ip, dst_ip, ip_proto, + src_port, dst_port); + if (filter) { + if (filter->rxq_index == rxq_index) + goto out; + + filter->rxq_index = rxq_index; + } else { + filter = mlx4_en_filter_alloc(priv, rxq_index, + src_ip, dst_ip, ip_proto, + src_port, dst_port, flow_id); + if (!filter) { + ret = -ENOMEM; + goto err; + } + } + + queue_work(priv->mdev->workqueue, &filter->work); + +out: + ret = filter->id; +err: + spin_unlock_bh(&priv->filters_lock); + + return ret; +} + +void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv) +{ + struct mlx4_en_filter *filter, *tmp; + LIST_HEAD(del_list); + + spin_lock_bh(&priv->filters_lock); + list_for_each_entry_safe(filter, tmp, &priv->filters, next) { + list_move(&filter->next, &del_list); + hlist_del(&filter->filter_chain); + } + spin_unlock_bh(&priv->filters_lock); + + list_for_each_entry_safe(filter, tmp, &del_list, next) { + cancel_work_sync(&filter->work); + mlx4_en_filter_free(filter); + } +} + +static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv) +{ + struct mlx4_en_filter *filter = NULL, *tmp, *last_filter = NULL; + LIST_HEAD(del_list); + int i = 0; + + spin_lock_bh(&priv->filters_lock); + list_for_each_entry_safe(filter, tmp, &priv->filters, next) { + if (i > MLX4_EN_FILTER_EXPIRY_QUOTA) + break; + + if (filter->activated && + !work_pending(&filter->work) && + rps_may_expire_flow(priv->dev, + filter->rxq_index, filter->flow_id, + filter->id)) { + list_move(&filter->next, &del_list); + hlist_del(&filter->filter_chain); + } else + last_filter = filter; + + i++; + } + + if (last_filter && (&last_filter->next != priv->filters.next)) + list_move(&priv->filters, &last_filter->next); + + spin_unlock_bh(&priv->filters_lock); + + list_for_each_entry_safe(filter, tmp, &del_list, next) + mlx4_en_filter_free(filter); +} +#endif + +static void mlx4_en_vlan_rx_add_vid(void *arg, struct net_device *dev, u16 vid) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int err; + int idx; + + if (arg != priv) + return; + + en_dbg(HW, priv, "adding VLAN:%d\n", vid); + + set_bit(vid, priv->active_vlans); + + /* Add VID to port VLAN filter */ + mutex_lock(&mdev->state_lock); + if (mdev->device_up && priv->port_up) { + err = mlx4_SET_VLAN_FLTR(mdev->dev, priv); + if (err) + en_err(priv, "Failed configuring VLAN filter\n"); + } + if (mlx4_register_vlan(mdev->dev, priv->port, vid, &idx)) + en_dbg(HW, priv, "failed adding vlan %d\n", vid); + mutex_unlock(&mdev->state_lock); + +} + +static void mlx4_en_vlan_rx_kill_vid(void *arg, struct net_device *dev, u16 vid) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int err; + + if (arg != priv) + return; + + en_dbg(HW, priv, "Killing VID:%d\n", vid); + + clear_bit(vid, priv->active_vlans); + + /* Remove VID from port VLAN filter */ + mutex_lock(&mdev->state_lock); + mlx4_unregister_vlan(mdev->dev, priv->port, vid); + + if (mdev->device_up && priv->port_up) { + err = mlx4_SET_VLAN_FLTR(mdev->dev, priv); + if (err) + en_err(priv, "Failed configuring VLAN filter\n"); + } + mutex_unlock(&mdev->state_lock); + +} + +static int mlx4_en_tunnel_steer_add(struct mlx4_en_priv *priv, unsigned char *addr, + int qpn, u64 *reg_id) +{ + int err; + + if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN || + priv->mdev->dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_STATIC) + return 0; /* do nothing */ + + err = mlx4_tunnel_steer_add(priv->mdev->dev, addr, priv->port, qpn, + MLX4_DOMAIN_NIC, reg_id); + if (err) { + en_err(priv, "failed to add vxlan steering rule, err %d\n", err); + return err; + } + en_dbg(DRV, priv, "added vxlan steering rule, mac %pM reg_id %llx\n", addr, (long long)*reg_id); + return 0; +} + +static int mlx4_en_uc_steer_add(struct mlx4_en_priv *priv, + unsigned char *mac, int *qpn, u64 *reg_id) +{ + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_dev *dev = mdev->dev; + int err; + + switch (dev->caps.steering_mode) { + case MLX4_STEERING_MODE_B0: { + struct mlx4_qp qp; + u8 gid[16] = {0}; + + qp.qpn = *qpn; + memcpy(&gid[10], mac, ETH_ALEN); + gid[5] = priv->port; + + err = mlx4_unicast_attach(dev, &qp, gid, 0, MLX4_PROT_ETH); + break; + } + case MLX4_STEERING_MODE_DEVICE_MANAGED: { + struct mlx4_spec_list spec_eth = { {NULL} }; + __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); + + struct mlx4_net_trans_rule rule = { + .queue_mode = MLX4_NET_TRANS_Q_FIFO, + .exclusive = 0, + .allow_loopback = 1, + .promisc_mode = MLX4_FS_REGULAR, + .priority = MLX4_DOMAIN_NIC, + }; + + rule.port = priv->port; + rule.qpn = *qpn; + INIT_LIST_HEAD(&rule.list); + + spec_eth.id = MLX4_NET_TRANS_RULE_ID_ETH; + memcpy(spec_eth.eth.dst_mac, mac, ETH_ALEN); + memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN); + list_add_tail(&spec_eth.list, &rule.list); + + err = mlx4_flow_attach(dev, &rule, reg_id); + break; + } + default: + return -EINVAL; + } + if (err) + en_warn(priv, "Failed Attaching Unicast\n"); + + return err; +} + +static void mlx4_en_uc_steer_release(struct mlx4_en_priv *priv, + unsigned char *mac, int qpn, u64 reg_id) +{ + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_dev *dev = mdev->dev; + + switch (dev->caps.steering_mode) { + case MLX4_STEERING_MODE_B0: { + struct mlx4_qp qp; + u8 gid[16] = {0}; + + qp.qpn = qpn; + memcpy(&gid[10], mac, ETH_ALEN); + gid[5] = priv->port; + + mlx4_unicast_detach(dev, &qp, gid, MLX4_PROT_ETH); + break; + } + case MLX4_STEERING_MODE_DEVICE_MANAGED: { + mlx4_flow_detach(dev, reg_id); + break; + } + default: + en_err(priv, "Invalid steering mode.\n"); + } +} + +static int mlx4_en_get_qp(struct mlx4_en_priv *priv) +{ + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_dev *dev = mdev->dev; + int index = 0; + int err = 0; + int *qpn = &priv->base_qpn; + u64 mac = mlx4_mac_to_u64(IF_LLADDR(priv->dev)); + + en_dbg(DRV, priv, "Registering MAC: %pM for adding\n", + IF_LLADDR(priv->dev)); + index = mlx4_register_mac(dev, priv->port, mac); + if (index < 0) { + err = index; + en_err(priv, "Failed adding MAC: %pM\n", + IF_LLADDR(priv->dev)); + return err; + } + + if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) { + int base_qpn = mlx4_get_base_qpn(dev, priv->port); + *qpn = base_qpn + index; + return 0; + } + + err = mlx4_qp_reserve_range(dev, 1, 1, qpn, MLX4_RESERVE_A0_QP); + en_dbg(DRV, priv, "Reserved qp %d\n", *qpn); + if (err) { + en_err(priv, "Failed to reserve qp for mac registration\n"); + mlx4_unregister_mac(dev, priv->port, mac); + return err; + } + + return 0; +} + +static void mlx4_en_put_qp(struct mlx4_en_priv *priv) +{ + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_dev *dev = mdev->dev; + int qpn = priv->base_qpn; + + if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) { + u64 mac = mlx4_mac_to_u64(IF_LLADDR(priv->dev)); + en_dbg(DRV, priv, "Registering MAC: %pM for deleting\n", + IF_LLADDR(priv->dev)); + mlx4_unregister_mac(dev, priv->port, mac); + } else { + en_dbg(DRV, priv, "Releasing qp: port %d, qpn %d\n", + priv->port, qpn); + mlx4_qp_release_range(dev, qpn, 1); + priv->flags &= ~MLX4_EN_FLAG_FORCE_PROMISC; + } +} + +static void mlx4_en_clear_uclist(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_addr_list *tmp, *uc_to_del; + + list_for_each_entry_safe(uc_to_del, tmp, &priv->uc_list, list) { + list_del(&uc_to_del->list); + kfree(uc_to_del); + } +} + +static void mlx4_en_cache_uclist(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_addr_list *tmp; + struct ifaddr *ifa; + + mlx4_en_clear_uclist(dev); + + if_addr_rlock(dev); + TAILQ_FOREACH(ifa, &dev->if_addrhead, ifa_link) { + if (ifa->ifa_addr->sa_family != AF_LINK) + continue; + if (((struct sockaddr_dl *)ifa->ifa_addr)->sdl_alen != + ETHER_ADDR_LEN) + continue; + tmp = kzalloc(sizeof(struct mlx4_en_addr_list), GFP_ATOMIC); + if (tmp == NULL) { + en_err(priv, "Failed to allocate address list\n"); + break; + } + memcpy(tmp->addr, + LLADDR((struct sockaddr_dl *)ifa->ifa_addr), ETH_ALEN); + list_add_tail(&tmp->list, &priv->uc_list); + } + if_addr_runlock(dev); +} + +static void mlx4_en_clear_mclist(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_addr_list *tmp, *mc_to_del; + + list_for_each_entry_safe(mc_to_del, tmp, &priv->mc_list, list) { + list_del(&mc_to_del->list); + kfree(mc_to_del); + } +} + +static void mlx4_en_cache_mclist(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_addr_list *tmp; + struct ifmultiaddr *ifma; + + mlx4_en_clear_mclist(dev); + + if_maddr_rlock(dev); + TAILQ_FOREACH(ifma, &dev->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + if (((struct sockaddr_dl *)ifma->ifma_addr)->sdl_alen != + ETHER_ADDR_LEN) + continue; + tmp = kzalloc(sizeof(struct mlx4_en_addr_list), GFP_ATOMIC); + if (tmp == NULL) { + en_err(priv, "Failed to allocate address list\n"); + break; + } + memcpy(tmp->addr, + LLADDR((struct sockaddr_dl *)ifma->ifma_addr), ETH_ALEN); + list_add_tail(&tmp->list, &priv->mc_list); + } + if_maddr_runlock(dev); +} + +static void update_addr_list_flags(struct mlx4_en_priv *priv, + struct list_head *dst, + struct list_head *src) +{ + struct mlx4_en_addr_list *dst_tmp, *src_tmp, *new_mc; + bool found; + + /* Find all the entries that should be removed from dst, + * These are the entries that are not found in src + */ + list_for_each_entry(dst_tmp, dst, list) { + found = false; + list_for_each_entry(src_tmp, src, list) { + if (!memcmp(dst_tmp->addr, src_tmp->addr, ETH_ALEN)) { + found = true; + break; + } + } + if (!found) + dst_tmp->action = MLX4_ADDR_LIST_REM; + } + + /* Add entries that exist in src but not in dst + * mark them as need to add + */ + list_for_each_entry(src_tmp, src, list) { + found = false; + list_for_each_entry(dst_tmp, dst, list) { + if (!memcmp(dst_tmp->addr, src_tmp->addr, ETH_ALEN)) { + dst_tmp->action = MLX4_ADDR_LIST_NONE; + found = true; + break; + } + } + if (!found) { + new_mc = kmalloc(sizeof(struct mlx4_en_addr_list), + GFP_KERNEL); + if (!new_mc) { + en_err(priv, "Failed to allocate current multicast list\n"); + return; + } + memcpy(new_mc, src_tmp, + sizeof(struct mlx4_en_addr_list)); + new_mc->action = MLX4_ADDR_LIST_ADD; + list_add_tail(&new_mc->list, dst); + } + } +} + +static void mlx4_en_set_rx_mode(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + + if (!priv->port_up) + return; + + queue_work(priv->mdev->workqueue, &priv->rx_mode_task); +} + +static void mlx4_en_set_promisc_mode(struct mlx4_en_priv *priv, + struct mlx4_en_dev *mdev) +{ + int err = 0; + + if (!(priv->flags & MLX4_EN_FLAG_PROMISC)) { + priv->flags |= MLX4_EN_FLAG_PROMISC; + + /* Enable promiscouos mode */ + switch (mdev->dev->caps.steering_mode) { + case MLX4_STEERING_MODE_DEVICE_MANAGED: + err = mlx4_flow_steer_promisc_add(mdev->dev, + priv->port, + priv->base_qpn, + MLX4_FS_ALL_DEFAULT); + if (err) + en_err(priv, "Failed enabling promiscuous mode\n"); + priv->flags |= MLX4_EN_FLAG_MC_PROMISC; + break; + + case MLX4_STEERING_MODE_B0: + err = mlx4_unicast_promisc_add(mdev->dev, + priv->base_qpn, + priv->port); + if (err) + en_err(priv, "Failed enabling unicast promiscuous mode\n"); + + /* Add the default qp number as multicast + * promisc + */ + if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) { + err = mlx4_multicast_promisc_add(mdev->dev, + priv->base_qpn, + priv->port); + if (err) + en_err(priv, "Failed enabling multicast promiscuous mode\n"); + priv->flags |= MLX4_EN_FLAG_MC_PROMISC; + } + break; + + case MLX4_STEERING_MODE_A0: + err = mlx4_SET_PORT_qpn_calc(mdev->dev, + priv->port, + priv->base_qpn, + 1); + if (err) + en_err(priv, "Failed enabling promiscuous mode\n"); + break; + } + + /* Disable port multicast filter (unconditionally) */ + err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, + 0, MLX4_MCAST_DISABLE); + if (err) + en_err(priv, "Failed disabling multicast filter\n"); + } +} + +static void mlx4_en_clear_promisc_mode(struct mlx4_en_priv *priv, + struct mlx4_en_dev *mdev) +{ + int err = 0; + + priv->flags &= ~MLX4_EN_FLAG_PROMISC; + + /* Disable promiscouos mode */ + switch (mdev->dev->caps.steering_mode) { + case MLX4_STEERING_MODE_DEVICE_MANAGED: + err = mlx4_flow_steer_promisc_remove(mdev->dev, + priv->port, + MLX4_FS_ALL_DEFAULT); + if (err) + en_err(priv, "Failed disabling promiscuous mode\n"); + priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; + break; + + case MLX4_STEERING_MODE_B0: + err = mlx4_unicast_promisc_remove(mdev->dev, + priv->base_qpn, + priv->port); + if (err) + en_err(priv, "Failed disabling unicast promiscuous mode\n"); + /* Disable Multicast promisc */ + if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { + err = mlx4_multicast_promisc_remove(mdev->dev, + priv->base_qpn, + priv->port); + if (err) + en_err(priv, "Failed disabling multicast promiscuous mode\n"); + priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; + } + break; + + case MLX4_STEERING_MODE_A0: + err = mlx4_SET_PORT_qpn_calc(mdev->dev, + priv->port, + priv->base_qpn, 0); + if (err) + en_err(priv, "Failed disabling promiscuous mode\n"); + break; + } +} + +static void mlx4_en_do_multicast(struct mlx4_en_priv *priv, + struct net_device *dev, + struct mlx4_en_dev *mdev) +{ + struct mlx4_en_addr_list *addr_list, *tmp; + u8 mc_list[16] = {0}; + int err = 0; + u64 mcast_addr = 0; + + + /* Enable/disable the multicast filter according to IFF_ALLMULTI */ + if (dev->if_flags & IFF_ALLMULTI) { + err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, + 0, MLX4_MCAST_DISABLE); + if (err) + en_err(priv, "Failed disabling multicast filter\n"); + + /* Add the default qp number as multicast promisc */ + if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) { + switch (mdev->dev->caps.steering_mode) { + case MLX4_STEERING_MODE_DEVICE_MANAGED: + err = mlx4_flow_steer_promisc_add(mdev->dev, + priv->port, + priv->base_qpn, + MLX4_FS_MC_DEFAULT); + break; + + case MLX4_STEERING_MODE_B0: + err = mlx4_multicast_promisc_add(mdev->dev, + priv->base_qpn, + priv->port); + break; + + case MLX4_STEERING_MODE_A0: + break; + } + if (err) + en_err(priv, "Failed entering multicast promisc mode\n"); + priv->flags |= MLX4_EN_FLAG_MC_PROMISC; + } + } else { + /* Disable Multicast promisc */ + if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { + switch (mdev->dev->caps.steering_mode) { + case MLX4_STEERING_MODE_DEVICE_MANAGED: + err = mlx4_flow_steer_promisc_remove(mdev->dev, + priv->port, + MLX4_FS_MC_DEFAULT); + break; + + case MLX4_STEERING_MODE_B0: + err = mlx4_multicast_promisc_remove(mdev->dev, + priv->base_qpn, + priv->port); + break; + + case MLX4_STEERING_MODE_A0: + break; + } + if (err) + en_err(priv, "Failed disabling multicast promiscuous mode\n"); + priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; + } + + /* Update unicast list */ + mlx4_en_cache_uclist(dev); + + update_addr_list_flags(priv, &priv->curr_uc_list, &priv->uc_list); + + list_for_each_entry_safe(addr_list, tmp, &priv->curr_uc_list, list) { + if (addr_list->action == MLX4_ADDR_LIST_REM) { + mlx4_en_uc_steer_release(priv, addr_list->addr, + priv->rss_map.indir_qp.qpn, + addr_list->reg_id); + /* remove from list */ + list_del(&addr_list->list); + kfree(addr_list); + } else if (addr_list->action == MLX4_ADDR_LIST_ADD) { + err = mlx4_en_uc_steer_add(priv, addr_list->addr, + &priv->rss_map.indir_qp.qpn, + &addr_list->reg_id); + if (err) + en_err(priv, "Fail to add unicast address\n"); + } + } + + err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, + 0, MLX4_MCAST_DISABLE); + if (err) + en_err(priv, "Failed disabling multicast filter\n"); + + /* Flush mcast filter and init it with broadcast address */ + mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, ETH_BCAST, + 1, MLX4_MCAST_CONFIG); + + /* Update multicast list - we cache all addresses so they won't + * change while HW is updated holding the command semaphor */ + mlx4_en_cache_mclist(dev); + list_for_each_entry(addr_list, &priv->mc_list, list) { + mcast_addr = mlx4_mac_to_u64(addr_list->addr); + mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, + mcast_addr, 0, MLX4_MCAST_CONFIG); + } + err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, + 0, MLX4_MCAST_ENABLE); + if (err) + en_err(priv, "Failed enabling multicast filter\n"); + + update_addr_list_flags(priv, &priv->curr_mc_list, &priv->mc_list); + + list_for_each_entry_safe(addr_list, tmp, &priv->curr_mc_list, list) { + if (addr_list->action == MLX4_ADDR_LIST_REM) { + /* detach this address and delete from list */ + memcpy(&mc_list[10], addr_list->addr, ETH_ALEN); + mc_list[5] = priv->port; + err = mlx4_multicast_detach(mdev->dev, + &priv->rss_map.indir_qp, + mc_list, + MLX4_PROT_ETH, + addr_list->reg_id); + if (err) + en_err(priv, "Fail to detach multicast address\n"); + + if (addr_list->tunnel_reg_id) { + err = mlx4_flow_detach(priv->mdev->dev, addr_list->tunnel_reg_id); + if (err) + en_err(priv, "Failed to detach multicast address\n"); + } + + /* remove from list */ + list_del(&addr_list->list); + kfree(addr_list); + } else if (addr_list->action == MLX4_ADDR_LIST_ADD) { + /* attach the address */ + memcpy(&mc_list[10], addr_list->addr, ETH_ALEN); + /* needed for B0 steering support */ + mc_list[5] = priv->port; + err = mlx4_multicast_attach(mdev->dev, + &priv->rss_map.indir_qp, + mc_list, + priv->port, 0, + MLX4_PROT_ETH, + &addr_list->reg_id); + if (err) + en_err(priv, "Fail to attach multicast address\n"); + + err = mlx4_en_tunnel_steer_add(priv, &mc_list[10], priv->base_qpn, + &addr_list->tunnel_reg_id); + if (err) + en_err(priv, "Failed to attach multicast address\n"); + } + } + } +} + +static void mlx4_en_do_set_rx_mode(struct work_struct *work) +{ + struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, + rx_mode_task); + struct mlx4_en_dev *mdev = priv->mdev; + struct net_device *dev = priv->dev; + + mutex_lock(&mdev->state_lock); + if (!mdev->device_up) { + en_dbg(HW, priv, "Card is not up, ignoring rx mode change.\n"); + goto out; + } + if (!priv->port_up) { + en_dbg(HW, priv, "Port is down, ignoring rx mode change.\n"); + goto out; + } + if (!mlx4_en_QUERY_PORT(mdev, priv->port)) { + if (priv->port_state.link_state) { + priv->last_link_state = MLX4_DEV_EVENT_PORT_UP; + /* update netif baudrate */ + priv->dev->if_baudrate = + IF_Mbps(priv->port_state.link_speed); + /* Important note: the following call for if_link_state_change + * is needed for interface up scenario (start port, link state + * change) */ + if_link_state_change(priv->dev, LINK_STATE_UP); + en_dbg(HW, priv, "Link Up\n"); + } + } + + /* Promsicuous mode: disable all filters */ + if ((dev->if_flags & IFF_PROMISC) || + (priv->flags & MLX4_EN_FLAG_FORCE_PROMISC)) { + mlx4_en_set_promisc_mode(priv, mdev); + goto out; + } + + /* Not in promiscuous mode */ + if (priv->flags & MLX4_EN_FLAG_PROMISC) + mlx4_en_clear_promisc_mode(priv, mdev); + + mlx4_en_do_multicast(priv, dev, mdev); +out: + mutex_unlock(&mdev->state_lock); +} + +static void mlx4_en_watchdog_timeout(void *arg) +{ + struct mlx4_en_priv *priv = arg; + struct mlx4_en_dev *mdev = priv->mdev; + + en_dbg(DRV, priv, "Scheduling watchdog\n"); + queue_work(mdev->workqueue, &priv->watchdog_task); + if (priv->port_up) + callout_reset(&priv->watchdog_timer, MLX4_EN_WATCHDOG_TIMEOUT, + mlx4_en_watchdog_timeout, priv); +} + + + +static void mlx4_en_set_default_moderation(struct mlx4_en_priv *priv) +{ + struct mlx4_en_cq *cq; + int i; + + /* If we haven't received a specific coalescing setting + * (module param), we set the moderation parameters as follows: + * - moder_cnt is set to the number of mtu sized packets to + * satisfy our coalescing target. + * - moder_time is set to a fixed value. + */ + priv->rx_frames = MLX4_EN_RX_COAL_TARGET; + priv->rx_usecs = MLX4_EN_RX_COAL_TIME; + priv->tx_frames = MLX4_EN_TX_COAL_PKTS; + priv->tx_usecs = MLX4_EN_TX_COAL_TIME; + en_dbg(INTR, priv, "Default coalesing params for mtu: %u - " + "rx_frames:%d rx_usecs:%d\n", + (unsigned)priv->dev->if_mtu, priv->rx_frames, priv->rx_usecs); + + /* Setup cq moderation params */ + for (i = 0; i < priv->rx_ring_num; i++) { + cq = priv->rx_cq[i]; + cq->moder_cnt = priv->rx_frames; + cq->moder_time = priv->rx_usecs; + priv->last_moder_time[i] = MLX4_EN_AUTO_CONF; + priv->last_moder_packets[i] = 0; + priv->last_moder_bytes[i] = 0; + } + + for (i = 0; i < priv->tx_ring_num; i++) { + cq = priv->tx_cq[i]; + cq->moder_cnt = priv->tx_frames; + cq->moder_time = priv->tx_usecs; + } + + /* Reset auto-moderation params */ + priv->pkt_rate_low = MLX4_EN_RX_RATE_LOW; + priv->rx_usecs_low = MLX4_EN_RX_COAL_TIME_LOW; + priv->pkt_rate_high = MLX4_EN_RX_RATE_HIGH; + priv->rx_usecs_high = MLX4_EN_RX_COAL_TIME_HIGH; + priv->sample_interval = MLX4_EN_SAMPLE_INTERVAL; + priv->adaptive_rx_coal = 1; + priv->last_moder_jiffies = 0; + priv->last_moder_tx_packets = 0; +} + +static void mlx4_en_auto_moderation(struct mlx4_en_priv *priv) +{ + unsigned long period = (unsigned long) (jiffies - priv->last_moder_jiffies); + struct mlx4_en_cq *cq; + unsigned long packets; + unsigned long rate; + unsigned long avg_pkt_size; + unsigned long rx_packets; + unsigned long rx_bytes; + unsigned long rx_pkt_diff; + int moder_time; + int ring, err; + + if (!priv->adaptive_rx_coal || period < priv->sample_interval * HZ) + return; + + for (ring = 0; ring < priv->rx_ring_num; ring++) { + spin_lock(&priv->stats_lock); + rx_packets = priv->rx_ring[ring]->packets; + rx_bytes = priv->rx_ring[ring]->bytes; + spin_unlock(&priv->stats_lock); + + rx_pkt_diff = ((unsigned long) (rx_packets - + priv->last_moder_packets[ring])); + packets = rx_pkt_diff; + rate = packets * HZ / period; + avg_pkt_size = packets ? ((unsigned long) (rx_bytes - + priv->last_moder_bytes[ring])) / packets : 0; + + /* Apply auto-moderation only when packet rate + * exceeds a rate that it matters */ + if (rate > (MLX4_EN_RX_RATE_THRESH / priv->rx_ring_num) && + avg_pkt_size > MLX4_EN_AVG_PKT_SMALL) { + if (rate < priv->pkt_rate_low) + moder_time = priv->rx_usecs_low; + else if (rate > priv->pkt_rate_high) + moder_time = priv->rx_usecs_high; + else + moder_time = (rate - priv->pkt_rate_low) * + (priv->rx_usecs_high - priv->rx_usecs_low) / + (priv->pkt_rate_high - priv->pkt_rate_low) + + priv->rx_usecs_low; + } else { + moder_time = priv->rx_usecs_low; + } + + if (moder_time != priv->last_moder_time[ring]) { + priv->last_moder_time[ring] = moder_time; + cq = priv->rx_cq[ring]; + cq->moder_time = moder_time; + cq->moder_cnt = priv->rx_frames; + err = mlx4_en_set_cq_moder(priv, cq); + if (err) + en_err(priv, "Failed modifying moderation for cq:%d\n", + ring); + } + priv->last_moder_packets[ring] = rx_packets; + priv->last_moder_bytes[ring] = rx_bytes; + } + + priv->last_moder_jiffies = jiffies; +} + +static void mlx4_en_do_get_stats(struct work_struct *work) +{ + struct delayed_work *delay = to_delayed_work(work); + struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv, + stats_task); + struct mlx4_en_dev *mdev = priv->mdev; + int err; + + mutex_lock(&mdev->state_lock); + if (mdev->device_up) { + if (priv->port_up) { + if (mlx4_is_slave(mdev->dev)) + err = mlx4_en_get_vport_stats(mdev, priv->port); + else + err = mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 0); + if (err) + en_dbg(HW, priv, "Could not update stats\n"); + + mlx4_en_auto_moderation(priv); + } + + queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY); + } + mutex_unlock(&mdev->state_lock); +} + +/* mlx4_en_service_task - Run service task for tasks that needed to be done + * periodically + */ +static void mlx4_en_service_task(struct work_struct *work) +{ + struct delayed_work *delay = to_delayed_work(work); + struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv, + service_task); + struct mlx4_en_dev *mdev = priv->mdev; + + mutex_lock(&mdev->state_lock); + if (mdev->device_up) { + queue_delayed_work(mdev->workqueue, &priv->service_task, + SERVICE_TASK_DELAY); + } + mutex_unlock(&mdev->state_lock); +} + +static void mlx4_en_linkstate(struct work_struct *work) +{ + struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, + linkstate_task); + struct mlx4_en_dev *mdev = priv->mdev; + int linkstate = priv->link_state; + + mutex_lock(&mdev->state_lock); + /* If observable port state changed set carrier state and + * report to system log */ + if (priv->last_link_state != linkstate) { + if (linkstate == MLX4_DEV_EVENT_PORT_DOWN) { + en_info(priv, "Link Down\n"); + if_link_state_change(priv->dev, LINK_STATE_DOWN); + /* update netif baudrate */ + priv->dev->if_baudrate = 0; + + /* make sure the port is up before notifying the OS. + * This is tricky since we get here on INIT_PORT and + * in such case we can't tell the OS the port is up. + * To solve this there is a call to if_link_state_change + * in set_rx_mode. + * */ + } else if (priv->port_up && (linkstate == MLX4_DEV_EVENT_PORT_UP)){ + if (mlx4_en_QUERY_PORT(priv->mdev, priv->port)) + en_info(priv, "Query port failed\n"); + priv->dev->if_baudrate = + IF_Mbps(priv->port_state.link_speed); + en_info(priv, "Link Up\n"); + if_link_state_change(priv->dev, LINK_STATE_UP); + } + } + priv->last_link_state = linkstate; + mutex_unlock(&mdev->state_lock); +} + + +int mlx4_en_start_port(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_cq *cq; + struct mlx4_en_tx_ring *tx_ring; + int rx_index = 0; + int tx_index = 0; + int err = 0; + int i; + int j; + u8 mc_list[16] = {0}; + + + if (priv->port_up) { + en_dbg(DRV, priv, "start port called while port already up\n"); + return 0; + } + + INIT_LIST_HEAD(&priv->mc_list); + INIT_LIST_HEAD(&priv->uc_list); + INIT_LIST_HEAD(&priv->curr_mc_list); + INIT_LIST_HEAD(&priv->curr_uc_list); + INIT_LIST_HEAD(&priv->ethtool_list); + + /* Calculate Rx buf size */ + dev->if_mtu = min(dev->if_mtu, priv->max_mtu); + mlx4_en_calc_rx_buf(dev); + en_dbg(DRV, priv, "Rx buf size:%d\n", priv->rx_mb_size); + + /* Configure rx cq's and rings */ + err = mlx4_en_activate_rx_rings(priv); + if (err) { + en_err(priv, "Failed to activate RX rings\n"); + return err; + } + for (i = 0; i < priv->rx_ring_num; i++) { + cq = priv->rx_cq[i]; + + mlx4_en_cq_init_lock(cq); + err = mlx4_en_activate_cq(priv, cq, i); + if (err) { + en_err(priv, "Failed activating Rx CQ\n"); + goto cq_err; + } + for (j = 0; j < cq->size; j++) + cq->buf[j].owner_sr_opcode = MLX4_CQE_OWNER_MASK; + err = mlx4_en_set_cq_moder(priv, cq); + if (err) { + en_err(priv, "Failed setting cq moderation parameters"); + mlx4_en_deactivate_cq(priv, cq); + goto cq_err; + } + mlx4_en_arm_cq(priv, cq); + priv->rx_ring[i]->cqn = cq->mcq.cqn; + ++rx_index; + } + + /* Set qp number */ + en_dbg(DRV, priv, "Getting qp number for port %d\n", priv->port); + err = mlx4_en_get_qp(priv); + if (err) { + en_err(priv, "Failed getting eth qp\n"); + goto cq_err; + } + mdev->mac_removed[priv->port] = 0; + + priv->counter_index = + mlx4_get_default_counter_index(mdev->dev, priv->port); + + err = mlx4_en_config_rss_steer(priv); + if (err) { + en_err(priv, "Failed configuring rss steering\n"); + goto mac_err; + } + + err = mlx4_en_create_drop_qp(priv); + if (err) + goto rss_err; + + /* Configure tx cq's and rings */ + for (i = 0; i < priv->tx_ring_num; i++) { + /* Configure cq */ + cq = priv->tx_cq[i]; + err = mlx4_en_activate_cq(priv, cq, i); + if (err) { + en_err(priv, "Failed activating Tx CQ\n"); + goto tx_err; + } + err = mlx4_en_set_cq_moder(priv, cq); + if (err) { + en_err(priv, "Failed setting cq moderation parameters"); + mlx4_en_deactivate_cq(priv, cq); + goto tx_err; + } + en_dbg(DRV, priv, "Resetting index of collapsed CQ:%d to -1\n", i); + cq->buf->wqe_index = cpu_to_be16(0xffff); + + /* Configure ring */ + tx_ring = priv->tx_ring[i]; + + err = mlx4_en_activate_tx_ring(priv, tx_ring, cq->mcq.cqn, + i / priv->num_tx_rings_p_up); + if (err) { + en_err(priv, "Failed activating Tx ring %d\n", i); + mlx4_en_deactivate_cq(priv, cq); + goto tx_err; + } + + /* Arm CQ for TX completions */ + mlx4_en_arm_cq(priv, cq); + + /* Set initial ownership of all Tx TXBBs to SW (1) */ + for (j = 0; j < tx_ring->buf_size; j += STAMP_STRIDE) + *((u32 *) (tx_ring->buf + j)) = INIT_OWNER_BIT; + ++tx_index; + } + + /* Configure port */ + err = mlx4_SET_PORT_general(mdev->dev, priv->port, + priv->rx_mb_size, + priv->prof->tx_pause, + priv->prof->tx_ppp, + priv->prof->rx_pause, + priv->prof->rx_ppp); + if (err) { + en_err(priv, "Failed setting port general configurations for port %d, with error %d\n", + priv->port, err); + goto tx_err; + } + /* Set default qp number */ + err = mlx4_SET_PORT_qpn_calc(mdev->dev, priv->port, priv->base_qpn, 0); + if (err) { + en_err(priv, "Failed setting default qp numbers\n"); + goto tx_err; + } + + /* Init port */ + en_dbg(HW, priv, "Initializing port\n"); + err = mlx4_INIT_PORT(mdev->dev, priv->port); + if (err) { + en_err(priv, "Failed Initializing port\n"); + goto tx_err; + } + + /* Attach rx QP to bradcast address */ + memset(&mc_list[10], 0xff, ETH_ALEN); + mc_list[5] = priv->port; /* needed for B0 steering support */ + if (mlx4_multicast_attach(mdev->dev, &priv->rss_map.indir_qp, mc_list, + priv->port, 0, MLX4_PROT_ETH, + &priv->broadcast_id)) + mlx4_warn(mdev, "Failed Attaching Broadcast\n"); + + /* Must redo promiscuous mode setup. */ + priv->flags &= ~(MLX4_EN_FLAG_PROMISC | MLX4_EN_FLAG_MC_PROMISC); + + /* Schedule multicast task to populate multicast list */ + queue_work(mdev->workqueue, &priv->rx_mode_task); + + priv->port_up = true; + + /* Enable the queues. */ + dev->if_drv_flags &= ~IFF_DRV_OACTIVE; + dev->if_drv_flags |= IFF_DRV_RUNNING; +#ifdef CONFIG_DEBUG_FS + mlx4_en_create_debug_files(priv); +#endif + callout_reset(&priv->watchdog_timer, MLX4_EN_WATCHDOG_TIMEOUT, + mlx4_en_watchdog_timeout, priv); + + + return 0; + +tx_err: + while (tx_index--) { + mlx4_en_deactivate_tx_ring(priv, priv->tx_ring[tx_index]); + mlx4_en_deactivate_cq(priv, priv->tx_cq[tx_index]); + } + mlx4_en_destroy_drop_qp(priv); +rss_err: + mlx4_en_release_rss_steer(priv); +mac_err: + mlx4_en_put_qp(priv); +cq_err: + while (rx_index--) + mlx4_en_deactivate_cq(priv, priv->rx_cq[rx_index]); + for (i = 0; i < priv->rx_ring_num; i++) + mlx4_en_deactivate_rx_ring(priv, priv->rx_ring[i]); + + return err; /* need to close devices */ +} + + +void mlx4_en_stop_port(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_addr_list *addr_list, *tmp; + int i; + u8 mc_list[16] = {0}; + + if (!priv->port_up) { + en_dbg(DRV, priv, "stop port called while port already down\n"); + return; + } + +#ifdef CONFIG_DEBUG_FS + mlx4_en_delete_debug_files(priv); +#endif + + /* close port*/ + mlx4_CLOSE_PORT(mdev->dev, priv->port); + + /* Set port as not active */ + priv->port_up = false; + priv->counter_index = MLX4_SINK_COUNTER_INDEX(mdev->dev); + + /* Promsicuous mode */ + if (mdev->dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) { + priv->flags &= ~(MLX4_EN_FLAG_PROMISC | + MLX4_EN_FLAG_MC_PROMISC); + mlx4_flow_steer_promisc_remove(mdev->dev, + priv->port, + MLX4_FS_ALL_DEFAULT); + mlx4_flow_steer_promisc_remove(mdev->dev, + priv->port, + MLX4_FS_MC_DEFAULT); + } else if (priv->flags & MLX4_EN_FLAG_PROMISC) { + priv->flags &= ~MLX4_EN_FLAG_PROMISC; + + /* Disable promiscouos mode */ + mlx4_unicast_promisc_remove(mdev->dev, priv->base_qpn, + priv->port); + + /* Disable Multicast promisc */ + if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { + mlx4_multicast_promisc_remove(mdev->dev, priv->base_qpn, + priv->port); + priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; + } + } + + /* Detach All unicasts */ + list_for_each_entry(addr_list, &priv->curr_uc_list, list) { + mlx4_en_uc_steer_release(priv, addr_list->addr, + priv->rss_map.indir_qp.qpn, + addr_list->reg_id); + } + mlx4_en_clear_uclist(dev); + list_for_each_entry_safe(addr_list, tmp, &priv->curr_uc_list, list) { + list_del(&addr_list->list); + kfree(addr_list); + } + + /* Detach All multicasts */ + memset(&mc_list[10], 0xff, ETH_ALEN); + mc_list[5] = priv->port; /* needed for B0 steering support */ + mlx4_multicast_detach(mdev->dev, &priv->rss_map.indir_qp, mc_list, + MLX4_PROT_ETH, priv->broadcast_id); + list_for_each_entry(addr_list, &priv->curr_mc_list, list) { + memcpy(&mc_list[10], addr_list->addr, ETH_ALEN); + mc_list[5] = priv->port; + mlx4_multicast_detach(mdev->dev, &priv->rss_map.indir_qp, + mc_list, MLX4_PROT_ETH, addr_list->reg_id); + } + mlx4_en_clear_mclist(dev); + list_for_each_entry_safe(addr_list, tmp, &priv->curr_mc_list, list) { + list_del(&addr_list->list); + kfree(addr_list); + } + + /* Flush multicast filter */ + mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 1, MLX4_MCAST_CONFIG); + mlx4_en_destroy_drop_qp(priv); + + /* Free TX Rings */ + for (i = 0; i < priv->tx_ring_num; i++) { + mlx4_en_deactivate_tx_ring(priv, priv->tx_ring[i]); + mlx4_en_deactivate_cq(priv, priv->tx_cq[i]); + } + msleep(10); + + for (i = 0; i < priv->tx_ring_num; i++) + mlx4_en_free_tx_buf(dev, priv->tx_ring[i]); + + /* Free RSS qps */ + mlx4_en_release_rss_steer(priv); + + /* Unregister Mac address for the port */ + mlx4_en_put_qp(priv); + mdev->mac_removed[priv->port] = 1; + + /* Free RX Rings */ + for (i = 0; i < priv->rx_ring_num; i++) { + struct mlx4_en_cq *cq = priv->rx_cq[i]; + mlx4_en_deactivate_rx_ring(priv, priv->rx_ring[i]); + mlx4_en_deactivate_cq(priv, cq); + } + + callout_stop(&priv->watchdog_timer); + + dev->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); +} + +static void mlx4_en_restart(struct work_struct *work) +{ + struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, + watchdog_task); + struct mlx4_en_dev *mdev = priv->mdev; + struct net_device *dev = priv->dev; + struct mlx4_en_tx_ring *ring; + int i; + + + if (priv->blocked == 0 || priv->port_up == 0) + return; + for (i = 0; i < priv->tx_ring_num; i++) { + ring = priv->tx_ring[i]; + if (ring->blocked && + ring->watchdog_time + MLX4_EN_WATCHDOG_TIMEOUT < ticks) + goto reset; + } + return; + +reset: + priv->port_stats.tx_timeout++; + en_dbg(DRV, priv, "Watchdog task called for port %d\n", priv->port); + + mutex_lock(&mdev->state_lock); + if (priv->port_up) { + mlx4_en_stop_port(dev); + //for (i = 0; i < priv->tx_ring_num; i++) + // netdev_tx_reset_queue(priv->tx_ring[i]->tx_queue); + if (mlx4_en_start_port(dev)) + en_err(priv, "Failed restarting port %d\n", priv->port); + } + mutex_unlock(&mdev->state_lock); +} + +static void mlx4_en_clear_stats(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int i; + + if (!mlx4_is_slave(mdev->dev)) + if (mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 1)) + en_dbg(HW, priv, "Failed dumping statistics\n"); + + memset(&priv->pstats, 0, sizeof(priv->pstats)); + memset(&priv->pkstats, 0, sizeof(priv->pkstats)); + memset(&priv->port_stats, 0, sizeof(priv->port_stats)); + memset(&priv->vport_stats, 0, sizeof(priv->vport_stats)); + + for (i = 0; i < priv->tx_ring_num; i++) { + priv->tx_ring[i]->bytes = 0; + priv->tx_ring[i]->packets = 0; + priv->tx_ring[i]->tx_csum = 0; + priv->tx_ring[i]->oversized_packets = 0; + } + for (i = 0; i < priv->rx_ring_num; i++) { + priv->rx_ring[i]->bytes = 0; + priv->rx_ring[i]->packets = 0; + priv->rx_ring[i]->csum_ok = 0; + priv->rx_ring[i]->csum_none = 0; + } +} + +static void mlx4_en_open(void* arg) +{ + + struct mlx4_en_priv *priv; + struct mlx4_en_dev *mdev; + struct net_device *dev; + int err = 0; + + priv = arg; + mdev = priv->mdev; + dev = priv->dev; + + + mutex_lock(&mdev->state_lock); + + if (!mdev->device_up) { + en_err(priv, "Cannot open - device down/disabled\n"); + goto out; + } + + /* Reset HW statistics and SW counters */ + mlx4_en_clear_stats(dev); + + err = mlx4_en_start_port(dev); + if (err) + en_err(priv, "Failed starting port:%d\n", priv->port); + +out: + mutex_unlock(&mdev->state_lock); + return; +} + +void mlx4_en_free_resources(struct mlx4_en_priv *priv) +{ + int i; + +#ifdef CONFIG_RFS_ACCEL + if (priv->dev->rx_cpu_rmap) { + free_irq_cpu_rmap(priv->dev->rx_cpu_rmap); + priv->dev->rx_cpu_rmap = NULL; + } +#endif + + for (i = 0; i < priv->tx_ring_num; i++) { + if (priv->tx_ring && priv->tx_ring[i]) + mlx4_en_destroy_tx_ring(priv, &priv->tx_ring[i]); + if (priv->tx_cq && priv->tx_cq[i]) + mlx4_en_destroy_cq(priv, &priv->tx_cq[i]); + } + + for (i = 0; i < priv->rx_ring_num; i++) { + if (priv->rx_ring[i]) + mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i], + priv->prof->rx_ring_size, priv->stride); + if (priv->rx_cq[i]) + mlx4_en_destroy_cq(priv, &priv->rx_cq[i]); + } + + if (priv->stat_sysctl != NULL) + sysctl_ctx_free(&priv->stat_ctx); +} + +int mlx4_en_alloc_resources(struct mlx4_en_priv *priv) +{ + struct mlx4_en_port_profile *prof = priv->prof; + int i; + int node = 0; + + /* Create rx Rings */ + for (i = 0; i < priv->rx_ring_num; i++) { + if (mlx4_en_create_cq(priv, &priv->rx_cq[i], + prof->rx_ring_size, i, RX, node)) + goto err; + + if (mlx4_en_create_rx_ring(priv, &priv->rx_ring[i], + prof->rx_ring_size, node)) + goto err; + } + + /* Create tx Rings */ + for (i = 0; i < priv->tx_ring_num; i++) { + if (mlx4_en_create_cq(priv, &priv->tx_cq[i], + prof->tx_ring_size, i, TX, node)) + goto err; + + if (mlx4_en_create_tx_ring(priv, &priv->tx_ring[i], + prof->tx_ring_size, TXBB_SIZE, node, i)) + goto err; + } + +#ifdef CONFIG_RFS_ACCEL + priv->dev->rx_cpu_rmap = alloc_irq_cpu_rmap(priv->rx_ring_num); + if (!priv->dev->rx_cpu_rmap) + goto err; +#endif + /* Re-create stat sysctls in case the number of rings changed. */ + mlx4_en_sysctl_stat(priv); + return 0; + +err: + en_err(priv, "Failed to allocate NIC resources\n"); + for (i = 0; i < priv->rx_ring_num; i++) { + if (priv->rx_ring[i]) + mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i], + prof->rx_ring_size, + priv->stride); + if (priv->rx_cq[i]) + mlx4_en_destroy_cq(priv, &priv->rx_cq[i]); + } + for (i = 0; i < priv->tx_ring_num; i++) { + if (priv->tx_ring[i]) + mlx4_en_destroy_tx_ring(priv, &priv->tx_ring[i]); + if (priv->tx_cq[i]) + mlx4_en_destroy_cq(priv, &priv->tx_cq[i]); + } + priv->port_up = false; + return -ENOMEM; +} + +struct en_port_attribute { + struct attribute attr; + ssize_t (*show)(struct en_port *, struct en_port_attribute *, char *buf); + ssize_t (*store)(struct en_port *, struct en_port_attribute *, char *buf, size_t count); +}; + +#define PORT_ATTR_RO(_name) \ +struct en_port_attribute en_port_attr_##_name = __ATTR_RO(_name) + +#define EN_PORT_ATTR(_name, _mode, _show, _store) \ +struct en_port_attribute en_port_attr_##_name = __ATTR(_name, _mode, _show, _store) + +void mlx4_en_destroy_netdev(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + + en_dbg(DRV, priv, "Destroying netdev on port:%d\n", priv->port); + + /* don't allow more IOCTLs */ + priv->gone = 1; + + /* XXX wait a bit to allow IOCTL handlers to complete */ + pause("W", hz); + + if (priv->vlan_attach != NULL) + EVENTHANDLER_DEREGISTER(vlan_config, priv->vlan_attach); + if (priv->vlan_detach != NULL) + EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach); + + /* Unregister device - this will close the port if it was up */ + if (priv->registered) { + mutex_lock(&mdev->state_lock); + ether_ifdetach(dev); + mutex_unlock(&mdev->state_lock); + } + + mutex_lock(&mdev->state_lock); + mlx4_en_stop_port(dev); + mutex_unlock(&mdev->state_lock); + + if (priv->allocated) + mlx4_free_hwq_res(mdev->dev, &priv->res, MLX4_EN_PAGE_SIZE); + + cancel_delayed_work(&priv->stats_task); + cancel_delayed_work(&priv->service_task); + /* flush any pending task for this netdev */ + flush_workqueue(mdev->workqueue); + callout_drain(&priv->watchdog_timer); + + /* Detach the netdev so tasks would not attempt to access it */ + mutex_lock(&mdev->state_lock); + mdev->pndev[priv->port] = NULL; + mutex_unlock(&mdev->state_lock); + + + mlx4_en_free_resources(priv); + + /* freeing the sysctl conf cannot be called from within mlx4_en_free_resources */ + if (priv->conf_sysctl != NULL) + sysctl_ctx_free(&priv->conf_ctx); + + kfree(priv->tx_ring); + kfree(priv->tx_cq); + + kfree(priv); + if_free(dev); + +} + +static int mlx4_en_change_mtu(struct net_device *dev, int new_mtu) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int err = 0; + + en_dbg(DRV, priv, "Change MTU called - current:%u new:%u\n", + (unsigned)dev->if_mtu, (unsigned)new_mtu); + + if ((new_mtu < MLX4_EN_MIN_MTU) || (new_mtu > priv->max_mtu)) { + en_err(priv, "Bad MTU size:%d, max %u.\n", new_mtu, + priv->max_mtu); + return -EPERM; + } + mutex_lock(&mdev->state_lock); + dev->if_mtu = new_mtu; + if (dev->if_drv_flags & IFF_DRV_RUNNING) { + if (!mdev->device_up) { + /* NIC is probably restarting - let watchdog task reset + * * the port */ + en_dbg(DRV, priv, "Change MTU called with card down!?\n"); + } else { + mlx4_en_stop_port(dev); + err = mlx4_en_start_port(dev); + if (err) { + en_err(priv, "Failed restarting port:%d\n", + priv->port); + queue_work(mdev->workqueue, &priv->watchdog_task); + } + } + } + mutex_unlock(&mdev->state_lock); + return 0; +} + +static int mlx4_en_calc_media(struct mlx4_en_priv *priv) +{ + int trans_type; + int active; + + active = IFM_ETHER; + if (priv->last_link_state == MLX4_DEV_EVENT_PORT_DOWN) + return (active); + active |= IFM_FDX; + trans_type = priv->port_state.transceiver; + /* XXX I don't know all of the transceiver values. */ + switch (priv->port_state.link_speed) { + case 100: + active |= IFM_100_T; + break; + case 1000: + active |= IFM_1000_T; + break; + case 10000: + if (trans_type > 0 && trans_type <= 0xC) + active |= IFM_10G_SR; + else if (trans_type == 0x80 || trans_type == 0) + active |= IFM_10G_CX4; + break; + case 40000: + active |= IFM_40G_CR4; + break; + } + if (priv->prof->tx_pause) + active |= IFM_ETH_TXPAUSE; + if (priv->prof->rx_pause) + active |= IFM_ETH_RXPAUSE; + + return (active); +} + +static void mlx4_en_media_status(struct ifnet *dev, struct ifmediareq *ifmr) +{ + struct mlx4_en_priv *priv; + + priv = dev->if_softc; + ifmr->ifm_status = IFM_AVALID; + if (priv->last_link_state != MLX4_DEV_EVENT_PORT_DOWN) + ifmr->ifm_status |= IFM_ACTIVE; + ifmr->ifm_active = mlx4_en_calc_media(priv); + + return; +} + +static int mlx4_en_media_change(struct ifnet *dev) +{ + struct mlx4_en_priv *priv; + struct ifmedia *ifm; + int rxpause; + int txpause; + int error; + + priv = dev->if_softc; + ifm = &priv->media; + rxpause = txpause = 0; + error = 0; + + if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) + return (EINVAL); + switch (IFM_SUBTYPE(ifm->ifm_media)) { + case IFM_AUTO: + break; + case IFM_10G_SR: + case IFM_10G_CX4: + case IFM_1000_T: + case IFM_40G_CR4: + if ((IFM_SUBTYPE(ifm->ifm_media) + == IFM_SUBTYPE(mlx4_en_calc_media(priv))) + && (ifm->ifm_media & IFM_FDX)) + break; + /* Fallthrough */ + default: + printf("%s: Only auto media type\n", if_name(dev)); + return (EINVAL); + } + /* Allow user to set/clear pause */ + if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_RXPAUSE) + rxpause = 1; + if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_TXPAUSE) + txpause = 1; + if (priv->prof->tx_pause != txpause || priv->prof->rx_pause != rxpause) { + priv->prof->tx_pause = txpause; + priv->prof->rx_pause = rxpause; + error = -mlx4_SET_PORT_general(priv->mdev->dev, priv->port, + priv->rx_mb_size + ETHER_CRC_LEN, priv->prof->tx_pause, + priv->prof->tx_ppp, priv->prof->rx_pause, + priv->prof->rx_ppp); + } + return (error); +} + +static int mlx4_en_ioctl(struct ifnet *dev, u_long command, caddr_t data) +{ + struct mlx4_en_priv *priv; + struct mlx4_en_dev *mdev; + struct ifreq *ifr; + int error; + int mask; + struct ifrsskey *ifrk; + const u32 *key; + struct ifrsshash *ifrh; + u8 rss_mask; + + error = 0; + mask = 0; + priv = dev->if_softc; + + /* check if detaching */ + if (priv == NULL || priv->gone != 0) + return (ENXIO); + + mdev = priv->mdev; + ifr = (struct ifreq *) data; + + switch (command) { + case SIOCSIFMTU: + error = -mlx4_en_change_mtu(dev, ifr->ifr_mtu); + break; + case SIOCSIFFLAGS: + if (dev->if_flags & IFF_UP) { + if ((dev->if_drv_flags & IFF_DRV_RUNNING) == 0) { + mutex_lock(&mdev->state_lock); + mlx4_en_start_port(dev); + mutex_unlock(&mdev->state_lock); + } else { + mlx4_en_set_rx_mode(dev); + } + } else { + mutex_lock(&mdev->state_lock); + if (dev->if_drv_flags & IFF_DRV_RUNNING) { + mlx4_en_stop_port(dev); + if_link_state_change(dev, LINK_STATE_DOWN); + } + mutex_unlock(&mdev->state_lock); + } + break; + case SIOCADDMULTI: + case SIOCDELMULTI: + mlx4_en_set_rx_mode(dev); + break; + case SIOCSIFMEDIA: + case SIOCGIFMEDIA: + error = ifmedia_ioctl(dev, ifr, &priv->media, command); + break; + case SIOCSIFCAP: + mutex_lock(&mdev->state_lock); + mask = ifr->ifr_reqcap ^ dev->if_capenable; + if (mask & IFCAP_TXCSUM) { + dev->if_capenable ^= IFCAP_TXCSUM; + dev->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); + + if (IFCAP_TSO4 & dev->if_capenable && + !(IFCAP_TXCSUM & dev->if_capenable)) { + dev->if_capenable &= ~IFCAP_TSO4; + dev->if_hwassist &= ~CSUM_IP_TSO; + if_printf(dev, + "tso4 disabled due to -txcsum.\n"); + } + } + if (mask & IFCAP_TXCSUM_IPV6) { + dev->if_capenable ^= IFCAP_TXCSUM_IPV6; + dev->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); + + if (IFCAP_TSO6 & dev->if_capenable && + !(IFCAP_TXCSUM_IPV6 & dev->if_capenable)) { + dev->if_capenable &= ~IFCAP_TSO6; + dev->if_hwassist &= ~CSUM_IP6_TSO; + if_printf(dev, + "tso6 disabled due to -txcsum6.\n"); + } + } + if (mask & IFCAP_RXCSUM) + dev->if_capenable ^= IFCAP_RXCSUM; + if (mask & IFCAP_RXCSUM_IPV6) + dev->if_capenable ^= IFCAP_RXCSUM_IPV6; + + if (mask & IFCAP_TSO4) { + if (!(IFCAP_TSO4 & dev->if_capenable) && + !(IFCAP_TXCSUM & dev->if_capenable)) { + if_printf(dev, "enable txcsum first.\n"); + error = EAGAIN; + goto out; + } + dev->if_capenable ^= IFCAP_TSO4; + dev->if_hwassist ^= CSUM_IP_TSO; + } + if (mask & IFCAP_TSO6) { + if (!(IFCAP_TSO6 & dev->if_capenable) && + !(IFCAP_TXCSUM_IPV6 & dev->if_capenable)) { + if_printf(dev, "enable txcsum6 first.\n"); + error = EAGAIN; + goto out; + } + dev->if_capenable ^= IFCAP_TSO6; + dev->if_hwassist ^= CSUM_IP6_TSO; + } + if (mask & IFCAP_LRO) + dev->if_capenable ^= IFCAP_LRO; + if (mask & IFCAP_VLAN_HWTAGGING) + dev->if_capenable ^= IFCAP_VLAN_HWTAGGING; + if (mask & IFCAP_VLAN_HWFILTER) + dev->if_capenable ^= IFCAP_VLAN_HWFILTER; + if (mask & IFCAP_WOL_MAGIC) + dev->if_capenable ^= IFCAP_WOL_MAGIC; + if (dev->if_drv_flags & IFF_DRV_RUNNING) + mlx4_en_start_port(dev); +out: + mutex_unlock(&mdev->state_lock); + VLAN_CAPABILITIES(dev); + break; +#if __FreeBSD_version >= 1100036 + case SIOCGI2C: { + struct ifi2creq i2c; + + error = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); + if (error) + break; + if (i2c.len > sizeof(i2c.data)) { + error = EINVAL; + break; + } + /* + * Note that we ignore i2c.addr here. The driver hardcodes + * the address to 0x50, while standard expects it to be 0xA0. + */ + error = mlx4_get_module_info(mdev->dev, priv->port, + i2c.offset, i2c.len, i2c.data); + if (error < 0) { + error = -error; + break; + } + error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); + break; + } +#endif + case SIOCGIFRSSKEY: + ifrk = (struct ifrsskey *)data; + ifrk->ifrk_func = RSS_FUNC_TOEPLITZ; + mutex_lock(&mdev->state_lock); + key = mlx4_en_get_rss_key(priv, &ifrk->ifrk_keylen); + if (ifrk->ifrk_keylen > RSS_KEYLEN) + error = EINVAL; + else + memcpy(ifrk->ifrk_key, key, ifrk->ifrk_keylen); + mutex_unlock(&mdev->state_lock); + break; + + case SIOCGIFRSSHASH: + mutex_lock(&mdev->state_lock); + rss_mask = mlx4_en_get_rss_mask(priv); + mutex_unlock(&mdev->state_lock); + ifrh = (struct ifrsshash *)data; + ifrh->ifrh_func = RSS_FUNC_TOEPLITZ; + ifrh->ifrh_types = 0; + if (rss_mask & MLX4_RSS_IPV4) + ifrh->ifrh_types |= RSS_TYPE_IPV4; + if (rss_mask & MLX4_RSS_TCP_IPV4) + ifrh->ifrh_types |= RSS_TYPE_TCP_IPV4; + if (rss_mask & MLX4_RSS_IPV6) + ifrh->ifrh_types |= RSS_TYPE_IPV6; + if (rss_mask & MLX4_RSS_TCP_IPV6) + ifrh->ifrh_types |= RSS_TYPE_TCP_IPV6; + if (rss_mask & MLX4_RSS_UDP_IPV4) + ifrh->ifrh_types |= RSS_TYPE_UDP_IPV4; + if (rss_mask & MLX4_RSS_UDP_IPV6) + ifrh->ifrh_types |= RSS_TYPE_UDP_IPV6; + break; + + default: + error = ether_ioctl(dev, command, data); + break; + } + + return (error); +} + + +int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, + struct mlx4_en_port_profile *prof) +{ + struct net_device *dev; + struct mlx4_en_priv *priv; + uint8_t dev_addr[ETHER_ADDR_LEN]; + int err; + int i; + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + dev = priv->dev = if_alloc(IFT_ETHER); + if (dev == NULL) { + en_err(priv, "Net device allocation failed\n"); + kfree(priv); + return -ENOMEM; + } + dev->if_softc = priv; + if_initname(dev, "mlxen", (device_get_unit( + mdev->pdev->dev.bsddev) * MLX4_MAX_PORTS) + port - 1); + dev->if_mtu = ETHERMTU; + dev->if_init = mlx4_en_open; + dev->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; + dev->if_ioctl = mlx4_en_ioctl; + dev->if_transmit = mlx4_en_transmit; + dev->if_qflush = mlx4_en_qflush; + dev->if_snd.ifq_maxlen = prof->tx_ring_size; + + /* + * Initialize driver private data + */ + priv->counter_index = 0xff; + spin_lock_init(&priv->stats_lock); + INIT_WORK(&priv->rx_mode_task, mlx4_en_do_set_rx_mode); + INIT_WORK(&priv->watchdog_task, mlx4_en_restart); + INIT_WORK(&priv->linkstate_task, mlx4_en_linkstate); + INIT_DELAYED_WORK(&priv->stats_task, mlx4_en_do_get_stats); + INIT_DELAYED_WORK(&priv->service_task, mlx4_en_service_task); + callout_init(&priv->watchdog_timer, 1); +#ifdef CONFIG_RFS_ACCEL + INIT_LIST_HEAD(&priv->filters); + spin_lock_init(&priv->filters_lock); +#endif + + priv->msg_enable = MLX4_EN_MSG_LEVEL; + priv->dev = dev; + priv->mdev = mdev; + priv->ddev = &mdev->pdev->dev; + priv->prof = prof; + priv->port = port; + priv->port_up = false; + priv->flags = prof->flags; + + priv->num_tx_rings_p_up = mdev->profile.num_tx_rings_p_up; + priv->tx_ring_num = prof->tx_ring_num; + priv->tx_ring = kcalloc(MAX_TX_RINGS, + sizeof(struct mlx4_en_tx_ring *), GFP_KERNEL); + if (!priv->tx_ring) { + err = -ENOMEM; + goto out; + } + priv->tx_cq = kcalloc(sizeof(struct mlx4_en_cq *), MAX_TX_RINGS, + GFP_KERNEL); + if (!priv->tx_cq) { + err = -ENOMEM; + goto out; + } + + priv->rx_ring_num = prof->rx_ring_num; + priv->cqe_factor = (mdev->dev->caps.cqe_size == 64) ? 1 : 0; + priv->mac_index = -1; + priv->last_ifq_jiffies = 0; + priv->if_counters_rx_errors = 0; + priv->if_counters_rx_no_buffer = 0; +#ifdef CONFIG_MLX4_EN_DCB + if (!mlx4_is_slave(priv->mdev->dev)) { + priv->dcbx_cap = DCB_CAP_DCBX_HOST; + priv->flags |= MLX4_EN_FLAG_DCB_ENABLED; + if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETS_CFG) { + dev->dcbnl_ops = &mlx4_en_dcbnl_ops; + } else { + en_info(priv, "QoS disabled - no HW support\n"); + dev->dcbnl_ops = &mlx4_en_dcbnl_pfc_ops; + } + } +#endif + + /* Query for default mac and max mtu */ + priv->max_mtu = mdev->dev->caps.eth_mtu_cap[priv->port]; + priv->mac = mdev->dev->caps.def_mac[priv->port]; + if (ILLEGAL_MAC(priv->mac)) { +#if BITS_PER_LONG == 64 + en_err(priv, "Port: %d, invalid mac burned: 0x%lx, quiting\n", + priv->port, priv->mac); +#elif BITS_PER_LONG == 32 + en_err(priv, "Port: %d, invalid mac burned: 0x%llx, quiting\n", + priv->port, priv->mac); +#endif + err = -EINVAL; + goto out; + } + + priv->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + + DS_SIZE); + + mlx4_en_sysctl_conf(priv); + + err = mlx4_en_alloc_resources(priv); + if (err) + goto out; + + /* Allocate page for receive rings */ + err = mlx4_alloc_hwq_res(mdev->dev, &priv->res, + MLX4_EN_PAGE_SIZE, MLX4_EN_PAGE_SIZE); + if (err) { + en_err(priv, "Failed to allocate page for rx qps\n"); + goto out; + } + priv->allocated = 1; + + /* + * Set driver features + */ + dev->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6; + dev->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING; + dev->if_capabilities |= IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER; + dev->if_capabilities |= IFCAP_LINKSTATE | IFCAP_JUMBO_MTU; + dev->if_capabilities |= IFCAP_LRO; + dev->if_capabilities |= IFCAP_HWSTATS; + + if (mdev->LSO_support) + dev->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6 | IFCAP_VLAN_HWTSO; + +#if __FreeBSD_version >= 1100000 + /* set TSO limits so that we don't have to drop TX packets */ + dev->if_hw_tsomax = MLX4_EN_TX_MAX_PAYLOAD_SIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN) /* hdr */; + dev->if_hw_tsomaxsegcount = MLX4_EN_TX_MAX_MBUF_FRAGS - 1 /* hdr */; + dev->if_hw_tsomaxsegsize = MLX4_EN_TX_MAX_MBUF_SIZE; +#endif + + dev->if_capenable = dev->if_capabilities; + + dev->if_hwassist = 0; + if (dev->if_capenable & (IFCAP_TSO4 | IFCAP_TSO6)) + dev->if_hwassist |= CSUM_TSO; + if (dev->if_capenable & IFCAP_TXCSUM) + dev->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP); + if (dev->if_capenable & IFCAP_TXCSUM_IPV6) + dev->if_hwassist |= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); + + + /* Register for VLAN events */ + priv->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, + mlx4_en_vlan_rx_add_vid, priv, EVENTHANDLER_PRI_FIRST); + priv->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, + mlx4_en_vlan_rx_kill_vid, priv, EVENTHANDLER_PRI_FIRST); + + mdev->pndev[priv->port] = dev; + + priv->last_link_state = MLX4_DEV_EVENT_PORT_DOWN; + mlx4_en_set_default_moderation(priv); + + /* Set default MAC */ + for (i = 0; i < ETHER_ADDR_LEN; i++) + dev_addr[ETHER_ADDR_LEN - 1 - i] = (u8) (priv->mac >> (8 * i)); + + + ether_ifattach(dev, dev_addr); + if_link_state_change(dev, LINK_STATE_DOWN); + ifmedia_init(&priv->media, IFM_IMASK | IFM_ETH_FMASK, + mlx4_en_media_change, mlx4_en_media_status); + ifmedia_add(&priv->media, IFM_ETHER | IFM_FDX | IFM_1000_T, 0, NULL); + ifmedia_add(&priv->media, IFM_ETHER | IFM_FDX | IFM_10G_SR, 0, NULL); + ifmedia_add(&priv->media, IFM_ETHER | IFM_FDX | IFM_10G_CX4, 0, NULL); + ifmedia_add(&priv->media, IFM_ETHER | IFM_FDX | IFM_40G_CR4, 0, NULL); + ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO); + + en_warn(priv, "Using %d TX rings\n", prof->tx_ring_num); + en_warn(priv, "Using %d RX rings\n", prof->rx_ring_num); + + priv->registered = 1; + + en_warn(priv, "Using %d TX rings\n", prof->tx_ring_num); + en_warn(priv, "Using %d RX rings\n", prof->rx_ring_num); + + + priv->rx_mb_size = dev->if_mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN; + err = mlx4_SET_PORT_general(mdev->dev, priv->port, + priv->rx_mb_size, + prof->tx_pause, prof->tx_ppp, + prof->rx_pause, prof->rx_ppp); + if (err) { + en_err(priv, "Failed setting port general configurations " + "for port %d, with error %d\n", priv->port, err); + goto out; + } + + /* Init port */ + en_warn(priv, "Initializing port\n"); + err = mlx4_INIT_PORT(mdev->dev, priv->port); + if (err) { + en_err(priv, "Failed Initializing port\n"); + goto out; + } + + queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY); + + if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) + queue_delayed_work(mdev->workqueue, &priv->service_task, SERVICE_TASK_DELAY); + + return 0; + +out: + mlx4_en_destroy_netdev(dev); + return err; +} + +static int mlx4_en_set_ring_size(struct net_device *dev, + int rx_size, int tx_size) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int port_up = 0; + int err = 0; + + rx_size = roundup_pow_of_two(rx_size); + rx_size = max_t(u32, rx_size, MLX4_EN_MIN_RX_SIZE); + rx_size = min_t(u32, rx_size, MLX4_EN_MAX_RX_SIZE); + tx_size = roundup_pow_of_two(tx_size); + tx_size = max_t(u32, tx_size, MLX4_EN_MIN_TX_SIZE); + tx_size = min_t(u32, tx_size, MLX4_EN_MAX_TX_SIZE); + + if (rx_size == (priv->port_up ? + priv->rx_ring[0]->actual_size : priv->rx_ring[0]->size) && + tx_size == priv->tx_ring[0]->size) + return 0; + mutex_lock(&mdev->state_lock); + if (priv->port_up) { + port_up = 1; + mlx4_en_stop_port(dev); + } + mlx4_en_free_resources(priv); + priv->prof->tx_ring_size = tx_size; + priv->prof->rx_ring_size = rx_size; + err = mlx4_en_alloc_resources(priv); + if (err) { + en_err(priv, "Failed reallocating port resources\n"); + goto out; + } + if (port_up) { + err = mlx4_en_start_port(dev); + if (err) + en_err(priv, "Failed starting port\n"); + } +out: + mutex_unlock(&mdev->state_lock); + return err; +} +static int mlx4_en_set_rx_ring_size(SYSCTL_HANDLER_ARGS) +{ + struct mlx4_en_priv *priv; + int size; + int error; + + priv = arg1; + size = priv->prof->rx_ring_size; + error = sysctl_handle_int(oidp, &size, 0, req); + if (error || !req->newptr) + return (error); + error = -mlx4_en_set_ring_size(priv->dev, size, + priv->prof->tx_ring_size); + return (error); +} + +static int mlx4_en_set_tx_ring_size(SYSCTL_HANDLER_ARGS) +{ + struct mlx4_en_priv *priv; + int size; + int error; + + priv = arg1; + size = priv->prof->tx_ring_size; + error = sysctl_handle_int(oidp, &size, 0, req); + if (error || !req->newptr) + return (error); + error = -mlx4_en_set_ring_size(priv->dev, priv->prof->rx_ring_size, + size); + + return (error); +} + +static int mlx4_en_get_module_info(struct net_device *dev, + struct ethtool_modinfo *modinfo) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int ret; + u8 data[4]; + + /* Read first 2 bytes to get Module & REV ID */ + ret = mlx4_get_module_info(mdev->dev, priv->port, + 0/*offset*/, 2/*size*/, data); + + if (ret < 2) { + en_err(priv, "Failed to read eeprom module first two bytes, error: 0x%x\n", -ret); + return -EIO; + } + + switch (data[0] /* identifier */) { + case MLX4_MODULE_ID_QSFP: + modinfo->type = ETH_MODULE_SFF_8436; + modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN; + break; + case MLX4_MODULE_ID_QSFP_PLUS: + if (data[1] >= 0x3) { /* revision id */ + modinfo->type = ETH_MODULE_SFF_8636; + modinfo->eeprom_len = ETH_MODULE_SFF_8636_LEN; + } else { + modinfo->type = ETH_MODULE_SFF_8436; + modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN; + } + break; + case MLX4_MODULE_ID_QSFP28: + modinfo->type = ETH_MODULE_SFF_8636; + modinfo->eeprom_len = ETH_MODULE_SFF_8636_LEN; + break; + case MLX4_MODULE_ID_SFP: + modinfo->type = ETH_MODULE_SFF_8472; + modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN; + break; + default: + en_err(priv, "mlx4_en_get_module_info : Not recognized cable type\n"); + return -EINVAL; + } + + return 0; +} + +static int mlx4_en_get_module_eeprom(struct net_device *dev, + struct ethtool_eeprom *ee, + u8 *data) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int offset = ee->offset; + int i = 0, ret; + + if (ee->len == 0) + return -EINVAL; + + memset(data, 0, ee->len); + + while (i < ee->len) { + en_dbg(DRV, priv, + "mlx4_get_module_info i(%d) offset(%d) len(%d)\n", + i, offset, ee->len - i); + + ret = mlx4_get_module_info(mdev->dev, priv->port, + offset, ee->len - i, data + i); + + if (!ret) /* Done reading */ + return 0; + + if (ret < 0) { + en_err(priv, + "mlx4_get_module_info i(%d) offset(%d) bytes_to_read(%d) - FAILED (0x%x)\n", + i, offset, ee->len - i, ret); + return -1; + } + + i += ret; + offset += ret; + } + return 0; +} + +static void mlx4_en_print_eeprom(u8 *data, __u32 len) +{ + int i; + int j = 0; + int row = 0; + const int NUM_OF_BYTES = 16; + + printf("\nOffset\t\tValues\n"); + printf("------\t\t------\n"); + while(row < len){ + printf("0x%04x\t\t",row); + for(i=0; i < NUM_OF_BYTES; i++){ + printf("%02x ", data[j]); + row++; + j++; + } + printf("\n"); + } +} + +/* Read cable EEPROM module information by first inspecting the first + * two bytes to get the length and then read the rest of the information. + * The information is printed to dmesg. */ +static int mlx4_en_read_eeprom(SYSCTL_HANDLER_ARGS) +{ + + u8* data; + int error; + int result = 0; + struct mlx4_en_priv *priv; + struct net_device *dev; + struct ethtool_modinfo modinfo; + struct ethtool_eeprom ee; + + error = sysctl_handle_int(oidp, &result, 0, req); + if (error || !req->newptr) + return (error); + + if (result == 1) { + priv = arg1; + dev = priv->dev; + data = kmalloc(PAGE_SIZE, GFP_KERNEL); + + error = mlx4_en_get_module_info(dev, &modinfo); + if (error) { + en_err(priv, + "mlx4_en_get_module_info returned with error - FAILED (0x%x)\n", + -error); + goto out; + } + + ee.len = modinfo.eeprom_len; + ee.offset = 0; + + error = mlx4_en_get_module_eeprom(dev, &ee, data); + if (error) { + en_err(priv, + "mlx4_en_get_module_eeprom returned with error - FAILED (0x%x)\n", + -error); + /* Continue printing partial information in case of an error */ + } + + /* EEPROM information will be printed in dmesg */ + mlx4_en_print_eeprom(data, ee.len); +out: + kfree(data); + } + /* Return zero to prevent sysctl failure. */ + return (0); +} + +static int mlx4_en_set_tx_ppp(SYSCTL_HANDLER_ARGS) +{ + struct mlx4_en_priv *priv; + int ppp; + int error; + + priv = arg1; + ppp = priv->prof->tx_ppp; + error = sysctl_handle_int(oidp, &ppp, 0, req); + if (error || !req->newptr) + return (error); + if (ppp > 0xff || ppp < 0) + return (-EINVAL); + priv->prof->tx_ppp = ppp; + error = -mlx4_SET_PORT_general(priv->mdev->dev, priv->port, + priv->rx_mb_size + ETHER_CRC_LEN, + priv->prof->tx_pause, + priv->prof->tx_ppp, + priv->prof->rx_pause, + priv->prof->rx_ppp); + + return (error); +} + +static int mlx4_en_set_rx_ppp(SYSCTL_HANDLER_ARGS) +{ + struct mlx4_en_priv *priv; + struct mlx4_en_dev *mdev; + int ppp; + int error; + int port_up; + + port_up = 0; + priv = arg1; + mdev = priv->mdev; + ppp = priv->prof->rx_ppp; + error = sysctl_handle_int(oidp, &ppp, 0, req); + if (error || !req->newptr) + return (error); + if (ppp > 0xff || ppp < 0) + return (-EINVAL); + /* See if we have to change the number of tx queues. */ + if (!ppp != !priv->prof->rx_ppp) { + mutex_lock(&mdev->state_lock); + if (priv->port_up) { + port_up = 1; + mlx4_en_stop_port(priv->dev); + } + mlx4_en_free_resources(priv); + priv->prof->rx_ppp = ppp; + error = -mlx4_en_alloc_resources(priv); + if (error) + en_err(priv, "Failed reallocating port resources\n"); + if (error == 0 && port_up) { + error = -mlx4_en_start_port(priv->dev); + if (error) + en_err(priv, "Failed starting port\n"); + } + mutex_unlock(&mdev->state_lock); + return (error); + + } + priv->prof->rx_ppp = ppp; + error = -mlx4_SET_PORT_general(priv->mdev->dev, priv->port, + priv->rx_mb_size + ETHER_CRC_LEN, + priv->prof->tx_pause, + priv->prof->tx_ppp, + priv->prof->rx_pause, + priv->prof->rx_ppp); + + return (error); +} + +static void mlx4_en_sysctl_conf(struct mlx4_en_priv *priv) +{ + struct net_device *dev; + struct sysctl_ctx_list *ctx; + struct sysctl_oid *node; + struct sysctl_oid_list *node_list; + struct sysctl_oid *coal; + struct sysctl_oid_list *coal_list; + const char *pnameunit; + dev = priv->dev; + ctx = &priv->conf_ctx; + pnameunit = device_get_nameunit(priv->mdev->pdev->dev.bsddev); + + sysctl_ctx_init(ctx); + priv->conf_sysctl = SYSCTL_ADD_NODE(ctx, SYSCTL_STATIC_CHILDREN(_hw), + OID_AUTO, dev->if_xname, CTLFLAG_RD, 0, "mlx4 10gig ethernet"); + node = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(priv->conf_sysctl), OID_AUTO, + "conf", CTLFLAG_RD, NULL, "Configuration"); + node_list = SYSCTL_CHILDREN(node); + + SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "msg_enable", + CTLFLAG_RW, &priv->msg_enable, 0, + "Driver message enable bitfield"); + SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "rx_rings", + CTLFLAG_RD, &priv->rx_ring_num, 0, + "Number of receive rings"); + SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "tx_rings", + CTLFLAG_RD, &priv->tx_ring_num, 0, + "Number of transmit rings"); + SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "rx_size", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, + mlx4_en_set_rx_ring_size, "I", "Receive ring size"); + SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "tx_size", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, + mlx4_en_set_tx_ring_size, "I", "Transmit ring size"); + SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "tx_ppp", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, + mlx4_en_set_tx_ppp, "I", "TX Per-priority pause"); + SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "rx_ppp", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, + mlx4_en_set_rx_ppp, "I", "RX Per-priority pause"); + SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "port_num", + CTLFLAG_RD, &priv->port, 0, + "Port Number"); + SYSCTL_ADD_STRING(ctx, node_list, OID_AUTO, "device_name", + CTLFLAG_RD, __DECONST(void *, pnameunit), 0, + "PCI device name"); + /* Add coalescer configuration. */ + coal = SYSCTL_ADD_NODE(ctx, node_list, OID_AUTO, + "coalesce", CTLFLAG_RD, NULL, "Interrupt coalesce configuration"); + coal_list = SYSCTL_CHILDREN(coal); + SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "pkt_rate_low", + CTLFLAG_RW, &priv->pkt_rate_low, 0, + "Packets per-second for minimum delay"); + SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "rx_usecs_low", + CTLFLAG_RW, &priv->rx_usecs_low, 0, + "Minimum RX delay in micro-seconds"); + SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "pkt_rate_high", + CTLFLAG_RW, &priv->pkt_rate_high, 0, + "Packets per-second for maximum delay"); + SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "rx_usecs_high", + CTLFLAG_RW, &priv->rx_usecs_high, 0, + "Maximum RX delay in micro-seconds"); + SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "sample_interval", + CTLFLAG_RW, &priv->sample_interval, 0, + "adaptive frequency in units of HZ ticks"); + SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "adaptive_rx_coal", + CTLFLAG_RW, &priv->adaptive_rx_coal, 0, + "Enable adaptive rx coalescing"); + /* EEPROM support */ + SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "eeprom_info", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, + mlx4_en_read_eeprom, "I", "EEPROM information"); +} + +static void mlx4_en_sysctl_stat(struct mlx4_en_priv *priv) +{ + struct sysctl_ctx_list *ctx; + struct sysctl_oid_list *node_list; + struct sysctl_oid *ring_node; + struct sysctl_oid_list *ring_list; + struct mlx4_en_tx_ring *tx_ring; + struct mlx4_en_rx_ring *rx_ring; + char namebuf[128]; + int i; + + ctx = &priv->stat_ctx; + sysctl_ctx_init(ctx); + priv->stat_sysctl = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(priv->conf_sysctl), OID_AUTO, + "stat", CTLFLAG_RD, NULL, "Statistics"); + node_list = SYSCTL_CHILDREN(priv->stat_sysctl); + +#ifdef MLX4_EN_PERF_STAT + SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "tx_poll", CTLFLAG_RD, + &priv->pstats.tx_poll, "TX Poll calls"); + SYSCTL_ADD_QUAD(ctx, node_list, OID_AUTO, "tx_pktsz_avg", CTLFLAG_RD, + &priv->pstats.tx_pktsz_avg, "TX average packet size"); + SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "inflight_avg", CTLFLAG_RD, + &priv->pstats.inflight_avg, "TX average packets in-flight"); + SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "tx_coal_avg", CTLFLAG_RD, + &priv->pstats.tx_coal_avg, "TX average coalesced completions"); + SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "rx_coal_avg", CTLFLAG_RD, + &priv->pstats.rx_coal_avg, "RX average coalesced completions"); +#endif + + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tso_packets", CTLFLAG_RD, + &priv->port_stats.tso_packets, 0, "TSO packets sent"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "queue_stopped", CTLFLAG_RD, + &priv->port_stats.queue_stopped, 0, "Queue full"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "wake_queue", CTLFLAG_RD, + &priv->port_stats.wake_queue, 0, "Queue resumed after full"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_timeout", CTLFLAG_RD, + &priv->port_stats.tx_timeout, 0, "Transmit timeouts"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_oversized_packets", CTLFLAG_RD, + &priv->port_stats.oversized_packets, 0, "TX oversized packets, m_defrag failed"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_alloc_failed", CTLFLAG_RD, + &priv->port_stats.rx_alloc_failed, 0, "RX failed to allocate mbuf"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_chksum_good", CTLFLAG_RD, + &priv->port_stats.rx_chksum_good, 0, "RX checksum offload success"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_chksum_none", CTLFLAG_RD, + &priv->port_stats.rx_chksum_none, 0, "RX without checksum offload"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_chksum_offload", + CTLFLAG_RD, &priv->port_stats.tx_chksum_offload, 0, + "TX checksum offloads"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "defrag_attempts", + CTLFLAG_RD, &priv->port_stats.defrag_attempts, 0, + "Oversized chains defragged"); + + /* Could strdup the names and add in a loop. This is simpler. */ + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_bytes", CTLFLAG_RD, + &priv->pkstats.rx_bytes, 0, "RX Bytes"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_packets", CTLFLAG_RD, + &priv->pkstats.rx_packets, 0, "RX packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_multicast_packets", CTLFLAG_RD, + &priv->pkstats.rx_multicast_packets, 0, "RX Multicast Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_broadcast_packets", CTLFLAG_RD, + &priv->pkstats.rx_broadcast_packets, 0, "RX Broadcast Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_errors", CTLFLAG_RD, + &priv->pkstats.rx_errors, 0, "RX Errors"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_dropped", CTLFLAG_RD, + &priv->pkstats.rx_dropped, 0, "RX Dropped"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_length_errors", CTLFLAG_RD, + &priv->pkstats.rx_length_errors, 0, "RX Length Errors"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_over_errors", CTLFLAG_RD, + &priv->pkstats.rx_over_errors, 0, "RX Over Errors"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_crc_errors", CTLFLAG_RD, + &priv->pkstats.rx_crc_errors, 0, "RX CRC Errors"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_jabbers", CTLFLAG_RD, + &priv->pkstats.rx_jabbers, 0, "RX Jabbers"); + + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_in_range_length_error", CTLFLAG_RD, + &priv->pkstats.rx_in_range_length_error, 0, "RX IN_Range Length Error"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_out_range_length_error", + CTLFLAG_RD, &priv->pkstats.rx_out_range_length_error, 0, + "RX Out Range Length Error"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_lt_64_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_lt_64_bytes_packets, 0, "RX Lt 64 Bytes Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_127_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_127_bytes_packets, 0, "RX 127 bytes Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_255_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_255_bytes_packets, 0, "RX 255 bytes Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_511_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_511_bytes_packets, 0, "RX 511 bytes Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_1023_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_1023_bytes_packets, 0, "RX 1023 bytes Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_1518_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_1518_bytes_packets, 0, "RX 1518 bytes Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_1522_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_1522_bytes_packets, 0, "RX 1522 bytes Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_1548_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_1548_bytes_packets, 0, "RX 1548 bytes Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "rx_gt_1548_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_gt_1548_bytes_packets, 0, + "RX Greater Then 1548 bytes Packets"); + + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_packets", CTLFLAG_RD, + &priv->pkstats.tx_packets, 0, "TX packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_bytes", CTLFLAG_RD, + &priv->pkstats.tx_bytes, 0, "TX Bytes"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_multicast_packets", CTLFLAG_RD, + &priv->pkstats.tx_multicast_packets, 0, "TX Multicast Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_broadcast_packets", CTLFLAG_RD, + &priv->pkstats.tx_broadcast_packets, 0, "TX Broadcast Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_errors", CTLFLAG_RD, + &priv->pkstats.tx_errors, 0, "TX Errors"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_dropped", CTLFLAG_RD, + &priv->pkstats.tx_dropped, 0, "TX Dropped"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_lt_64_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_lt_64_bytes_packets, 0, "TX Less Then 64 Bytes Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_127_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_127_bytes_packets, 0, "TX 127 Bytes Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_255_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_255_bytes_packets, 0, "TX 255 Bytes Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_511_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_511_bytes_packets, 0, "TX 511 Bytes Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_1023_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_1023_bytes_packets, 0, "TX 1023 Bytes Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_1518_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_1518_bytes_packets, 0, "TX 1518 Bytes Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_1522_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_1522_bytes_packets, 0, "TX 1522 Bytes Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_1548_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_1548_bytes_packets, 0, "TX 1548 Bytes Packets"); + SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, "tx_gt_1548_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_gt_1548_bytes_packets, 0, + "TX Greater Then 1548 Bytes Packets"); + + for (i = 0; i < priv->tx_ring_num; i++) { + tx_ring = priv->tx_ring[i]; + snprintf(namebuf, sizeof(namebuf), "tx_ring%d", i); + ring_node = SYSCTL_ADD_NODE(ctx, node_list, OID_AUTO, namebuf, + CTLFLAG_RD, NULL, "TX Ring"); + ring_list = SYSCTL_CHILDREN(ring_node); + SYSCTL_ADD_U64(ctx, ring_list, OID_AUTO, "packets", + CTLFLAG_RD, &tx_ring->packets, 0, "TX packets"); + SYSCTL_ADD_U64(ctx, ring_list, OID_AUTO, "bytes", + CTLFLAG_RD, &tx_ring->bytes, 0, "TX bytes"); + SYSCTL_ADD_U64(ctx, ring_list, OID_AUTO, "tso_packets", + CTLFLAG_RD, &tx_ring->tso_packets, 0, "TSO packets"); + SYSCTL_ADD_U64(ctx, ring_list, OID_AUTO, "defrag_attempts", + CTLFLAG_RD, &tx_ring->defrag_attempts, 0, + "Oversized chains defragged"); + } + + for (i = 0; i < priv->rx_ring_num; i++) { + rx_ring = priv->rx_ring[i]; + snprintf(namebuf, sizeof(namebuf), "rx_ring%d", i); + ring_node = SYSCTL_ADD_NODE(ctx, node_list, OID_AUTO, namebuf, + CTLFLAG_RD, NULL, "RX Ring"); + ring_list = SYSCTL_CHILDREN(ring_node); + SYSCTL_ADD_U64(ctx, ring_list, OID_AUTO, "packets", + CTLFLAG_RD, &rx_ring->packets, 0, "RX packets"); + SYSCTL_ADD_U64(ctx, ring_list, OID_AUTO, "bytes", + CTLFLAG_RD, &rx_ring->bytes, 0, "RX bytes"); + SYSCTL_ADD_U64(ctx, ring_list, OID_AUTO, "error", + CTLFLAG_RD, &rx_ring->errors, 0, "RX soft errors"); + } +} Property changes on: stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_port.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_port.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_port.c (revision 329159) @@ -0,0 +1,508 @@ +/* + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#include +#include + +#include +#include + +#include "en_port.h" +#include "en.h" + + +int mlx4_SET_VLAN_FLTR(struct mlx4_dev *dev, struct mlx4_en_priv *priv) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_set_vlan_fltr_mbox *filter; + int i; + int j; + int index = 0; + u32 entry; + int err = 0; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + filter = mailbox->buf; + for (i = VLAN_FLTR_SIZE - 1; i >= 0; i--) { + entry = 0; + for (j = 0; j < 32; j++) { + if (test_bit(index, priv->active_vlans)) + entry |= 1 << j; + index++; + } + filter->entry[i] = cpu_to_be32(entry); + } + err = mlx4_cmd(dev, mailbox->dma, priv->port, 0, MLX4_CMD_SET_VLAN_FLTR, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +int mlx4_en_QUERY_PORT(struct mlx4_en_dev *mdev, u8 port) +{ + struct mlx4_en_query_port_context *qport_context; + struct mlx4_en_priv *priv = netdev_priv(mdev->pndev[port]); + struct mlx4_en_port_state *state = &priv->port_state; + struct mlx4_cmd_mailbox *mailbox; + int err; + + mailbox = mlx4_alloc_cmd_mailbox(mdev->dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + err = mlx4_cmd_box(mdev->dev, 0, mailbox->dma, port, 0, + MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); + if (err) + goto out; + qport_context = mailbox->buf; + + /* This command is always accessed from Ethtool context + * already synchronized, no need in locking */ + state->link_state = !!(qport_context->link_up & MLX4_EN_LINK_UP_MASK); + switch (qport_context->link_speed & MLX4_EN_SPEED_MASK) { + case MLX4_EN_100M_SPEED: + state->link_speed = 100; + break; + case MLX4_EN_1G_SPEED: + state->link_speed = 1000; + break; + case MLX4_EN_10G_SPEED_XAUI: + case MLX4_EN_10G_SPEED_XFI: + state->link_speed = 10000; + break; + case MLX4_EN_20G_SPEED: + state->link_speed = 20000; + break; + case MLX4_EN_40G_SPEED: + state->link_speed = 40000; + break; + case MLX4_EN_56G_SPEED: + state->link_speed = 56000; + break; + default: + state->link_speed = -1; + break; + } + + state->transceiver = qport_context->transceiver; + + state->flags = 0; /* Reset and recalculate the port flags */ + state->flags |= (qport_context->link_up & MLX4_EN_ANC_MASK) ? + MLX4_EN_PORT_ANC : 0; + state->flags |= (qport_context->autoneg & MLX4_EN_AUTONEG_MASK) ? + MLX4_EN_PORT_ANE : 0; + +out: + mlx4_free_cmd_mailbox(mdev->dev, mailbox); + return err; +} + +/* Each counter set is located in struct mlx4_en_stat_out_mbox + * with a const offset between its prio components. + * This function runs over a counter set and sum all of it's prio components. + */ +static u64 en_stats_adder(__be64 *start, __be64 *next, int num) +{ + __be64 *curr = start; + u64 ret = 0; + int i; + int offset = next - start; + + for (i = 0; i < num; i++) { + ret += be64_to_cpu(*curr); + curr += offset; + } + + return ret; +} + +static void mlx4_en_fold_software_stats(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + u64 packets, bytes; + int i; + + if (!priv->port_up || mlx4_is_master(mdev->dev)) + return; + + packets = 0; + bytes = 0; + for (i = 0; i < priv->rx_ring_num; i++) { + const struct mlx4_en_rx_ring *ring = priv->rx_ring[i]; + + packets += READ_ONCE(ring->packets); + bytes += READ_ONCE(ring->bytes); + } + priv->pkstats.rx_packets = packets; + priv->pkstats.rx_bytes = bytes; + + packets = 0; + bytes = 0; + for (i = 0; i < priv->tx_ring_num; i++) { + const struct mlx4_en_tx_ring *ring = priv->tx_ring[i]; + + packets += READ_ONCE(ring->packets); + bytes += READ_ONCE(ring->bytes); + } + priv->pkstats.tx_packets = packets; + priv->pkstats.tx_bytes = bytes; +} + +int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset) +{ + struct mlx4_counter tmp_vport_stats; + struct mlx4_en_stat_out_mbox *mlx4_en_stats; + struct mlx4_en_stat_out_flow_control_mbox *flowstats; + struct net_device *dev = mdev->pndev[port]; + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_vport_stats *vport_stats = &priv->vport_stats; + struct mlx4_cmd_mailbox *mailbox; + u64 in_mod = reset << 8 | port; + int err; + int i, counter_index; + + mailbox = mlx4_alloc_cmd_mailbox(mdev->dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + err = mlx4_cmd_box(mdev->dev, 0, mailbox->dma, in_mod, 0, + MLX4_CMD_DUMP_ETH_STATS, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); + if (err) + goto out; + + mlx4_en_stats = mailbox->buf; + + spin_lock(&priv->stats_lock); + + priv->port_stats.rx_chksum_good = 0; + priv->port_stats.rx_chksum_none = 0; + for (i = 0; i < priv->rx_ring_num; i++) { + priv->port_stats.rx_chksum_good += priv->rx_ring[i]->csum_ok; + priv->port_stats.rx_chksum_none += priv->rx_ring[i]->csum_none; + } + priv->port_stats.tx_chksum_offload = 0; + priv->port_stats.queue_stopped = 0; + priv->port_stats.wake_queue = 0; + priv->port_stats.oversized_packets = 0; + priv->port_stats.tso_packets = 0; + priv->port_stats.defrag_attempts = 0; + + for (i = 0; i < priv->tx_ring_num; i++) { + const struct mlx4_en_tx_ring *ring; + ring = priv->tx_ring[i]; + + priv->port_stats.tx_chksum_offload += ring->tx_csum; + priv->port_stats.queue_stopped += ring->queue_stopped; + priv->port_stats.wake_queue += ring->wake_queue; + priv->port_stats.oversized_packets += ring->oversized_packets; + priv->port_stats.tso_packets += ring->tso_packets; + priv->port_stats.defrag_attempts += ring->defrag_attempts; + } + + priv->pkstats.rx_errors = + be64_to_cpu(mlx4_en_stats->PCS) + + be32_to_cpu(mlx4_en_stats->RJBBR) + + be32_to_cpu(mlx4_en_stats->RCRC) + + be32_to_cpu(mlx4_en_stats->RRUNT) + + be64_to_cpu(mlx4_en_stats->RInRangeLengthErr) + + be64_to_cpu(mlx4_en_stats->ROutRangeLengthErr) + + be32_to_cpu(mlx4_en_stats->RSHORT) + + en_stats_adder(&mlx4_en_stats->RGIANT_prio_0, + &mlx4_en_stats->RGIANT_prio_1, + NUM_PRIORITIES); + priv->pkstats.tx_errors = + en_stats_adder(&mlx4_en_stats->TGIANT_prio_0, + &mlx4_en_stats->TGIANT_prio_1, + NUM_PRIORITIES); + priv->pkstats.rx_multicast_packets = + en_stats_adder(&mlx4_en_stats->MCAST_prio_0, + &mlx4_en_stats->MCAST_prio_1, + NUM_PRIORITIES); + priv->pkstats.rx_dropped = be32_to_cpu(mlx4_en_stats->RDROP); + priv->pkstats.rx_length_errors = be32_to_cpu(mlx4_en_stats->RdropLength); + priv->pkstats.rx_over_errors = be32_to_cpu(mlx4_en_stats->RdropOvflw); + priv->pkstats.rx_crc_errors = be32_to_cpu(mlx4_en_stats->RCRC); + priv->pkstats.rx_dropped = be32_to_cpu(mlx4_en_stats->RdropOvflw); + priv->pkstats.tx_dropped = be32_to_cpu(mlx4_en_stats->TDROP); + + /* RX stats */ + priv->pkstats.rx_packets = en_stats_adder(&mlx4_en_stats->RTOT_prio_0, + &mlx4_en_stats->RTOT_prio_1, + NUM_PRIORITIES); + priv->pkstats.rx_bytes = en_stats_adder(&mlx4_en_stats->ROCT_prio_0, + &mlx4_en_stats->ROCT_prio_1, + NUM_PRIORITIES); + priv->pkstats.rx_broadcast_packets = + en_stats_adder(&mlx4_en_stats->RBCAST_prio_0, + &mlx4_en_stats->RBCAST_prio_1, + NUM_PRIORITIES); + priv->pkstats.rx_jabbers = be32_to_cpu(mlx4_en_stats->RJBBR); + priv->pkstats.rx_in_range_length_error = + be64_to_cpu(mlx4_en_stats->RInRangeLengthErr); + priv->pkstats.rx_out_range_length_error = + be64_to_cpu(mlx4_en_stats->ROutRangeLengthErr); + + /* Tx stats */ + priv->pkstats.tx_packets = en_stats_adder(&mlx4_en_stats->TTOT_prio_0, + &mlx4_en_stats->TTOT_prio_1, + NUM_PRIORITIES); + priv->pkstats.tx_bytes = en_stats_adder(&mlx4_en_stats->TOCT_prio_0, + &mlx4_en_stats->TOCT_prio_1, + NUM_PRIORITIES); + priv->pkstats.tx_multicast_packets = + en_stats_adder(&mlx4_en_stats->TMCAST_prio_0, + &mlx4_en_stats->TMCAST_prio_1, + NUM_PRIORITIES); + priv->pkstats.tx_broadcast_packets = + en_stats_adder(&mlx4_en_stats->TBCAST_prio_0, + &mlx4_en_stats->TBCAST_prio_1, + NUM_PRIORITIES); + + priv->pkstats.rx_prio[0][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_0); + priv->pkstats.rx_prio[0][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_0); + priv->pkstats.rx_prio[1][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_1); + priv->pkstats.rx_prio[1][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_1); + priv->pkstats.rx_prio[2][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_2); + priv->pkstats.rx_prio[2][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_2); + priv->pkstats.rx_prio[3][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_3); + priv->pkstats.rx_prio[3][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_3); + priv->pkstats.rx_prio[4][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_4); + priv->pkstats.rx_prio[4][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_4); + priv->pkstats.rx_prio[5][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_5); + priv->pkstats.rx_prio[5][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_5); + priv->pkstats.rx_prio[6][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_6); + priv->pkstats.rx_prio[6][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_6); + priv->pkstats.rx_prio[7][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_7); + priv->pkstats.rx_prio[7][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_7); + priv->pkstats.rx_prio[8][0] = be64_to_cpu(mlx4_en_stats->RTOT_novlan); + priv->pkstats.rx_prio[8][1] = be64_to_cpu(mlx4_en_stats->ROCT_novlan); + priv->pkstats.tx_prio[0][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_0); + priv->pkstats.tx_prio[0][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_0); + priv->pkstats.tx_prio[1][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_1); + priv->pkstats.tx_prio[1][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_1); + priv->pkstats.tx_prio[2][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_2); + priv->pkstats.tx_prio[2][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_2); + priv->pkstats.tx_prio[3][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_3); + priv->pkstats.tx_prio[3][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_3); + priv->pkstats.tx_prio[4][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_4); + priv->pkstats.tx_prio[4][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_4); + priv->pkstats.tx_prio[5][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_5); + priv->pkstats.tx_prio[5][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_5); + priv->pkstats.tx_prio[6][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_6); + priv->pkstats.tx_prio[6][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_6); + priv->pkstats.tx_prio[7][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_7); + priv->pkstats.tx_prio[7][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_7); + priv->pkstats.tx_prio[8][0] = be64_to_cpu(mlx4_en_stats->TTOT_novlan); + priv->pkstats.tx_prio[8][1] = be64_to_cpu(mlx4_en_stats->TOCT_novlan); + + mlx4_en_fold_software_stats(dev); + + spin_unlock(&priv->stats_lock); + + memset(&tmp_vport_stats, 0, sizeof(tmp_vport_stats)); + counter_index = mlx4_get_default_counter_index(mdev->dev, port); + err = mlx4_get_counter_stats(mdev->dev, counter_index, + &tmp_vport_stats, reset); + + spin_lock(&priv->stats_lock); + if (!err) { + /* ethtool stats format */ + vport_stats->rx_bytes = be64_to_cpu(tmp_vport_stats.rx_bytes); + vport_stats->rx_frames = be64_to_cpu(tmp_vport_stats.rx_frames); + vport_stats->tx_bytes = be64_to_cpu(tmp_vport_stats.tx_bytes); + vport_stats->tx_frames = be64_to_cpu(tmp_vport_stats.tx_frames); + } + +#if __FreeBSD_version >= 1100000 + if (reset == 0) { + if_inc_counter(dev, IFCOUNTER_IPACKETS, + priv->pkstats.rx_packets - priv->pkstats_last.rx_packets); + if_inc_counter(dev, IFCOUNTER_OPACKETS, + priv->pkstats.tx_packets - priv->pkstats_last.tx_packets); + if_inc_counter(dev, IFCOUNTER_IBYTES, + priv->pkstats.rx_bytes - priv->pkstats_last.rx_bytes); + if_inc_counter(dev, IFCOUNTER_OBYTES, + priv->pkstats.tx_bytes - priv->pkstats_last.tx_bytes); + if_inc_counter(dev, IFCOUNTER_IERRORS, + priv->pkstats.rx_errors - priv->pkstats_last.rx_errors); + if_inc_counter(dev, IFCOUNTER_IQDROPS, + priv->pkstats.rx_dropped - priv->pkstats_last.rx_dropped); + if_inc_counter(dev, IFCOUNTER_IMCASTS, + priv->pkstats.rx_multicast_packets - priv->pkstats_last.rx_multicast_packets); + if_inc_counter(dev, IFCOUNTER_OMCASTS, + priv->pkstats.tx_multicast_packets - priv->pkstats_last.tx_multicast_packets); + } + priv->pkstats_last = priv->pkstats; +#else + dev->if_ipackets = priv->pkstats.rx_packets; + dev->if_opackets = priv->pkstats.tx_packets; + dev->if_ibytes = priv->pkstats.rx_bytes; + dev->if_obytes = priv->pkstats.tx_bytes; + dev->if_ierrors = priv->pkstats.rx_errors; + dev->if_iqdrops = priv->pkstats.rx_dropped; + dev->if_imcasts = priv->pkstats.rx_multicast_packets; + dev->if_omcasts = priv->pkstats.tx_multicast_packets; + dev->if_collisions = 0; +#endif + + spin_unlock(&priv->stats_lock); + + /* 0xffs indicates invalid value */ + memset(mailbox->buf, 0xff, sizeof(*flowstats) * MLX4_NUM_PRIORITIES); + + if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) { + memset(mailbox->buf, 0, + sizeof(*flowstats) * MLX4_NUM_PRIORITIES); + err = mlx4_cmd_box(mdev->dev, 0, mailbox->dma, + in_mod | MLX4_DUMP_ETH_STATS_FLOW_CONTROL, + 0, MLX4_CMD_DUMP_ETH_STATS, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); + if (err) + goto out; + } + + flowstats = mailbox->buf; + + spin_lock(&priv->stats_lock); + + for (i = 0; i < MLX4_NUM_PRIORITIES; i++) { + priv->rx_priority_flowstats[i].rx_pause = + be64_to_cpu(flowstats[i].rx_pause); + priv->rx_priority_flowstats[i].rx_pause_duration = + be64_to_cpu(flowstats[i].rx_pause_duration); + priv->rx_priority_flowstats[i].rx_pause_transition = + be64_to_cpu(flowstats[i].rx_pause_transition); + priv->tx_priority_flowstats[i].tx_pause = + be64_to_cpu(flowstats[i].tx_pause); + priv->tx_priority_flowstats[i].tx_pause_duration = + be64_to_cpu(flowstats[i].tx_pause_duration); + priv->tx_priority_flowstats[i].tx_pause_transition = + be64_to_cpu(flowstats[i].tx_pause_transition); + } + + /* if pfc is not in use, all priorities counters have the same value */ + priv->rx_flowstats.rx_pause = + be64_to_cpu(flowstats[0].rx_pause); + priv->rx_flowstats.rx_pause_duration = + be64_to_cpu(flowstats[0].rx_pause_duration); + priv->rx_flowstats.rx_pause_transition = + be64_to_cpu(flowstats[0].rx_pause_transition); + priv->tx_flowstats.tx_pause = + be64_to_cpu(flowstats[0].tx_pause); + priv->tx_flowstats.tx_pause_duration = + be64_to_cpu(flowstats[0].tx_pause_duration); + priv->tx_flowstats.tx_pause_transition = + be64_to_cpu(flowstats[0].tx_pause_transition); + + spin_unlock(&priv->stats_lock); + +out: + mlx4_free_cmd_mailbox(mdev->dev, mailbox); + return err; +} + +int mlx4_en_get_vport_stats(struct mlx4_en_dev *mdev, u8 port) +{ + struct mlx4_en_priv *priv = netdev_priv(mdev->pndev[port]); + struct mlx4_counter tmp_vport_stats; + struct mlx4_en_vf_stats *vf_stats = &priv->vf_stats; + int err, i, counter_index; + + spin_lock(&priv->stats_lock); + + priv->pkstats.rx_packets = 0; + priv->pkstats.rx_bytes = 0; + priv->port_stats.rx_chksum_good = 0; + priv->port_stats.rx_chksum_none = 0; + for (i = 0; i < priv->rx_ring_num; i++) { + priv->pkstats.rx_packets += priv->rx_ring[i]->packets; + priv->pkstats.rx_bytes += priv->rx_ring[i]->bytes; + priv->port_stats.rx_chksum_good += priv->rx_ring[i]->csum_ok; + priv->port_stats.rx_chksum_none += priv->rx_ring[i]->csum_none; + } + priv->pkstats.tx_packets = 0; + priv->pkstats.tx_bytes = 0; + priv->port_stats.tx_chksum_offload = 0; + priv->port_stats.queue_stopped = 0; + priv->port_stats.wake_queue = 0; + + for (i = 0; i < priv->tx_ring_num; i++) { + const struct mlx4_en_tx_ring *ring = priv->tx_ring[i]; + + priv->pkstats.tx_packets += ring->packets; + priv->pkstats.tx_bytes += ring->bytes; + priv->port_stats.tx_chksum_offload += ring->tx_csum; + priv->port_stats.queue_stopped += ring->queue_stopped; + priv->port_stats.wake_queue += ring->wake_queue; + priv->port_stats.oversized_packets += priv->tx_ring[i]->oversized_packets; + } + + spin_unlock(&priv->stats_lock); + + memset(&tmp_vport_stats, 0, sizeof(tmp_vport_stats)); + + counter_index = mlx4_get_default_counter_index(mdev->dev, port); + err = mlx4_get_counter_stats(mdev->dev, counter_index, + &tmp_vport_stats, 0); + + if (!err) { + spin_lock(&priv->stats_lock); + + vf_stats->rx_bytes = be64_to_cpu(tmp_vport_stats.rx_bytes); + vf_stats->rx_frames = be64_to_cpu(tmp_vport_stats.rx_frames); + vf_stats->tx_bytes = be64_to_cpu(tmp_vport_stats.tx_bytes); + vf_stats->tx_frames = be64_to_cpu(tmp_vport_stats.tx_frames); + + priv->pkstats.rx_packets = vf_stats->rx_frames; + priv->pkstats.rx_bytes = vf_stats->rx_bytes; + priv->pkstats.tx_packets = vf_stats->tx_frames; + priv->pkstats.tx_bytes = vf_stats->tx_bytes; + + /* PF&VFs are not expected to report errors in ifconfig. + * rx_errors will be reprted in PF's ethtool statistics, + * see: mlx4_en_DUMP_ETH_STATS + */ + priv->pkstats.rx_errors = 0; + priv->pkstats.rx_dropped = 0; + priv->pkstats.tx_dropped = 0; + priv->pkstats.rx_multicast_packets = 0; + + spin_unlock(&priv->stats_lock); + } + + return err; +} Property changes on: stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_port.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_resources.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_resources.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_resources.c (revision 329159) @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#include +#include +#include + +#include "en.h" + + +void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride, + int is_tx, int rss, int qpn, int cqn, + int user_prio, struct mlx4_qp_context *context) +{ + struct mlx4_en_dev *mdev = priv->mdev; + struct net_device *dev = priv->dev; + + memset(context, 0, sizeof *context); + context->flags = cpu_to_be32(7 << 16 | rss << MLX4_RSS_QPC_FLAG_OFFSET); + context->pd = cpu_to_be32(mdev->priv_pdn); + context->mtu_msgmax = 0xff; + if (!is_tx && !rss) + context->rq_size_stride = ilog2(size) << 3 | (ilog2(stride) - 4); + if (is_tx) + context->sq_size_stride = ilog2(size) << 3 | (ilog2(stride) - 4); + else + context->sq_size_stride = ilog2(TXBB_SIZE) - 4; + context->usr_page = cpu_to_be32(mdev->priv_uar.index); + context->local_qpn = cpu_to_be32(qpn); + context->pri_path.ackto = 1 & 0x07; + context->pri_path.sched_queue = 0x83 | (priv->port - 1) << 6; + if (user_prio >= 0) { + context->pri_path.sched_queue |= user_prio << 3; + context->pri_path.feup = 1 << 6; + } + context->pri_path.counter_index = (u8)(priv->counter_index); + if (!rss && + (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_LB_SRC_CHK) && + context->pri_path.counter_index != 0xFF) { + /* disable multicast loopback to qp with same counter */ + context->pri_path.fl |= MLX4_FL_ETH_SRC_CHECK_MC_LB; + context->pri_path.vlan_control |= + MLX4_CTRL_ETH_SRC_CHECK_IF_COUNTER; + } + + context->cqn_send = cpu_to_be32(cqn); + context->cqn_recv = cpu_to_be32(cqn); + context->db_rec_addr = cpu_to_be64(priv->res.db.dma << 2); + if (!(dev->if_capabilities & IFCAP_VLAN_HWCSUM)) + context->param3 |= cpu_to_be32(1 << 30); +} + + +int mlx4_en_map_buffer(struct mlx4_buf *buf) +{ + struct page **pages; + int i; + + // if nbufs == 1 - there is no need to vmap + // if buf->direct.buf is not NULL it means that vmap was already done by mlx4_alloc_buff + if (buf->direct.buf != NULL || buf->nbufs == 1) + return 0; + + pages = kmalloc(sizeof *pages * buf->nbufs, GFP_KERNEL); + if (!pages) + return -ENOMEM; + + for (i = 0; i < buf->nbufs; ++i) + pages[i] = virt_to_page(buf->page_list[i].buf); + + buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL); + kfree(pages); + if (!buf->direct.buf) + return -ENOMEM; + + return 0; +} + +void mlx4_en_unmap_buffer(struct mlx4_buf *buf) +{ + if (BITS_PER_LONG == 64 || buf->nbufs == 1) + return; + + vunmap(buf->direct.buf); +} + +void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event) +{ + return; +} + Property changes on: stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_resources.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c (revision 329159) @@ -0,0 +1,1023 @@ +/* + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ +#include "opt_inet.h" +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_NET_RX_BUSY_POLL +#include +#endif + +#include "en.h" + + +static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring *ring, + int index) +{ + struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) + (ring->buf + (ring->stride * index)); + int possible_frags; + int i; + + /* Set size and memtype fields */ + rx_desc->data[0].byte_count = cpu_to_be32(priv->rx_mb_size - MLX4_NET_IP_ALIGN); + rx_desc->data[0].lkey = cpu_to_be32(priv->mdev->mr.key); + + /* + * If the number of used fragments does not fill up the ring + * stride, remaining (unused) fragments must be padded with + * null address/size and a special memory key: + */ + possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE; + for (i = 1; i < possible_frags; i++) { + rx_desc->data[i].byte_count = 0; + rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); + rx_desc->data[i].addr = 0; + } +} + +static int +mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, + __be64 *pdma, struct mlx4_en_rx_mbuf *mb_list) +{ + bus_dma_segment_t segs[1]; + bus_dmamap_t map; + struct mbuf *mb; + int nsegs; + int err; + + /* try to allocate a new spare mbuf */ + if (unlikely(ring->spare.mbuf == NULL)) { + mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); + if (unlikely(mb == NULL)) + return (-ENOMEM); + /* setup correct length */ + mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size; + + /* make sure IP header gets aligned */ + m_adj(mb, MLX4_NET_IP_ALIGN); + + /* load spare mbuf into BUSDMA */ + err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, ring->spare.dma_map, + mb, segs, &nsegs, BUS_DMA_NOWAIT); + if (unlikely(err != 0)) { + m_freem(mb); + return (err); + } + + /* store spare info */ + ring->spare.mbuf = mb; + ring->spare.paddr_be = cpu_to_be64(segs[0].ds_addr); + + bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, + BUS_DMASYNC_PREREAD); + } + + /* synchronize and unload the current mbuf, if any */ + if (likely(mb_list->mbuf != NULL)) { + bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(ring->dma_tag, mb_list->dma_map); + } + + mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); + if (unlikely(mb == NULL)) + goto use_spare; + + /* setup correct length */ + mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size; + + /* make sure IP header gets aligned */ + m_adj(mb, MLX4_NET_IP_ALIGN); + + err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, mb_list->dma_map, + mb, segs, &nsegs, BUS_DMA_NOWAIT); + if (unlikely(err != 0)) { + m_freem(mb); + goto use_spare; + } + + *pdma = cpu_to_be64(segs[0].ds_addr); + mb_list->mbuf = mb; + + bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, BUS_DMASYNC_PREREAD); + return (0); + +use_spare: + /* swap DMA maps */ + map = mb_list->dma_map; + mb_list->dma_map = ring->spare.dma_map; + ring->spare.dma_map = map; + + /* swap MBUFs */ + mb_list->mbuf = ring->spare.mbuf; + ring->spare.mbuf = NULL; + + /* store physical address */ + *pdma = ring->spare.paddr_be; + return (0); +} + +static void +mlx4_en_free_buf(struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_mbuf *mb_list) +{ + bus_dmamap_t map = mb_list->dma_map; + bus_dmamap_sync(ring->dma_tag, map, BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(ring->dma_tag, map); + m_freem(mb_list->mbuf); + mb_list->mbuf = NULL; /* safety clearing */ +} + +static int +mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring *ring, int index) +{ + struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) + (ring->buf + (index * ring->stride)); + struct mlx4_en_rx_mbuf *mb_list = ring->mbuf + index; + + mb_list->mbuf = NULL; + + if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) { + priv->port_stats.rx_alloc_failed++; + return (-ENOMEM); + } + return (0); +} + +static inline void +mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring) +{ + *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff); +} + +static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv) +{ + struct mlx4_en_rx_ring *ring; + int ring_ind; + int buf_ind; + int new_size; + int err; + + for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) { + for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { + ring = priv->rx_ring[ring_ind]; + + err = mlx4_en_prepare_rx_desc(priv, ring, + ring->actual_size); + if (err) { + if (ring->actual_size == 0) { + en_err(priv, "Failed to allocate " + "enough rx buffers\n"); + return -ENOMEM; + } else { + new_size = + rounddown_pow_of_two(ring->actual_size); + en_warn(priv, "Only %d buffers allocated " + "reducing ring size to %d\n", + ring->actual_size, new_size); + goto reduce_rings; + } + } + ring->actual_size++; + ring->prod++; + } + } + return 0; + +reduce_rings: + for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { + ring = priv->rx_ring[ring_ind]; + while (ring->actual_size > new_size) { + ring->actual_size--; + ring->prod--; + mlx4_en_free_buf(ring, + ring->mbuf + ring->actual_size); + } + } + + return 0; +} + +static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring *ring) +{ + int index; + + en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n", + ring->cons, ring->prod); + + /* Unmap and free Rx buffers */ + BUG_ON((u32) (ring->prod - ring->cons) > ring->actual_size); + while (ring->cons != ring->prod) { + index = ring->cons & ring->size_mask; + en_dbg(DRV, priv, "Processing descriptor:%d\n", index); + mlx4_en_free_buf(ring, ring->mbuf + index); + ++ring->cons; + } +} + +void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev) +{ + int i; + int num_of_eqs; + int num_rx_rings; + struct mlx4_dev *dev = mdev->dev; + + mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { + num_of_eqs = max_t(int, MIN_RX_RINGS, + min_t(int, + mlx4_get_eqs_per_port(mdev->dev, i), + DEF_RX_RINGS)); + + num_rx_rings = mlx4_low_memory_profile() ? MIN_RX_RINGS : + num_of_eqs; + mdev->profile.prof[i].rx_ring_num = + rounddown_pow_of_two(num_rx_rings); + } +} + +void mlx4_en_calc_rx_buf(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + int eff_mtu = dev->if_mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN + + MLX4_NET_IP_ALIGN; + + if (eff_mtu > MJUM16BYTES) { + en_err(priv, "MTU(%u) is too big\n", (unsigned)dev->if_mtu); + eff_mtu = MJUM16BYTES; + } else if (eff_mtu > MJUM9BYTES) { + eff_mtu = MJUM16BYTES; + } else if (eff_mtu > MJUMPAGESIZE) { + eff_mtu = MJUM9BYTES; + } else if (eff_mtu > MCLBYTES) { + eff_mtu = MJUMPAGESIZE; + } else { + eff_mtu = MCLBYTES; + } + + priv->rx_mb_size = eff_mtu; + + en_dbg(DRV, priv, "Effective RX MTU: %d bytes\n", eff_mtu); +} + +int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring **pring, + u32 size, int node) +{ + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_rx_ring *ring; + int err; + int tmp; + uint32_t x; + + ring = kzalloc(sizeof(struct mlx4_en_rx_ring), GFP_KERNEL); + if (!ring) { + en_err(priv, "Failed to allocate RX ring structure\n"); + return -ENOMEM; + } + + /* Create DMA descriptor TAG */ + if ((err = -bus_dma_tag_create( + bus_get_dma_tag(mdev->pdev->dev.bsddev), + 1, /* any alignment */ + 0, /* no boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MJUM16BYTES, /* maxsize */ + 1, /* nsegments */ + MJUM16BYTES, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockfuncarg */ + &ring->dma_tag))) { + en_err(priv, "Failed to create DMA tag\n"); + goto err_ring; + } + + ring->prod = 0; + ring->cons = 0; + ring->size = size; + ring->size_mask = size - 1; + ring->stride = roundup_pow_of_two( + sizeof(struct mlx4_en_rx_desc) + DS_SIZE); + ring->log_stride = ffs(ring->stride) - 1; + ring->buf_size = ring->size * ring->stride + TXBB_SIZE; + + tmp = size * sizeof(struct mlx4_en_rx_mbuf); + + ring->mbuf = kzalloc(tmp, GFP_KERNEL); + if (ring->mbuf == NULL) { + err = -ENOMEM; + goto err_dma_tag; + } + + err = -bus_dmamap_create(ring->dma_tag, 0, &ring->spare.dma_map); + if (err != 0) + goto err_info; + + for (x = 0; x != size; x++) { + err = -bus_dmamap_create(ring->dma_tag, 0, + &ring->mbuf[x].dma_map); + if (err != 0) { + while (x--) + bus_dmamap_destroy(ring->dma_tag, + ring->mbuf[x].dma_map); + goto err_info; + } + } + en_dbg(DRV, priv, "Allocated MBUF ring at addr:%p size:%d\n", + ring->mbuf, tmp); + + err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, + ring->buf_size, 2 * PAGE_SIZE); + if (err) + goto err_dma_map; + + err = mlx4_en_map_buffer(&ring->wqres.buf); + if (err) { + en_err(priv, "Failed to map RX buffer\n"); + goto err_hwq; + } + ring->buf = ring->wqres.buf.direct.buf; + *pring = ring; + return 0; + +err_hwq: + mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); +err_dma_map: + for (x = 0; x != size; x++) { + bus_dmamap_destroy(ring->dma_tag, + ring->mbuf[x].dma_map); + } + bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map); +err_info: + vfree(ring->mbuf); +err_dma_tag: + bus_dma_tag_destroy(ring->dma_tag); +err_ring: + kfree(ring); + return (err); +} + +int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv) +{ + struct mlx4_en_rx_ring *ring; + int i; + int ring_ind; + int err; + int stride = roundup_pow_of_two( + sizeof(struct mlx4_en_rx_desc) + DS_SIZE); + + for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { + ring = priv->rx_ring[ring_ind]; + + ring->prod = 0; + ring->cons = 0; + ring->actual_size = 0; + ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn; + ring->rx_mb_size = priv->rx_mb_size; + + ring->stride = stride; + if (ring->stride <= TXBB_SIZE) { + /* Stamp first unused send wqe */ + __be32 *ptr = (__be32 *)ring->buf; + __be32 stamp = cpu_to_be32(1 << STAMP_SHIFT); + *ptr = stamp; + /* Move pointer to start of rx section */ + ring->buf += TXBB_SIZE; + } + + ring->log_stride = ffs(ring->stride) - 1; + ring->buf_size = ring->size * ring->stride; + + memset(ring->buf, 0, ring->buf_size); + mlx4_en_update_rx_prod_db(ring); + + /* Initialize all descriptors */ + for (i = 0; i < ring->size; i++) + mlx4_en_init_rx_desc(priv, ring, i); + +#ifdef INET + /* Configure lro mngr */ + if (priv->dev->if_capenable & IFCAP_LRO) { + if (tcp_lro_init(&ring->lro)) + priv->dev->if_capenable &= ~IFCAP_LRO; + else + ring->lro.ifp = priv->dev; + } +#endif + } + + + err = mlx4_en_fill_rx_buffers(priv); + if (err) + goto err_buffers; + + for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { + ring = priv->rx_ring[ring_ind]; + + ring->size_mask = ring->actual_size - 1; + mlx4_en_update_rx_prod_db(ring); + } + + return 0; + +err_buffers: + for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) + mlx4_en_free_rx_buf(priv, priv->rx_ring[ring_ind]); + + ring_ind = priv->rx_ring_num - 1; + + while (ring_ind >= 0) { + ring = priv->rx_ring[ring_ind]; + if (ring->stride <= TXBB_SIZE) + ring->buf -= TXBB_SIZE; + ring_ind--; + } + + return err; +} + + +void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring **pring, + u32 size, u16 stride) +{ + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_rx_ring *ring = *pring; + uint32_t x; + + mlx4_en_unmap_buffer(&ring->wqres.buf); + mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE); + for (x = 0; x != size; x++) + bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map); + /* free spare mbuf, if any */ + if (ring->spare.mbuf != NULL) { + bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(ring->dma_tag, ring->spare.dma_map); + m_freem(ring->spare.mbuf); + } + bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map); + vfree(ring->mbuf); + bus_dma_tag_destroy(ring->dma_tag); + kfree(ring); + *pring = NULL; +#ifdef CONFIG_RFS_ACCEL + mlx4_en_cleanup_filters(priv, ring); +#endif +} + +void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring *ring) +{ +#ifdef INET + tcp_lro_free(&ring->lro); +#endif + mlx4_en_free_rx_buf(priv, ring); + if (ring->stride <= TXBB_SIZE) + ring->buf -= TXBB_SIZE; +} + + +static void validate_loopback(struct mlx4_en_priv *priv, struct mbuf *mb) +{ + int i; + int offset = ETHER_HDR_LEN; + + for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++, offset++) { + if (*(mb->m_data + offset) != (unsigned char) (i & 0xff)) + goto out_loopback; + } + /* Loopback found */ + priv->loopback_ok = 1; + +out_loopback: + m_freem(mb); +} + + +static inline int invalid_cqe(struct mlx4_en_priv *priv, + struct mlx4_cqe *cqe) +{ + /* Drop packet on bad receive or bad checksum */ + if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == + MLX4_CQE_OPCODE_ERROR)) { + en_err(priv, "CQE completed in error - vendor syndrom:%d syndrom:%d\n", + ((struct mlx4_err_cqe *)cqe)->vendor_err_syndrome, + ((struct mlx4_err_cqe *)cqe)->syndrome); + return 1; + } + if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) { + en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n"); + return 1; + } + + return 0; +} + +static struct mbuf * +mlx4_en_rx_mb(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, + struct mlx4_en_rx_desc *rx_desc, struct mlx4_en_rx_mbuf *mb_list, + int length) +{ + struct mbuf *mb; + + /* get mbuf */ + mb = mb_list->mbuf; + + /* collect used fragment while atomically replacing it */ + if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) + return (NULL); + + /* range check hardware computed value */ + if (unlikely(length > mb->m_len)) + length = mb->m_len; + + /* update total packet length in packet header */ + mb->m_len = mb->m_pkthdr.len = length; + return (mb); +} + +static __inline int +mlx4_en_rss_hash(__be16 status, int udp_rss) +{ + enum { + status_all = cpu_to_be16( + MLX4_CQE_STATUS_IPV4 | + MLX4_CQE_STATUS_IPV4F | + MLX4_CQE_STATUS_IPV6 | + MLX4_CQE_STATUS_TCP | + MLX4_CQE_STATUS_UDP), + status_ipv4_tcp = cpu_to_be16( + MLX4_CQE_STATUS_IPV4 | + MLX4_CQE_STATUS_TCP), + status_ipv6_tcp = cpu_to_be16( + MLX4_CQE_STATUS_IPV6 | + MLX4_CQE_STATUS_TCP), + status_ipv4_udp = cpu_to_be16( + MLX4_CQE_STATUS_IPV4 | + MLX4_CQE_STATUS_UDP), + status_ipv6_udp = cpu_to_be16( + MLX4_CQE_STATUS_IPV6 | + MLX4_CQE_STATUS_UDP), + status_ipv4 = cpu_to_be16(MLX4_CQE_STATUS_IPV4), + status_ipv6 = cpu_to_be16(MLX4_CQE_STATUS_IPV6) + }; + + status &= status_all; + switch (status) { + case status_ipv4_tcp: + return (M_HASHTYPE_RSS_TCP_IPV4); + case status_ipv6_tcp: + return (M_HASHTYPE_RSS_TCP_IPV6); + case status_ipv4_udp: + return (udp_rss ? M_HASHTYPE_RSS_UDP_IPV4 + : M_HASHTYPE_RSS_IPV4); + case status_ipv6_udp: + return (udp_rss ? M_HASHTYPE_RSS_UDP_IPV6 + : M_HASHTYPE_RSS_IPV6); + default: + if (status & status_ipv4) + return (M_HASHTYPE_RSS_IPV4); + if (status & status_ipv6) + return (M_HASHTYPE_RSS_IPV6); + return (M_HASHTYPE_OPAQUE_HASH); + } +} + +/* For cpu arch with cache line of 64B the performance is better when cqe size==64B + * To enlarge cqe size from 32B to 64B --> 32B of garbage (i.e. 0xccccccc) + * was added in the beginning of each cqe (the real data is in the corresponding 32B). + * The following calc ensures that when factor==1, it means we are aligned to 64B + * and we get the real cqe data*/ +#define CQE_FACTOR_INDEX(index, factor) (((index) << (factor)) + (factor)) +int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_cqe *cqe; + struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring]; + struct mlx4_en_rx_mbuf *mb_list; + struct mlx4_en_rx_desc *rx_desc; + struct mbuf *mb; + struct mlx4_cq *mcq = &cq->mcq; + struct mlx4_cqe *buf = cq->buf; + int index; + unsigned int length; + int polled = 0; + u32 cons_index = mcq->cons_index; + u32 size_mask = ring->size_mask; + int size = cq->size; + int factor = priv->cqe_factor; + const int udp_rss = priv->mdev->profile.udp_rss; + + if (!priv->port_up) + return 0; + + /* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx + * descriptor offset can be deducted from the CQE index instead of + * reading 'cqe->index' */ + index = cons_index & size_mask; + cqe = &buf[CQE_FACTOR_INDEX(index, factor)]; + + /* Process all completed CQEs */ + while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, + cons_index & size)) { + mb_list = ring->mbuf + index; + rx_desc = (struct mlx4_en_rx_desc *) + (ring->buf + (index << ring->log_stride)); + + /* + * make sure we read the CQE after we read the ownership bit + */ + rmb(); + + if (invalid_cqe(priv, cqe)) { + goto next; + } + /* + * Packet is OK - process it. + */ + length = be32_to_cpu(cqe->byte_cnt); + length -= ring->fcs_del; + + mb = mlx4_en_rx_mb(priv, ring, rx_desc, mb_list, length); + if (unlikely(!mb)) { + ring->errors++; + goto next; + } + + ring->bytes += length; + ring->packets++; + + if (unlikely(priv->validate_loopback)) { + validate_loopback(priv, mb); + goto next; + } + + /* forward Toeplitz compatible hash value */ + mb->m_pkthdr.flowid = be32_to_cpu(cqe->immed_rss_invalid); + M_HASHTYPE_SET(mb, mlx4_en_rss_hash(cqe->status, udp_rss)); + mb->m_pkthdr.rcvif = dev; + if (be32_to_cpu(cqe->vlan_my_qpn) & + MLX4_CQE_CVLAN_PRESENT_MASK) { + mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->sl_vid); + mb->m_flags |= M_VLANTAG; + } + if (likely(dev->if_capenable & + (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) && + (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) && + (cqe->checksum == cpu_to_be16(0xffff))) { + priv->port_stats.rx_chksum_good++; + mb->m_pkthdr.csum_flags = + CSUM_IP_CHECKED | CSUM_IP_VALID | + CSUM_DATA_VALID | CSUM_PSEUDO_HDR; + mb->m_pkthdr.csum_data = htons(0xffff); + /* This packet is eligible for LRO if it is: + * - DIX Ethernet (type interpretation) + * - TCP/IP (v4) + * - without IP options + * - not an IP fragment + */ +#ifdef INET + if (mlx4_en_can_lro(cqe->status) && + (dev->if_capenable & IFCAP_LRO)) { + if (ring->lro.lro_cnt != 0 && + tcp_lro_rx(&ring->lro, mb, 0) == 0) + goto next; + } + +#endif + /* LRO not possible, complete processing here */ + INC_PERF_COUNTER(priv->pstats.lro_misses); + } else { + mb->m_pkthdr.csum_flags = 0; + priv->port_stats.rx_chksum_none++; + } + + /* Push it up the stack */ + dev->if_input(dev, mb); + +next: + ++cons_index; + index = cons_index & size_mask; + cqe = &buf[CQE_FACTOR_INDEX(index, factor)]; + if (++polled == budget) + goto out; + } + /* Flush all pending IP reassembly sessions */ +out: +#ifdef INET + tcp_lro_flush_all(&ring->lro); +#endif + AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled); + mcq->cons_index = cons_index; + mlx4_cq_set_ci(mcq); + wmb(); /* ensure HW sees CQ consumer before we post new buffers */ + ring->cons = mcq->cons_index; + ring->prod += polled; /* Polled descriptors were realocated in place */ + mlx4_en_update_rx_prod_db(ring); + return polled; + +} + +/* Rx CQ polling - called by NAPI */ +static int mlx4_en_poll_rx_cq(struct mlx4_en_cq *cq, int budget) +{ + struct net_device *dev = cq->dev; + int done; + + done = mlx4_en_process_rx_cq(dev, cq, budget); + cq->tot_rx += done; + + return done; + +} +void mlx4_en_rx_irq(struct mlx4_cq *mcq) +{ + struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq); + struct mlx4_en_priv *priv = netdev_priv(cq->dev); + int done; + + // Shoot one within the irq context + // Because there is no NAPI in freeBSD + done = mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET); + if (priv->port_up && (done == MLX4_EN_RX_BUDGET) ) { + cq->curr_poll_rx_cpu_id = curcpu; + taskqueue_enqueue(cq->tq, &cq->cq_task); + } + else { + mlx4_en_arm_cq(priv, cq); + } +} + +void mlx4_en_rx_que(void *context, int pending) +{ + struct mlx4_en_cq *cq; + struct thread *td; + + cq = context; + td = curthread; + + thread_lock(td); + sched_bind(td, cq->curr_poll_rx_cpu_id); + thread_unlock(td); + + while (mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET) + == MLX4_EN_RX_BUDGET); + mlx4_en_arm_cq(cq->dev->if_softc, cq); +} + + +/* RSS related functions */ + +static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn, + struct mlx4_en_rx_ring *ring, + enum mlx4_qp_state *state, + struct mlx4_qp *qp) +{ + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_qp_context *context; + int err = 0; + + context = kmalloc(sizeof *context , GFP_KERNEL); + if (!context) { + en_err(priv, "Failed to allocate qp context\n"); + return -ENOMEM; + } + + err = mlx4_qp_alloc(mdev->dev, qpn, qp, GFP_KERNEL); + if (err) { + en_err(priv, "Failed to allocate qp #%x\n", qpn); + goto out; + } + qp->event = mlx4_en_sqp_event; + + memset(context, 0, sizeof *context); + mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0, + qpn, ring->cqn, -1, context); + context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma); + + /* Cancel FCS removal if FW allows */ + if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) { + context->param3 |= cpu_to_be32(1 << 29); + ring->fcs_del = ETH_FCS_LEN; + } else + ring->fcs_del = 0; + + err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state); + if (err) { + mlx4_qp_remove(mdev->dev, qp); + mlx4_qp_free(mdev->dev, qp); + } + mlx4_en_update_rx_prod_db(ring); +out: + kfree(context); + return err; +} + +int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv) +{ + int err; + u32 qpn; + + err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn, 0); + if (err) { + en_err(priv, "Failed reserving drop qpn\n"); + return err; + } + err = mlx4_qp_alloc(priv->mdev->dev, qpn, &priv->drop_qp, GFP_KERNEL); + if (err) { + en_err(priv, "Failed allocating drop qp\n"); + mlx4_qp_release_range(priv->mdev->dev, qpn, 1); + return err; + } + + return 0; +} + +void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv) +{ + u32 qpn; + + qpn = priv->drop_qp.qpn; + mlx4_qp_remove(priv->mdev->dev, &priv->drop_qp); + mlx4_qp_free(priv->mdev->dev, &priv->drop_qp); + mlx4_qp_release_range(priv->mdev->dev, qpn, 1); +} + +const u32 * +mlx4_en_get_rss_key(struct mlx4_en_priv *priv __unused, + u16 *keylen) +{ + static const u32 rsskey[10] = { + cpu_to_be32(0xD181C62C), + cpu_to_be32(0xF7F4DB5B), + cpu_to_be32(0x1983A2FC), + cpu_to_be32(0x943E1ADB), + cpu_to_be32(0xD9389E6B), + cpu_to_be32(0xD1039C2C), + cpu_to_be32(0xA74499AD), + cpu_to_be32(0x593D56D9), + cpu_to_be32(0xF3253C06), + cpu_to_be32(0x2ADC1FFC) + }; + + if (keylen != NULL) + *keylen = sizeof(rsskey); + return (rsskey); +} + +u8 mlx4_en_get_rss_mask(struct mlx4_en_priv *priv) +{ + u8 rss_mask = (MLX4_RSS_IPV4 | MLX4_RSS_TCP_IPV4 | MLX4_RSS_IPV6 | + MLX4_RSS_TCP_IPV6); + + if (priv->mdev->profile.udp_rss) + rss_mask |= MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6; + return (rss_mask); +} + +/* Allocate rx qp's and configure them according to rss map */ +int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv) +{ + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_rss_map *rss_map = &priv->rss_map; + struct mlx4_qp_context context; + struct mlx4_rss_context *rss_context; + const u32 *key; + int rss_rings; + void *ptr; + int i; + int err = 0; + int good_qps = 0; + + en_dbg(DRV, priv, "Configuring rss steering\n"); + err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num, + priv->rx_ring_num, + &rss_map->base_qpn, 0); + if (err) { + en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num); + return err; + } + + for (i = 0; i < priv->rx_ring_num; i++) { + priv->rx_ring[i]->qpn = rss_map->base_qpn + i; + err = mlx4_en_config_rss_qp(priv, priv->rx_ring[i]->qpn, + priv->rx_ring[i], + &rss_map->state[i], + &rss_map->qps[i]); + if (err) + goto rss_err; + + ++good_qps; + } + + /* Configure RSS indirection qp */ + err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp, GFP_KERNEL); + if (err) { + en_err(priv, "Failed to allocate RSS indirection QP\n"); + goto rss_err; + } + rss_map->indir_qp.event = mlx4_en_sqp_event; + mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn, + priv->rx_ring[0]->cqn, -1, &context); + + if (!priv->prof->rss_rings || priv->prof->rss_rings > priv->rx_ring_num) + rss_rings = priv->rx_ring_num; + else + rss_rings = priv->prof->rss_rings; + + ptr = ((u8 *)&context) + offsetof(struct mlx4_qp_context, pri_path) + + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH; + rss_context = ptr; + rss_context->base_qpn = cpu_to_be32(ilog2(rss_rings) << 24 | + (rss_map->base_qpn)); + rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn); + if (priv->mdev->profile.udp_rss) + rss_context->base_qpn_udp = rss_context->default_qpn; + rss_context->flags = mlx4_en_get_rss_mask(priv); + rss_context->hash_fn = MLX4_RSS_HASH_TOP; + key = mlx4_en_get_rss_key(priv, NULL); + for (i = 0; i < 10; i++) + rss_context->rss_key[i] = key[i]; + + err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context, + &rss_map->indir_qp, &rss_map->indir_state); + if (err) + goto indir_err; + + return 0; + +indir_err: + mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state, + MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp); + mlx4_qp_remove(mdev->dev, &rss_map->indir_qp); + mlx4_qp_free(mdev->dev, &rss_map->indir_qp); +rss_err: + for (i = 0; i < good_qps; i++) { + mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i], + MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]); + mlx4_qp_remove(mdev->dev, &rss_map->qps[i]); + mlx4_qp_free(mdev->dev, &rss_map->qps[i]); + } + mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num); + return err; +} + +void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv) +{ + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_rss_map *rss_map = &priv->rss_map; + int i; + + mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state, + MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp); + mlx4_qp_remove(mdev->dev, &rss_map->indir_qp); + mlx4_qp_free(mdev->dev, &rss_map->indir_qp); + + for (i = 0; i < priv->rx_ring_num; i++) { + mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i], + MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]); + mlx4_qp_remove(mdev->dev, &rss_map->qps[i]); + mlx4_qp_free(mdev->dev, &rss_map->qps[i]); + } + mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num); +} + Property changes on: stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_tx.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_tx.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_tx.c (revision 329159) @@ -0,0 +1,1089 @@ +/* + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#define LINUXKPI_PARAM_PREFIX mlx4_ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "en.h" + +int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring **pring, u32 size, + u16 stride, int node, int queue_idx) +{ + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_tx_ring *ring; + uint32_t x; + int tmp; + int err; + + ring = kzalloc_node(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL, node); + if (!ring) { + ring = kzalloc(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL); + if (!ring) { + en_err(priv, "Failed allocating TX ring\n"); + return -ENOMEM; + } + } + + /* Create DMA descriptor TAG */ + if ((err = -bus_dma_tag_create( + bus_get_dma_tag(mdev->pdev->dev.bsddev), + 1, /* any alignment */ + 0, /* no boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MLX4_EN_TX_MAX_PAYLOAD_SIZE, /* maxsize */ + MLX4_EN_TX_MAX_MBUF_FRAGS, /* nsegments */ + MLX4_EN_TX_MAX_MBUF_SIZE, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockfuncarg */ + &ring->dma_tag))) + goto done; + + ring->size = size; + ring->size_mask = size - 1; + ring->stride = stride; + ring->inline_thold = MAX(MIN_PKT_LEN, MIN(priv->prof->inline_thold, MAX_INLINE)); + mtx_init(&ring->tx_lock.m, "mlx4 tx", NULL, MTX_DEF); + mtx_init(&ring->comp_lock.m, "mlx4 comp", NULL, MTX_DEF); + + /* Allocate the buf ring */ + ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF, + M_WAITOK, &ring->tx_lock.m); + if (ring->br == NULL) { + en_err(priv, "Failed allocating tx_info ring\n"); + err = -ENOMEM; + goto err_free_dma_tag; + } + + tmp = size * sizeof(struct mlx4_en_tx_info); + ring->tx_info = kzalloc_node(tmp, GFP_KERNEL, node); + if (!ring->tx_info) { + ring->tx_info = kzalloc(tmp, GFP_KERNEL); + if (!ring->tx_info) { + err = -ENOMEM; + goto err_ring; + } + } + + /* Create DMA descriptor MAPs */ + for (x = 0; x != size; x++) { + err = -bus_dmamap_create(ring->dma_tag, 0, + &ring->tx_info[x].dma_map); + if (err != 0) { + while (x--) { + bus_dmamap_destroy(ring->dma_tag, + ring->tx_info[x].dma_map); + } + goto err_info; + } + } + + en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n", + ring->tx_info, tmp); + + ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE); + + /* Allocate HW buffers on provided NUMA node */ + err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size, + 2 * PAGE_SIZE); + if (err) { + en_err(priv, "Failed allocating hwq resources\n"); + goto err_dma_map; + } + + err = mlx4_en_map_buffer(&ring->wqres.buf); + if (err) { + en_err(priv, "Failed to map TX buffer\n"); + goto err_hwq_res; + } + + ring->buf = ring->wqres.buf.direct.buf; + + en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d " + "buf_size:%d dma:%llx\n", ring, ring->buf, ring->size, + ring->buf_size, (unsigned long long) ring->wqres.buf.direct.map); + + err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn, + MLX4_RESERVE_ETH_BF_QP); + if (err) { + en_err(priv, "failed reserving qp for TX ring\n"); + goto err_map; + } + + err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp, GFP_KERNEL); + if (err) { + en_err(priv, "Failed allocating qp %d\n", ring->qpn); + goto err_reserve; + } + ring->qp.event = mlx4_en_sqp_event; + + err = mlx4_bf_alloc(mdev->dev, &ring->bf, node); + if (err) { + en_dbg(DRV, priv, "working without blueflame (%d)", err); + ring->bf.uar = &mdev->priv_uar; + ring->bf.uar->map = mdev->uar_map; + ring->bf_enabled = false; + } else + ring->bf_enabled = true; + ring->queue_index = queue_idx; + + *pring = ring; + return 0; + +err_reserve: + mlx4_qp_release_range(mdev->dev, ring->qpn, 1); +err_map: + mlx4_en_unmap_buffer(&ring->wqres.buf); +err_hwq_res: + mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); +err_dma_map: + for (x = 0; x != size; x++) + bus_dmamap_destroy(ring->dma_tag, ring->tx_info[x].dma_map); +err_info: + vfree(ring->tx_info); +err_ring: + buf_ring_free(ring->br, M_DEVBUF); +err_free_dma_tag: + bus_dma_tag_destroy(ring->dma_tag); +done: + kfree(ring); + return err; +} + +void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring **pring) +{ + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_tx_ring *ring = *pring; + uint32_t x; + en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn); + + buf_ring_free(ring->br, M_DEVBUF); + if (ring->bf_enabled) + mlx4_bf_free(mdev->dev, &ring->bf); + mlx4_qp_remove(mdev->dev, &ring->qp); + mlx4_qp_free(mdev->dev, &ring->qp); + mlx4_qp_release_range(priv->mdev->dev, ring->qpn, 1); + mlx4_en_unmap_buffer(&ring->wqres.buf); + mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); + for (x = 0; x != ring->size; x++) + bus_dmamap_destroy(ring->dma_tag, ring->tx_info[x].dma_map); + vfree(ring->tx_info); + mtx_destroy(&ring->tx_lock.m); + mtx_destroy(&ring->comp_lock.m); + bus_dma_tag_destroy(ring->dma_tag); + kfree(ring); + *pring = NULL; +} + +int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring *ring, + int cq, int user_prio) +{ + struct mlx4_en_dev *mdev = priv->mdev; + int err; + + ring->cqn = cq; + ring->prod = 0; + ring->cons = 0xffffffff; + ring->last_nr_txbb = 1; + ring->poll_cnt = 0; + ring->blocked = 0; + memset(ring->buf, 0, ring->buf_size); + + ring->qp_state = MLX4_QP_STATE_RST; + ring->doorbell_qpn = ring->qp.qpn << 8; + + mlx4_en_fill_qp_context(priv, ring->size, ring->stride, 1, 0, ring->qpn, + ring->cqn, user_prio, &ring->context); + if (ring->bf_enabled) + ring->context.usr_page = cpu_to_be32(ring->bf.uar->index); + + err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, &ring->context, + &ring->qp, &ring->qp_state); + return err; +} + +void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring *ring) +{ + struct mlx4_en_dev *mdev = priv->mdev; + + mlx4_qp_modify(mdev->dev, NULL, ring->qp_state, + MLX4_QP_STATE_RST, NULL, 0, 0, &ring->qp); +} + +static volatile struct mlx4_wqe_data_seg * +mlx4_en_store_inline_lso_data(volatile struct mlx4_wqe_data_seg *dseg, + struct mbuf *mb, int len, __be32 owner_bit) +{ + uint8_t *inl = __DEVOLATILE(uint8_t *, dseg); + + /* copy data into place */ + m_copydata(mb, 0, len, inl + 4); + dseg += DIV_ROUND_UP(4 + len, DS_SIZE_ALIGNMENT); + return (dseg); +} + +static void +mlx4_en_store_inline_lso_header(volatile struct mlx4_wqe_data_seg *dseg, + int len, __be32 owner_bit) +{ +} + +static void +mlx4_en_stamp_wqe(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring *ring, u32 index, u8 owner) +{ + struct mlx4_en_tx_info *tx_info = &ring->tx_info[index]; + struct mlx4_en_tx_desc *tx_desc = (struct mlx4_en_tx_desc *) + (ring->buf + (index * TXBB_SIZE)); + volatile __be32 *ptr = (__be32 *)tx_desc; + const __be32 stamp = cpu_to_be32(STAMP_VAL | + ((u32)owner << STAMP_SHIFT)); + u32 i; + + /* Stamp the freed descriptor */ + for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) { + *ptr = stamp; + ptr += STAMP_DWORDS; + } +} + +static u32 +mlx4_en_free_tx_desc(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring *ring, u32 index) +{ + struct mlx4_en_tx_info *tx_info; + struct mbuf *mb; + + tx_info = &ring->tx_info[index]; + mb = tx_info->mb; + + if (mb == NULL) + goto done; + + bus_dmamap_sync(ring->dma_tag, tx_info->dma_map, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(ring->dma_tag, tx_info->dma_map); + + m_freem(mb); +done: + return (tx_info->nr_txbb); +} + +int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + int cnt = 0; + + /* Skip last polled descriptor */ + ring->cons += ring->last_nr_txbb; + en_dbg(DRV, priv, "Freeing Tx buf - cons:0x%x prod:0x%x\n", + ring->cons, ring->prod); + + if ((u32) (ring->prod - ring->cons) > ring->size) { + en_warn(priv, "Tx consumer passed producer!\n"); + return 0; + } + + while (ring->cons != ring->prod) { + ring->last_nr_txbb = mlx4_en_free_tx_desc(priv, ring, + ring->cons & ring->size_mask); + ring->cons += ring->last_nr_txbb; + cnt++; + } + + if (cnt) + en_dbg(DRV, priv, "Freed %d uncompleted tx descriptors\n", cnt); + + return cnt; +} + +static bool +mlx4_en_tx_ring_is_full(struct mlx4_en_tx_ring *ring) +{ + int wqs; + wqs = ring->size - (ring->prod - ring->cons); + return (wqs < (HEADROOM + (2 * MLX4_EN_TX_WQE_MAX_WQEBBS))); +} + +static int mlx4_en_process_tx_cq(struct net_device *dev, + struct mlx4_en_cq *cq) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_cq *mcq = &cq->mcq; + struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring]; + struct mlx4_cqe *cqe; + u16 index; + u16 new_index, ring_index, stamp_index; + u32 txbbs_skipped = 0; + u32 txbbs_stamp = 0; + u32 cons_index = mcq->cons_index; + int size = cq->size; + u32 size_mask = ring->size_mask; + struct mlx4_cqe *buf = cq->buf; + int factor = priv->cqe_factor; + + if (!priv->port_up) + return 0; + + index = cons_index & size_mask; + cqe = &buf[(index << factor) + factor]; + ring_index = ring->cons & size_mask; + stamp_index = ring_index; + + /* Process all completed CQEs */ + while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, + cons_index & size)) { + /* + * make sure we read the CQE after we read the + * ownership bit + */ + rmb(); + + if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == + MLX4_CQE_OPCODE_ERROR)) { + en_err(priv, "CQE completed in error - vendor syndrom: 0x%x syndrom: 0x%x\n", + ((struct mlx4_err_cqe *)cqe)-> + vendor_err_syndrome, + ((struct mlx4_err_cqe *)cqe)->syndrome); + } + + /* Skip over last polled CQE */ + new_index = be16_to_cpu(cqe->wqe_index) & size_mask; + + do { + txbbs_skipped += ring->last_nr_txbb; + ring_index = (ring_index + ring->last_nr_txbb) & size_mask; + /* free next descriptor */ + ring->last_nr_txbb = mlx4_en_free_tx_desc( + priv, ring, ring_index); + mlx4_en_stamp_wqe(priv, ring, stamp_index, + !!((ring->cons + txbbs_stamp) & + ring->size)); + stamp_index = ring_index; + txbbs_stamp = txbbs_skipped; + } while (ring_index != new_index); + + ++cons_index; + index = cons_index & size_mask; + cqe = &buf[(index << factor) + factor]; + } + + + /* + * To prevent CQ overflow we first update CQ consumer and only then + * the ring consumer. + */ + mcq->cons_index = cons_index; + mlx4_cq_set_ci(mcq); + wmb(); + ring->cons += txbbs_skipped; + + /* Wakeup Tx queue if it was stopped and ring is not full */ + if (unlikely(ring->blocked) && !mlx4_en_tx_ring_is_full(ring)) { + ring->blocked = 0; + if (atomic_fetchadd_int(&priv->blocked, -1) == 1) + atomic_clear_int(&dev->if_drv_flags ,IFF_DRV_OACTIVE); + priv->port_stats.wake_queue++; + ring->wake_queue++; + } + return (0); +} + +void mlx4_en_tx_irq(struct mlx4_cq *mcq) +{ + struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq); + struct mlx4_en_priv *priv = netdev_priv(cq->dev); + struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring]; + + if (priv->port_up == 0 || !spin_trylock(&ring->comp_lock)) + return; + mlx4_en_process_tx_cq(cq->dev, cq); + mod_timer(&cq->timer, jiffies + 1); + spin_unlock(&ring->comp_lock); +} + +void mlx4_en_poll_tx_cq(unsigned long data) +{ + struct mlx4_en_cq *cq = (struct mlx4_en_cq *) data; + struct mlx4_en_priv *priv = netdev_priv(cq->dev); + struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring]; + u32 inflight; + + INC_PERF_COUNTER(priv->pstats.tx_poll); + + if (priv->port_up == 0) + return; + if (!spin_trylock(&ring->comp_lock)) { + mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT); + return; + } + mlx4_en_process_tx_cq(cq->dev, cq); + inflight = (u32) (ring->prod - ring->cons - ring->last_nr_txbb); + + /* If there are still packets in flight and the timer has not already + * been scheduled by the Tx routine then schedule it here to guarantee + * completion processing of these packets */ + if (inflight && priv->port_up) + mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT); + + spin_unlock(&ring->comp_lock); +} + +static inline void mlx4_en_xmit_poll(struct mlx4_en_priv *priv, int tx_ind) +{ + struct mlx4_en_cq *cq = priv->tx_cq[tx_ind]; + struct mlx4_en_tx_ring *ring = priv->tx_ring[tx_ind]; + + if (priv->port_up == 0) + return; + + /* If we don't have a pending timer, set one up to catch our recent + post in case the interface becomes idle */ + if (!timer_pending(&cq->timer)) + mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT); + + /* Poll the CQ every mlx4_en_TX_MODER_POLL packets */ + if ((++ring->poll_cnt & (MLX4_EN_TX_POLL_MODER - 1)) == 0) + if (spin_trylock(&ring->comp_lock)) { + mlx4_en_process_tx_cq(priv->dev, cq); + spin_unlock(&ring->comp_lock); + } +} + +static u16 +mlx4_en_get_inline_hdr_size(struct mlx4_en_tx_ring *ring, struct mbuf *mb) +{ + u16 retval; + + /* only copy from first fragment, if possible */ + retval = MIN(ring->inline_thold, mb->m_len); + + /* check for too little data */ + if (unlikely(retval < MIN_PKT_LEN)) + retval = MIN(ring->inline_thold, mb->m_pkthdr.len); + return (retval); +} + +static int +mlx4_en_get_header_size(struct mbuf *mb) +{ + struct ether_vlan_header *eh; + struct tcphdr *th; + struct ip *ip; + int ip_hlen, tcp_hlen; + struct ip6_hdr *ip6; + uint16_t eth_type; + int eth_hdr_len; + + eh = mtod(mb, struct ether_vlan_header *); + if (mb->m_len < ETHER_HDR_LEN) + return (0); + if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { + eth_type = ntohs(eh->evl_proto); + eth_hdr_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; + } else { + eth_type = ntohs(eh->evl_encap_proto); + eth_hdr_len = ETHER_HDR_LEN; + } + if (mb->m_len < eth_hdr_len) + return (0); + switch (eth_type) { + case ETHERTYPE_IP: + ip = (struct ip *)(mb->m_data + eth_hdr_len); + if (mb->m_len < eth_hdr_len + sizeof(*ip)) + return (0); + if (ip->ip_p != IPPROTO_TCP) + return (0); + ip_hlen = ip->ip_hl << 2; + eth_hdr_len += ip_hlen; + break; + case ETHERTYPE_IPV6: + ip6 = (struct ip6_hdr *)(mb->m_data + eth_hdr_len); + if (mb->m_len < eth_hdr_len + sizeof(*ip6)) + return (0); + if (ip6->ip6_nxt != IPPROTO_TCP) + return (0); + eth_hdr_len += sizeof(*ip6); + break; + default: + return (0); + } + if (mb->m_len < eth_hdr_len + sizeof(*th)) + return (0); + th = (struct tcphdr *)(mb->m_data + eth_hdr_len); + tcp_hlen = th->th_off << 2; + eth_hdr_len += tcp_hlen; + if (mb->m_len < eth_hdr_len) + return (0); + return (eth_hdr_len); +} + +static volatile struct mlx4_wqe_data_seg * +mlx4_en_store_inline_data(volatile struct mlx4_wqe_data_seg *dseg, + struct mbuf *mb, int len, __be32 owner_bit) +{ + uint8_t *inl = __DEVOLATILE(uint8_t *, dseg); + const int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - 4; + + if (unlikely(len < MIN_PKT_LEN)) { + m_copydata(mb, 0, len, inl + 4); + memset(inl + 4 + len, 0, MIN_PKT_LEN - len); + dseg += DIV_ROUND_UP(4 + MIN_PKT_LEN, DS_SIZE_ALIGNMENT); + } else if (len <= spc) { + m_copydata(mb, 0, len, inl + 4); + dseg += DIV_ROUND_UP(4 + len, DS_SIZE_ALIGNMENT); + } else { + m_copydata(mb, 0, spc, inl + 4); + m_copydata(mb, spc, len - spc, inl + 8 + spc); + dseg += DIV_ROUND_UP(8 + len, DS_SIZE_ALIGNMENT); + } + return (dseg); +} + +static void +mlx4_en_store_inline_header(volatile struct mlx4_wqe_data_seg *dseg, + int len, __be32 owner_bit) +{ + uint8_t *inl = __DEVOLATILE(uint8_t *, dseg); + const int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - 4; + + if (unlikely(len < MIN_PKT_LEN)) { + *(volatile uint32_t *)inl = + SET_BYTE_COUNT((1 << 31) | MIN_PKT_LEN); + } else if (len <= spc) { + *(volatile uint32_t *)inl = + SET_BYTE_COUNT((1 << 31) | len); + } else { + *(volatile uint32_t *)(inl + 4 + spc) = + SET_BYTE_COUNT((1 << 31) | (len - spc)); + wmb(); + *(volatile uint32_t *)inl = + SET_BYTE_COUNT((1 << 31) | spc); + } +} + +static uint32_t hashrandom; +static void hashrandom_init(void *arg) +{ + /* + * It is assumed that the random subsystem has been + * initialized when this function is called: + */ + hashrandom = m_ether_tcpip_hash_init(); +} +SYSINIT(hashrandom_init, SI_SUB_RANDOM, SI_ORDER_ANY, &hashrandom_init, NULL); + +u16 mlx4_en_select_queue(struct net_device *dev, struct mbuf *mb) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + u32 rings_p_up = priv->num_tx_rings_p_up; + u32 up = 0; + u32 queue_index; + +#if (MLX4_EN_NUM_UP > 1) + /* Obtain VLAN information if present */ + if (mb->m_flags & M_VLANTAG) { + u32 vlan_tag = mb->m_pkthdr.ether_vtag; + up = (vlan_tag >> 13) % MLX4_EN_NUM_UP; + } +#endif + queue_index = m_ether_tcpip_hash(MBUF_HASHFLAG_L3 | MBUF_HASHFLAG_L4, mb, hashrandom); + + return ((queue_index % rings_p_up) + (up * rings_p_up)); +} + +static void mlx4_bf_copy(void __iomem *dst, volatile unsigned long *src, unsigned bytecnt) +{ + __iowrite64_copy(dst, __DEVOLATILE(void *, src), bytecnt / 8); +} + +static int mlx4_en_xmit(struct mlx4_en_priv *priv, int tx_ind, struct mbuf **mbp) +{ + enum { + DS_FACT = TXBB_SIZE / DS_SIZE_ALIGNMENT, + CTRL_FLAGS = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE | + MLX4_WQE_CTRL_SOLICITED), + }; + bus_dma_segment_t segs[MLX4_EN_TX_MAX_MBUF_FRAGS]; + volatile struct mlx4_wqe_data_seg *dseg; + volatile struct mlx4_wqe_data_seg *dseg_inline; + volatile struct mlx4_en_tx_desc *tx_desc; + struct mlx4_en_tx_ring *ring = priv->tx_ring[tx_ind]; + struct ifnet *ifp = priv->dev; + struct mlx4_en_tx_info *tx_info; + struct mbuf *mb = *mbp; + struct mbuf *m; + __be32 owner_bit; + int nr_segs; + int pad; + int err; + u32 bf_size; + u32 bf_prod; + u32 opcode; + u16 index; + u16 ds_cnt; + u16 ihs; + + if (unlikely(!priv->port_up)) { + err = EINVAL; + goto tx_drop; + } + + /* check if TX ring is full */ + if (unlikely(mlx4_en_tx_ring_is_full(ring))) { + /* every full native Tx ring stops queue */ + if (ring->blocked == 0) + atomic_add_int(&priv->blocked, 1); + /* Set HW-queue-is-full flag */ + atomic_set_int(&ifp->if_drv_flags, IFF_DRV_OACTIVE); + priv->port_stats.queue_stopped++; + ring->blocked = 1; + ring->queue_stopped++; + + /* Use interrupts to find out when queue opened */ + mlx4_en_arm_cq(priv, priv->tx_cq[tx_ind]); + return (ENOBUFS); + } + + /* sanity check we are not wrapping around */ + KASSERT(((~ring->prod) & ring->size_mask) >= + (MLX4_EN_TX_WQE_MAX_WQEBBS - 1), ("Wrapping around TX ring")); + + /* Track current inflight packets for performance analysis */ + AVG_PERF_COUNTER(priv->pstats.inflight_avg, + (u32) (ring->prod - ring->cons - 1)); + + /* Track current mbuf packet header length */ + AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, mb->m_pkthdr.len); + + /* Grab an index and try to transmit packet */ + owner_bit = (ring->prod & ring->size) ? + cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0; + index = ring->prod & ring->size_mask; + tx_desc = (volatile struct mlx4_en_tx_desc *) + (ring->buf + index * TXBB_SIZE); + tx_info = &ring->tx_info[index]; + dseg = &tx_desc->data; + + /* send a copy of the frame to the BPF listener, if any */ + if (ifp != NULL && ifp->if_bpf != NULL) + ETHER_BPF_MTAP(ifp, mb); + + /* get default flags */ + tx_desc->ctrl.srcrb_flags = CTRL_FLAGS; + + if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) + tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM); + + if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | + CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) + tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_TCP_UDP_CSUM); + + /* do statistics */ + if (likely(tx_desc->ctrl.srcrb_flags != CTRL_FLAGS)) { + priv->port_stats.tx_chksum_offload++; + ring->tx_csum++; + } + + /* check for VLAN tag */ + if (mb->m_flags & M_VLANTAG) { + tx_desc->ctrl.vlan_tag = cpu_to_be16(mb->m_pkthdr.ether_vtag); + tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_CVLAN; + } else { + tx_desc->ctrl.vlan_tag = 0; + tx_desc->ctrl.ins_vlan = 0; + } + + if (unlikely(mlx4_is_mfunc(priv->mdev->dev) || priv->validate_loopback)) { + /* + * Copy destination MAC address to WQE. This allows + * loopback in eSwitch, so that VFs and PF can + * communicate with each other: + */ + m_copydata(mb, 0, 2, __DEVOLATILE(void *, &tx_desc->ctrl.srcrb_flags16[0])); + m_copydata(mb, 2, 4, __DEVOLATILE(void *, &tx_desc->ctrl.imm)); + } else { + /* clear immediate field */ + tx_desc->ctrl.imm = 0; + } + + /* Handle LSO (TSO) packets */ + if (mb->m_pkthdr.csum_flags & CSUM_TSO) { + u32 payload_len; + u32 mss = mb->m_pkthdr.tso_segsz; + u32 num_pkts; + + opcode = cpu_to_be32(MLX4_OPCODE_LSO | MLX4_WQE_CTRL_RR) | + owner_bit; + ihs = mlx4_en_get_header_size(mb); + if (unlikely(ihs > MAX_INLINE)) { + ring->oversized_packets++; + err = EINVAL; + goto tx_drop; + } + tx_desc->lso.mss_hdr_size = cpu_to_be32((mss << 16) | ihs); + payload_len = mb->m_pkthdr.len - ihs; + if (unlikely(payload_len == 0)) + num_pkts = 1; + else + num_pkts = DIV_ROUND_UP(payload_len, mss); + ring->bytes += payload_len + (num_pkts * ihs); + ring->packets += num_pkts; + ring->tso_packets++; + /* store pointer to inline header */ + dseg_inline = dseg; + /* copy data inline */ + dseg = mlx4_en_store_inline_lso_data(dseg, + mb, ihs, owner_bit); + } else { + opcode = cpu_to_be32(MLX4_OPCODE_SEND) | + owner_bit; + ihs = mlx4_en_get_inline_hdr_size(ring, mb); + ring->bytes += max_t (unsigned int, + mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN); + ring->packets++; + /* store pointer to inline header */ + dseg_inline = dseg; + /* copy data inline */ + dseg = mlx4_en_store_inline_data(dseg, + mb, ihs, owner_bit); + } + m_adj(mb, ihs); + + err = bus_dmamap_load_mbuf_sg(ring->dma_tag, tx_info->dma_map, + mb, segs, &nr_segs, BUS_DMA_NOWAIT); + if (unlikely(err == EFBIG)) { + /* Too many mbuf fragments */ + ring->defrag_attempts++; + m = m_defrag(mb, M_NOWAIT); + if (m == NULL) { + ring->oversized_packets++; + goto tx_drop; + } + mb = m; + /* Try again */ + err = bus_dmamap_load_mbuf_sg(ring->dma_tag, tx_info->dma_map, + mb, segs, &nr_segs, BUS_DMA_NOWAIT); + } + /* catch errors */ + if (unlikely(err != 0)) { + ring->oversized_packets++; + goto tx_drop; + } + /* If there were no errors and we didn't load anything, don't sync. */ + if (nr_segs != 0) { + /* make sure all mbuf data is written to RAM */ + bus_dmamap_sync(ring->dma_tag, tx_info->dma_map, + BUS_DMASYNC_PREWRITE); + } else { + /* All data was inlined, free the mbuf. */ + bus_dmamap_unload(ring->dma_tag, tx_info->dma_map); + m_freem(mb); + mb = NULL; + } + + /* compute number of DS needed */ + ds_cnt = (dseg - ((volatile struct mlx4_wqe_data_seg *)tx_desc)) + nr_segs; + + /* + * Check if the next request can wrap around and fill the end + * of the current request with zero immediate data: + */ + pad = DIV_ROUND_UP(ds_cnt, DS_FACT); + pad = (~(ring->prod + pad)) & ring->size_mask; + + if (unlikely(pad < (MLX4_EN_TX_WQE_MAX_WQEBBS - 1))) { + /* + * Compute the least number of DS blocks we need to + * pad in order to achieve a TX ring wraparound: + */ + pad = (DS_FACT * (pad + 1)); + } else { + /* + * The hardware will automatically jump to the next + * TXBB. No need for padding. + */ + pad = 0; + } + + /* compute total number of DS blocks */ + ds_cnt += pad; + /* + * When modifying this code, please ensure that the following + * computation is always less than or equal to 0x3F: + * + * ((MLX4_EN_TX_WQE_MAX_WQEBBS - 1) * DS_FACT) + + * (MLX4_EN_TX_WQE_MAX_WQEBBS * DS_FACT) + * + * Else the "ds_cnt" variable can become too big. + */ + tx_desc->ctrl.fence_size = (ds_cnt & 0x3f); + + /* store pointer to mbuf */ + tx_info->mb = mb; + tx_info->nr_txbb = DIV_ROUND_UP(ds_cnt, DS_FACT); + bf_size = ds_cnt * DS_SIZE_ALIGNMENT; + bf_prod = ring->prod; + + /* compute end of "dseg" array */ + dseg += nr_segs + pad; + + /* pad using zero immediate dseg */ + while (pad--) { + dseg--; + dseg->addr = 0; + dseg->lkey = 0; + wmb(); + dseg->byte_count = SET_BYTE_COUNT((1 << 31)|0); + } + + /* fill segment list */ + while (nr_segs--) { + if (unlikely(segs[nr_segs].ds_len == 0)) { + dseg--; + dseg->addr = 0; + dseg->lkey = 0; + wmb(); + dseg->byte_count = SET_BYTE_COUNT((1 << 31)|0); + } else { + dseg--; + dseg->addr = cpu_to_be64((uint64_t)segs[nr_segs].ds_addr); + dseg->lkey = cpu_to_be32(priv->mdev->mr.key); + wmb(); + dseg->byte_count = SET_BYTE_COUNT((uint32_t)segs[nr_segs].ds_len); + } + } + + wmb(); + + /* write owner bits in reverse order */ + if ((opcode & cpu_to_be32(0x1F)) == cpu_to_be32(MLX4_OPCODE_LSO)) + mlx4_en_store_inline_lso_header(dseg_inline, ihs, owner_bit); + else + mlx4_en_store_inline_header(dseg_inline, ihs, owner_bit); + + /* update producer counter */ + ring->prod += tx_info->nr_txbb; + + if (ring->bf_enabled && bf_size <= MAX_BF && + (tx_desc->ctrl.ins_vlan != MLX4_WQE_CTRL_INS_CVLAN)) { + + /* store doorbell number */ + *(volatile __be32 *) (&tx_desc->ctrl.vlan_tag) |= cpu_to_be32(ring->doorbell_qpn); + + /* or in producer number for this WQE */ + opcode |= cpu_to_be32((bf_prod & 0xffff) << 8); + + /* + * Ensure the new descriptor hits memory before + * setting ownership of this descriptor to HW: + */ + wmb(); + tx_desc->ctrl.owner_opcode = opcode; + wmb(); + mlx4_bf_copy(((u8 *)ring->bf.reg) + ring->bf.offset, + (volatile unsigned long *) &tx_desc->ctrl, bf_size); + wmb(); + ring->bf.offset ^= ring->bf.buf_size; + } else { + /* + * Ensure the new descriptor hits memory before + * setting ownership of this descriptor to HW: + */ + wmb(); + tx_desc->ctrl.owner_opcode = opcode; + wmb(); + writel(cpu_to_be32(ring->doorbell_qpn), + ((u8 *)ring->bf.uar->map) + MLX4_SEND_DOORBELL); + } + + return (0); +tx_drop: + *mbp = NULL; + m_freem(mb); + return (err); +} + +static int +mlx4_en_transmit_locked(struct ifnet *dev, int tx_ind, struct mbuf *m) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_tx_ring *ring; + struct mbuf *next; + int enqueued, err = 0; + + ring = priv->tx_ring[tx_ind]; + if ((dev->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != + IFF_DRV_RUNNING || priv->port_up == 0) { + if (m != NULL) + err = drbr_enqueue(dev, ring->br, m); + return (err); + } + + enqueued = 0; + if (m != NULL) + /* + * If we can't insert mbuf into drbr, try to xmit anyway. + * We keep the error we got so we could return that after xmit. + */ + err = drbr_enqueue(dev, ring->br, m); + + /* Process the queue */ + while ((next = drbr_peek(dev, ring->br)) != NULL) { + if (mlx4_en_xmit(priv, tx_ind, &next) != 0) { + if (next == NULL) { + drbr_advance(dev, ring->br); + } else { + drbr_putback(dev, ring->br, next); + } + break; + } + drbr_advance(dev, ring->br); + enqueued++; + if ((dev->if_drv_flags & IFF_DRV_RUNNING) == 0) + break; + } + + if (enqueued > 0) + ring->watchdog_time = ticks; + + return (err); +} + +void +mlx4_en_tx_que(void *context, int pending) +{ + struct mlx4_en_tx_ring *ring; + struct mlx4_en_priv *priv; + struct net_device *dev; + struct mlx4_en_cq *cq; + int tx_ind; + cq = context; + dev = cq->dev; + priv = dev->if_softc; + tx_ind = cq->ring; + ring = priv->tx_ring[tx_ind]; + + if (priv->port_up != 0 && + (dev->if_drv_flags & IFF_DRV_RUNNING) != 0) { + mlx4_en_xmit_poll(priv, tx_ind); + spin_lock(&ring->tx_lock); + if (!drbr_empty(dev, ring->br)) + mlx4_en_transmit_locked(dev, tx_ind, NULL); + spin_unlock(&ring->tx_lock); + } +} + +int +mlx4_en_transmit(struct ifnet *dev, struct mbuf *m) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_tx_ring *ring; + struct mlx4_en_cq *cq; + int i, err = 0; + + if (priv->port_up == 0) { + m_freem(m); + return (ENETDOWN); + } + + /* Compute which queue to use */ + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { + i = (m->m_pkthdr.flowid % 128) % priv->tx_ring_num; + } + else { + i = mlx4_en_select_queue(dev, m); + } + + ring = priv->tx_ring[i]; + if (spin_trylock(&ring->tx_lock)) { + err = mlx4_en_transmit_locked(dev, i, m); + spin_unlock(&ring->tx_lock); + /* Poll CQ here */ + mlx4_en_xmit_poll(priv, i); + } else { + err = drbr_enqueue(dev, ring->br, m); + cq = priv->tx_cq[i]; + taskqueue_enqueue(cq->tq, &cq->cq_task); + } + +#if __FreeBSD_version >= 1100000 + if (unlikely(err != 0)) + if_inc_counter(dev, IFCOUNTER_IQDROPS, 1); +#endif + return (err); +} + +/* + * Flush ring buffers. + */ +void +mlx4_en_qflush(struct ifnet *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_tx_ring *ring; + struct mbuf *m; + + if (priv->port_up == 0) + return; + + for (int i = 0; i < priv->tx_ring_num; i++) { + ring = priv->tx_ring[i]; + spin_lock(&ring->tx_lock); + while ((m = buf_ring_dequeue_sc(ring->br)) != NULL) + m_freem(m); + spin_unlock(&ring->tx_lock); + } + if_qflush(dev); +} Property changes on: stable/11/sys/dev/mlx4/mlx4_en/mlx4_en_tx.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib.h =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib.h (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib.h (revision 329159) @@ -0,0 +1,862 @@ +/* + * Copyright (c) 2006, 2007 Cisco Systems. All rights reserved. + * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MLX4_IB_H +#define MLX4_IB_H + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#define MLX4_IB_DRV_NAME "mlx4_ib" + +#ifdef pr_fmt +#undef pr_fmt +#endif +#define pr_fmt(fmt) "<" MLX4_IB_DRV_NAME "> %s: " fmt, __func__ + +#define mlx4_ib_warn(ibdev, format, arg...) \ + dev_warn((ibdev)->dma_device, MLX4_IB_DRV_NAME ": " format, ## arg) + +#define mlx4_ib_info(ibdev, format, arg...) \ + dev_info((ibdev)->dma_device, MLX4_IB_DRV_NAME ": " format, ## arg) + +enum { + MLX4_IB_SQ_MIN_WQE_SHIFT = 6, + MLX4_IB_MAX_HEADROOM = 2048 +}; + +#define MLX4_IB_SQ_HEADROOM(shift) ((MLX4_IB_MAX_HEADROOM >> (shift)) + 1) +#define MLX4_IB_SQ_MAX_SPARE (MLX4_IB_SQ_HEADROOM(MLX4_IB_SQ_MIN_WQE_SHIFT)) + +/*module param to indicate if SM assigns the alias_GUID*/ +extern int mlx4_ib_sm_guid_assign; +extern struct proc_dir_entry *mlx4_mrs_dir_entry; + +#define MLX4_IB_UC_STEER_QPN_ALIGN 1 +#define MLX4_IB_UC_MAX_NUM_QPS (256 * 1024) + + +#define MLX4_IB_MMAP_CMD_MASK 0xFF +#define MLX4_IB_MMAP_CMD_BITS 8 + +struct mlx4_ib_ucontext { + struct ib_ucontext ibucontext; + struct mlx4_uar uar; + struct list_head db_page_list; + struct mutex db_page_mutex; +}; + +struct mlx4_ib_pd { + struct ib_pd ibpd; + u32 pdn; +}; + +struct mlx4_ib_xrcd { + struct ib_xrcd ibxrcd; + u32 xrcdn; + struct ib_pd *pd; + struct ib_cq *cq; +}; + +struct mlx4_ib_cq_buf { + struct mlx4_buf buf; + struct mlx4_mtt mtt; + int entry_size; +}; + +struct mlx4_ib_cq_resize { + struct mlx4_ib_cq_buf buf; + int cqe; +}; + +struct mlx4_shared_mr_info { + int mr_id; + struct ib_umem *umem; +}; + +struct mlx4_ib_cq { + struct ib_cq ibcq; + struct mlx4_cq mcq; + struct mlx4_ib_cq_buf buf; + struct mlx4_ib_cq_resize *resize_buf; + struct mlx4_db db; + spinlock_t lock; + struct mutex resize_mutex; + struct ib_umem *umem; + struct ib_umem *resize_umem; + int create_flags; +}; + +struct mlx4_ib_mr { + struct ib_mr ibmr; + struct mlx4_mr mmr; + struct ib_umem *umem; + struct mlx4_shared_mr_info *smr_info; + atomic_t invalidated; + struct completion invalidation_comp; +}; + +struct mlx4_ib_mw { + struct ib_mw ibmw; + struct mlx4_mw mmw; +}; + +struct mlx4_ib_fast_reg_page_list { + struct ib_fast_reg_page_list ibfrpl; + __be64 *mapped_page_list; + dma_addr_t map; +}; + +struct mlx4_ib_fmr { + struct ib_fmr ibfmr; + struct mlx4_fmr mfmr; +}; + +struct mlx4_ib_flow { + struct ib_flow ibflow; + /* translating DMFS verbs sniffer rule to FW API requires two reg IDs */ + u64 reg_id[2]; +}; + +struct mlx4_ib_wq { + u64 *wrid; + spinlock_t lock; + int wqe_cnt; + int max_post; + int max_gs; + int offset; + int wqe_shift; + unsigned head; + unsigned tail; +}; + +enum mlx4_ib_qp_flags { + MLX4_IB_QP_LSO = IB_QP_CREATE_IPOIB_UD_LSO, + MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK = IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK, + MLX4_IB_QP_CAP_CROSS_CHANNEL = IB_QP_CREATE_CROSS_CHANNEL, + MLX4_IB_QP_CAP_MANAGED_SEND = IB_QP_CREATE_MANAGED_SEND, + MLX4_IB_QP_CAP_MANAGED_RECV = IB_QP_CREATE_MANAGED_RECV, + MLX4_IB_QP_NETIF = IB_QP_CREATE_NETIF_QP, + MLX4_IB_SRIOV_TUNNEL_QP = 1 << 30, + MLX4_IB_SRIOV_SQP = 1 << 31, +}; + +struct mlx4_ib_gid_entry { + struct list_head list; + union ib_gid gid; + int added; + u8 port; +}; + +enum mlx4_ib_mmap_cmd { + MLX4_IB_MMAP_UAR_PAGE = 0, + MLX4_IB_MMAP_BLUE_FLAME_PAGE = 1, + MLX4_IB_MMAP_GET_CONTIGUOUS_PAGES = 2, + MLX4_IB_MMAP_GET_HW_CLOCK = 3, +}; + +enum mlx4_ib_qp_type { + /* + * IB_QPT_SMI and IB_QPT_GSI have to be the first two entries + * here (and in that order) since the MAD layer uses them as + * indices into a 2-entry table. + */ + MLX4_IB_QPT_SMI = IB_QPT_SMI, + MLX4_IB_QPT_GSI = IB_QPT_GSI, + + MLX4_IB_QPT_RC = IB_QPT_RC, + MLX4_IB_QPT_UC = IB_QPT_UC, + MLX4_IB_QPT_UD = IB_QPT_UD, + MLX4_IB_QPT_RAW_IPV6 = IB_QPT_RAW_IPV6, + MLX4_IB_QPT_RAW_ETHERTYPE = IB_QPT_RAW_ETHERTYPE, + MLX4_IB_QPT_RAW_PACKET = IB_QPT_RAW_PACKET, + MLX4_IB_QPT_XRC_INI = IB_QPT_XRC_INI, + MLX4_IB_QPT_XRC_TGT = IB_QPT_XRC_TGT, + + MLX4_IB_QPT_PROXY_SMI_OWNER = 1 << 16, + MLX4_IB_QPT_PROXY_SMI = 1 << 17, + MLX4_IB_QPT_PROXY_GSI = 1 << 18, + MLX4_IB_QPT_TUN_SMI_OWNER = 1 << 19, + MLX4_IB_QPT_TUN_SMI = 1 << 20, + MLX4_IB_QPT_TUN_GSI = 1 << 21, +}; + +#define MLX4_IB_QPT_ANY_SRIOV (MLX4_IB_QPT_PROXY_SMI_OWNER | \ + MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER | \ + MLX4_IB_QPT_TUN_SMI | MLX4_IB_QPT_TUN_GSI) + +enum mlx4_ib_mad_ifc_flags { + MLX4_MAD_IFC_IGNORE_MKEY = 1, + MLX4_MAD_IFC_IGNORE_BKEY = 2, + MLX4_MAD_IFC_IGNORE_KEYS = (MLX4_MAD_IFC_IGNORE_MKEY | + MLX4_MAD_IFC_IGNORE_BKEY), + MLX4_MAD_IFC_NET_VIEW = 4, +}; + +enum { + MLX4_NUM_TUNNEL_BUFS = 256, +}; + +struct mlx4_ib_tunnel_header { + struct mlx4_av av; + __be32 remote_qpn; + __be32 qkey; + __be16 vlan; + u8 mac[6]; + __be16 pkey_index; + u8 reserved[6]; +}; + +struct mlx4_ib_buf { + void *addr; + dma_addr_t map; +}; + +struct mlx4_rcv_tunnel_hdr { + __be32 flags_src_qp; /* flags[6:5] is defined for VLANs: + * 0x0 - no vlan was in the packet + * 0x01 - C-VLAN was in the packet */ + u8 g_ml_path; /* gid bit stands for ipv6/4 header in RoCE */ + u8 reserved; + __be16 pkey_index; + __be16 sl_vid; + __be16 slid_mac_47_32; + __be32 mac_31_0; +}; + +struct mlx4_ib_proxy_sqp_hdr { + struct ib_grh grh; + struct mlx4_rcv_tunnel_hdr tun; +} __packed; + +struct mlx4_roce_smac_vlan_info { + u64 smac; + int smac_index; + int smac_port; + u64 candidate_smac; + int candidate_smac_index; + int candidate_smac_port; + u16 vid; + int vlan_index; + int vlan_port; + u16 candidate_vid; + int candidate_vlan_index; + int candidate_vlan_port; + int update_vid; +}; + +struct mlx4_ib_qpg_data { + unsigned long *tss_bitmap; + unsigned long *rss_bitmap; + struct mlx4_ib_qp *qpg_parent; + int tss_qpn_base; + int rss_qpn_base; + u32 tss_child_count; + u32 rss_child_count; + u32 qpg_tss_mask_sz; +}; + +struct mlx4_ib_qp { + struct ib_qp ibqp; + struct mlx4_qp mqp; + struct mlx4_buf buf; + + struct mlx4_db db; + struct mlx4_ib_wq rq; + + u32 doorbell_qpn; + __be32 sq_signal_bits; + unsigned sq_next_wqe; + int sq_max_wqes_per_wr; + int sq_spare_wqes; + struct mlx4_ib_wq sq; + + enum mlx4_ib_qp_type mlx4_ib_qp_type; + struct ib_umem *umem; + struct mlx4_mtt mtt; + int buf_size; + struct mutex mutex; + u16 xrcdn; + u32 flags; + u8 port; + u8 alt_port; + u8 atomic_rd_en; + u8 resp_depth; + u8 sq_no_prefetch; + u8 state; + int mlx_type; + enum ib_qpg_type qpg_type; + struct mlx4_ib_qpg_data *qpg_data; + struct list_head gid_list; + struct list_head steering_rules; + struct mlx4_ib_buf *sqp_proxy_rcv; + struct mlx4_roce_smac_vlan_info pri; + struct mlx4_roce_smac_vlan_info alt; + struct list_head rules_list; + u64 reg_id; + int max_inline_data; + struct mlx4_bf bf; + + /* + * Experimental data + */ + int max_inlr_data; +}; + +struct mlx4_ib_srq { + struct ib_srq ibsrq; + struct mlx4_srq msrq; + struct mlx4_buf buf; + struct mlx4_db db; + u64 *wrid; + spinlock_t lock; + int head; + int tail; + u16 wqe_ctr; + struct ib_umem *umem; + struct mlx4_mtt mtt; + struct mutex mutex; +}; + +struct mlx4_ib_ah { + struct ib_ah ibah; + union mlx4_ext_av av; +}; + +/****************************************/ +/* alias guid support */ +/****************************************/ +#define NUM_PORT_ALIAS_GUID 2 +#define NUM_ALIAS_GUID_IN_REC 8 +#define NUM_ALIAS_GUID_REC_IN_PORT 16 +#define GUID_REC_SIZE 8 +#define NUM_ALIAS_GUID_PER_PORT 128 +#define MLX4_NOT_SET_GUID (0x00LL) +#define MLX4_GUID_FOR_DELETE_VAL (~(0x00LL)) + +/****************************************/ +/* ioctl codes */ +/****************************************/ +#define MLX4_IOC_MAGIC 'm' +#define MLX4_IOCHWCLOCKOFFSET _IOR(MLX4_IOC_MAGIC, 1, int) + +enum mlx4_guid_alias_rec_status { + MLX4_GUID_INFO_STATUS_IDLE, + MLX4_GUID_INFO_STATUS_SET, + MLX4_GUID_INFO_STATUS_PENDING, +}; + +enum mlx4_guid_alias_rec_ownership { + MLX4_GUID_DRIVER_ASSIGN, + MLX4_GUID_SYSADMIN_ASSIGN, + MLX4_GUID_NONE_ASSIGN, /*init state of each record*/ +}; + +enum mlx4_guid_alias_rec_method { + MLX4_GUID_INFO_RECORD_SET = IB_MGMT_METHOD_SET, + MLX4_GUID_INFO_RECORD_DELETE = IB_SA_METHOD_DELETE, +}; + +struct mlx4_sriov_alias_guid_info_rec_det { + u8 all_recs[GUID_REC_SIZE * NUM_ALIAS_GUID_IN_REC]; + ib_sa_comp_mask guid_indexes; /*indicates what from the 8 records are valid*/ + enum mlx4_guid_alias_rec_status status; /*indicates the administraively status of the record.*/ + u8 method; /*set or delete*/ + enum mlx4_guid_alias_rec_ownership ownership; /*indicates who assign that alias_guid record*/ +}; + +struct mlx4_sriov_alias_guid_port_rec_det { + struct mlx4_sriov_alias_guid_info_rec_det all_rec_per_port[NUM_ALIAS_GUID_REC_IN_PORT]; + struct workqueue_struct *wq; + struct delayed_work alias_guid_work; + u8 port; + struct mlx4_sriov_alias_guid *parent; + struct list_head cb_list; +}; + +struct mlx4_sriov_alias_guid { + struct mlx4_sriov_alias_guid_port_rec_det ports_guid[MLX4_MAX_PORTS]; + spinlock_t ag_work_lock; + struct ib_sa_client *sa_client; +}; + +struct mlx4_ib_demux_work { + struct work_struct work; + struct mlx4_ib_dev *dev; + int slave; + int do_init; + u8 port; + +}; + +struct mlx4_ib_tun_tx_buf { + struct mlx4_ib_buf buf; + struct ib_ah *ah; +}; + +struct mlx4_ib_demux_pv_qp { + struct ib_qp *qp; + enum ib_qp_type proxy_qpt; + struct mlx4_ib_buf *ring; + struct mlx4_ib_tun_tx_buf *tx_ring; + spinlock_t tx_lock; + unsigned tx_ix_head; + unsigned tx_ix_tail; +}; + +enum mlx4_ib_demux_pv_state { + DEMUX_PV_STATE_DOWN, + DEMUX_PV_STATE_STARTING, + DEMUX_PV_STATE_ACTIVE, + DEMUX_PV_STATE_DOWNING, +}; + +struct mlx4_ib_demux_pv_ctx { + int port; + int slave; + enum mlx4_ib_demux_pv_state state; + int has_smi; + struct ib_device *ib_dev; + struct ib_cq *cq; + struct ib_pd *pd; + struct ib_mr *mr; + struct work_struct work; + struct workqueue_struct *wq; + struct mlx4_ib_demux_pv_qp qp[2]; +}; + +struct mlx4_ib_demux_ctx { + struct ib_device *ib_dev; + int port; + struct workqueue_struct *wq; + struct workqueue_struct *ud_wq; + spinlock_t ud_lock; + __be64 subnet_prefix; + __be64 guid_cache[128]; + struct mlx4_ib_dev *dev; + /* the following lock protects both mcg_table and mcg_mgid0_list */ + struct mutex mcg_table_lock; + struct rb_root mcg_table; + struct list_head mcg_mgid0_list; + struct workqueue_struct *mcg_wq; + struct mlx4_ib_demux_pv_ctx **tun; + atomic_t tid; + int flushing; /* flushing the work queue */ +}; + +struct mlx4_ib_sriov { + struct mlx4_ib_demux_ctx demux[MLX4_MAX_PORTS]; + struct mlx4_ib_demux_pv_ctx *sqps[MLX4_MAX_PORTS]; + /* when using this spinlock you should use "irq" because + * it may be called from interrupt context.*/ + spinlock_t going_down_lock; + int is_going_down; + + struct mlx4_sriov_alias_guid alias_guid; + + /* CM paravirtualization fields */ + struct list_head cm_list; + spinlock_t id_map_lock; + struct rb_root sl_id_map; + struct idr pv_id_table; +}; + +struct mlx4_ib_iboe { + spinlock_t lock; + struct net_device *netdevs[MLX4_MAX_PORTS]; + struct net_device *masters[MLX4_MAX_PORTS]; + struct notifier_block nb; + struct notifier_block nb_inet; + union ib_gid gid_table[MLX4_MAX_PORTS][128]; +}; + +struct pkey_mgt { + u8 virt2phys_pkey[MLX4_MFUNC_MAX][MLX4_MAX_PORTS][MLX4_MAX_PORT_PKEYS]; + u16 phys_pkey_cache[MLX4_MAX_PORTS][MLX4_MAX_PORT_PKEYS]; + struct list_head pkey_port_list[MLX4_MFUNC_MAX]; + struct kobject *device_parent[MLX4_MFUNC_MAX]; +}; + +struct mlx4_ib_iov_sysfs_attr { + void *ctx; + struct kobject *kobj; + unsigned long data; + u32 entry_num; + char name[15]; + struct device_attribute dentry; + struct device *dev; +}; + +struct mlx4_ib_iov_sysfs_attr_ar { + struct mlx4_ib_iov_sysfs_attr dentries[3 * NUM_ALIAS_GUID_PER_PORT + 1]; +}; + +struct mlx4_ib_iov_port { + char name[100]; + u8 num; + struct mlx4_ib_dev *dev; + struct list_head list; + struct mlx4_ib_iov_sysfs_attr_ar *dentr_ar; + struct ib_port_attr attr; + struct kobject *cur_port; + struct kobject *admin_alias_parent; + struct kobject *gids_parent; + struct kobject *pkeys_parent; + struct kobject *mcgs_parent; + struct mlx4_ib_iov_sysfs_attr mcg_dentry; +}; + +struct mlx4_ib_counter { + int counter_index; + int status; +}; + +struct mlx4_ib_dev { + struct ib_device ib_dev; + struct mlx4_dev *dev; + int num_ports; + struct mlx4_uar priv_uar; + u32 priv_pdn; + MLX4_DECLARE_DOORBELL_LOCK(uar_lock); + + struct ib_mad_agent *send_agent[MLX4_MAX_PORTS][2]; + struct ib_ah *sm_ah[MLX4_MAX_PORTS]; + spinlock_t sm_lock; + struct mlx4_ib_sriov sriov; + + struct mutex cap_mask_mutex; + bool ib_active; + struct mlx4_ib_iboe iboe; + struct mlx4_ib_counter counters[MLX4_MAX_PORTS]; + int *eq_table; + int eq_added; + struct kobject *iov_parent; + struct kobject *ports_parent; + struct kobject *dev_ports_parent[MLX4_MFUNC_MAX]; + struct mlx4_ib_iov_port iov_ports[MLX4_MAX_PORTS]; + struct pkey_mgt pkeys; + unsigned long *ib_uc_qpns_bitmap; + int steer_qpn_count; + int steer_qpn_base; +}; + +struct ib_event_work { + struct work_struct work; + struct mlx4_ib_dev *ib_dev; + struct mlx4_eqe ib_eqe; +}; + +struct mlx4_ib_qp_tunnel_init_attr { + struct ib_qp_init_attr init_attr; + int slave; + enum ib_qp_type proxy_qp_type; + u8 port; +}; + +static inline struct mlx4_ib_dev *to_mdev(struct ib_device *ibdev) +{ + return container_of(ibdev, struct mlx4_ib_dev, ib_dev); +} + +static inline struct mlx4_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext) +{ + return container_of(ibucontext, struct mlx4_ib_ucontext, ibucontext); +} + +static inline struct mlx4_ib_pd *to_mpd(struct ib_pd *ibpd) +{ + return container_of(ibpd, struct mlx4_ib_pd, ibpd); +} + +static inline struct mlx4_ib_xrcd *to_mxrcd(struct ib_xrcd *ibxrcd) +{ + return container_of(ibxrcd, struct mlx4_ib_xrcd, ibxrcd); +} + +static inline struct mlx4_ib_cq *to_mcq(struct ib_cq *ibcq) +{ + return container_of(ibcq, struct mlx4_ib_cq, ibcq); +} + +static inline struct mlx4_ib_cq *to_mibcq(struct mlx4_cq *mcq) +{ + return container_of(mcq, struct mlx4_ib_cq, mcq); +} + +static inline struct mlx4_ib_mr *to_mmr(struct ib_mr *ibmr) +{ + return container_of(ibmr, struct mlx4_ib_mr, ibmr); +} + +static inline struct mlx4_ib_mw *to_mmw(struct ib_mw *ibmw) +{ + return container_of(ibmw, struct mlx4_ib_mw, ibmw); +} + +static inline struct mlx4_ib_fast_reg_page_list *to_mfrpl(struct ib_fast_reg_page_list *ibfrpl) +{ + return container_of(ibfrpl, struct mlx4_ib_fast_reg_page_list, ibfrpl); +} + +static inline struct mlx4_ib_fmr *to_mfmr(struct ib_fmr *ibfmr) +{ + return container_of(ibfmr, struct mlx4_ib_fmr, ibfmr); +} + +static inline struct mlx4_ib_flow *to_mflow(struct ib_flow *ibflow) +{ + return container_of(ibflow, struct mlx4_ib_flow, ibflow); +} + +static inline struct mlx4_ib_qp *to_mqp(struct ib_qp *ibqp) +{ + return container_of(ibqp, struct mlx4_ib_qp, ibqp); +} + +static inline struct mlx4_ib_qp *to_mibqp(struct mlx4_qp *mqp) +{ + return container_of(mqp, struct mlx4_ib_qp, mqp); +} + +static inline struct mlx4_ib_srq *to_msrq(struct ib_srq *ibsrq) +{ + return container_of(ibsrq, struct mlx4_ib_srq, ibsrq); +} + +static inline struct mlx4_ib_srq *to_mibsrq(struct mlx4_srq *msrq) +{ + return container_of(msrq, struct mlx4_ib_srq, msrq); +} + +static inline struct mlx4_ib_ah *to_mah(struct ib_ah *ibah) +{ + return container_of(ibah, struct mlx4_ib_ah, ibah); +} + +int mlx4_ib_init_sriov(struct mlx4_ib_dev *dev); +void mlx4_ib_close_sriov(struct mlx4_ib_dev *dev); + +int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context, unsigned long virt, + struct mlx4_db *db); +void mlx4_ib_db_unmap_user(struct mlx4_ib_ucontext *context, struct mlx4_db *db); + +struct ib_mr *mlx4_ib_get_dma_mr(struct ib_pd *pd, int acc); +int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt, + struct ib_umem *umem); +int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, + u64 start_va, + int *num_of_mtts); +struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, + u64 virt_addr, int access_flags, + struct ib_udata *udata, int mr_id); +int mlx4_ib_dereg_mr(struct ib_mr *mr); +struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type); +int mlx4_ib_bind_mw(struct ib_qp *qp, struct ib_mw *mw, + struct ib_mw_bind *mw_bind); +int mlx4_ib_dealloc_mw(struct ib_mw *mw); +struct ib_mr *mlx4_ib_alloc_fast_reg_mr(struct ib_pd *pd, + int max_page_list_len); +struct ib_fast_reg_page_list *mlx4_ib_alloc_fast_reg_page_list(struct ib_device *ibdev, + int page_list_len); +void mlx4_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list); + +int mlx4_ib_modify_cq(struct ib_cq *cq, + struct ib_cq_attr *cq_attr, + int cq_attr_mask); +int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata); +int mlx4_ib_ignore_overrun_cq(struct ib_cq *ibcq); +struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, + struct ib_cq_init_attr *attr, + struct ib_ucontext *context, + struct ib_udata *udata); +int mlx4_ib_destroy_cq(struct ib_cq *cq); +int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); +int mlx4_ib_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); +void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq); +void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq); + +struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr); +int mlx4_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr); +int mlx4_ib_destroy_ah(struct ib_ah *ah); + +struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata); +int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, + enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); +int mlx4_ib_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr); +int mlx4_ib_destroy_srq(struct ib_srq *srq); +void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index); +int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr); + +struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata); +int mlx4_ib_destroy_qp(struct ib_qp *qp); +int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata); +int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, + struct ib_qp_init_attr *qp_init_attr); +int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, + struct ib_send_wr **bad_wr); +int mlx4_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr); + +int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int mad_ifc_flags, + int port, struct ib_wc *in_wc, struct ib_grh *in_grh, + void *in_mad, void *response_mad); +int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, + struct ib_wc *in_wc, struct ib_grh *in_grh, + struct ib_mad *in_mad, struct ib_mad *out_mad); +int mlx4_ib_mad_init(struct mlx4_ib_dev *dev); +void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev); + +struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int mr_access_flags, + struct ib_fmr_attr *fmr_attr); +int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, int npages, + u64 iova); +int mlx4_ib_unmap_fmr(struct list_head *fmr_list); +int mlx4_ib_fmr_dealloc(struct ib_fmr *fmr); +int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props, int netw_view); +int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, + u16 *pkey, int netw_view); + +int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index, + union ib_gid *gid, int netw_view); + +int mlx4_ib_resolve_grh(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah_attr, + u8 *mac, int *is_mcast, u8 port); + +static inline int mlx4_ib_ah_grh_present(struct mlx4_ib_ah *ah) +{ + u8 port = be32_to_cpu(ah->av.ib.port_pd) >> 24 & 3; + + if (rdma_port_get_link_layer(ah->ibah.device, port) == IB_LINK_LAYER_ETHERNET) + return 1; + + return !!(ah->av.ib.g_slid & 0x80); +} +static inline int mlx4_ib_qp_has_rq(struct ib_qp_init_attr *attr) +{ + if (attr->qp_type == IB_QPT_XRC_INI || attr->qp_type == IB_QPT_XRC_TGT) + return 0; + + return !attr->srq; +} + +int mlx4_ib_mcg_port_init(struct mlx4_ib_demux_ctx *ctx); +void mlx4_ib_mcg_port_cleanup(struct mlx4_ib_demux_ctx *ctx, int destroy_wq); +void clean_vf_mcast(struct mlx4_ib_demux_ctx *ctx, int slave); +int mlx4_ib_mcg_init(void); +void mlx4_ib_mcg_destroy(void); + +int mlx4_ib_find_real_gid(struct ib_device *ibdev, u8 port, __be64 guid); + +int mlx4_ib_mcg_multiplex_handler(struct ib_device *ibdev, int port, int slave, + struct ib_sa_mad *sa_mad); +int mlx4_ib_mcg_demux_handler(struct ib_device *ibdev, int port, int slave, + struct ib_sa_mad *mad); + +int mlx4_ib_add_mc(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp, + union ib_gid *gid); + +void mlx4_ib_dispatch_event(struct mlx4_ib_dev *dev, u8 port_num, + enum ib_event_type type); + +void mlx4_ib_tunnels_update_work(struct work_struct *work); + +int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port, + enum ib_qp_type qpt, struct ib_wc *wc, + struct ib_grh *grh, struct ib_mad *mad); +int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port, + enum ib_qp_type dest_qpt, u16 pkey_index, u32 remote_qpn, + u32 qkey, struct ib_ah_attr *attr, u8 *s_mac, struct ib_mad *mad); +__be64 mlx4_ib_get_new_demux_tid(struct mlx4_ib_demux_ctx *ctx); + +int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave, + struct ib_mad *mad, int is_eth); + +int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id, + struct ib_mad *mad); + +void mlx4_ib_cm_paravirt_init(struct mlx4_ib_dev *dev); +void mlx4_ib_cm_paravirt_clean(struct mlx4_ib_dev *dev, int slave_id); + +/* alias guid support */ +void mlx4_ib_init_alias_guid_work(struct mlx4_ib_dev *dev, int port); +int mlx4_ib_init_alias_guid_service(struct mlx4_ib_dev *dev); +void mlx4_ib_destroy_alias_guid_service(struct mlx4_ib_dev *dev); +void mlx4_ib_invalidate_all_guid_record(struct mlx4_ib_dev *dev, int port); + +void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev, + int block_num, + u8 port_num, u8 *p_data); + +void mlx4_ib_update_cache_on_guid_change(struct mlx4_ib_dev *dev, + int block_num, u8 port_num, + u8 *p_data); + +int add_sysfs_port_mcg_attr(struct mlx4_ib_dev *device, int port_num, + struct attribute *attr); +void del_sysfs_port_mcg_attr(struct mlx4_ib_dev *device, int port_num, + struct attribute *attr); +ib_sa_comp_mask mlx4_ib_get_aguid_comp_mask_from_ix(int index); + +int mlx4_ib_device_register_sysfs(struct mlx4_ib_dev *device) ; + +void mlx4_ib_device_unregister_sysfs(struct mlx4_ib_dev *device); + +__be64 mlx4_ib_gen_node_guid(void); + +int mlx4_ib_steer_qp_alloc(struct mlx4_ib_dev *dev, int count, int *qpn); +void mlx4_ib_steer_qp_free(struct mlx4_ib_dev *dev, u32 qpn, int count); +int mlx4_ib_steer_qp_reg(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp, + int is_attach); +int mlx4_ib_query_device(struct ib_device *ibdev, + struct ib_device_attr *props); + +#endif /* MLX4_IB_H */ Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_cq.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_cq.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_cq.c (revision 329159) @@ -0,0 +1,1001 @@ +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include + +#include "mlx4_ib.h" +#include "user.h" + +/* Which firmware version adds support for Resize CQ */ +#define MLX4_FW_VER_RESIZE_CQ mlx4_fw_ver(2, 5, 0) +#define MLX4_FW_VER_IGNORE_OVERRUN_CQ mlx4_fw_ver(2, 7, 8200) + +static void mlx4_ib_cq_comp(struct mlx4_cq *cq) +{ + struct ib_cq *ibcq = &to_mibcq(cq)->ibcq; + ibcq->comp_handler(ibcq, ibcq->cq_context); +} + +static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type) +{ + struct ib_event event; + struct ib_cq *ibcq; + + if (type != MLX4_EVENT_TYPE_CQ_ERROR) { + pr_warn("Unexpected event type %d " + "on CQ %06x\n", type, cq->cqn); + return; + } + + ibcq = &to_mibcq(cq)->ibcq; + if (ibcq->event_handler) { + event.device = ibcq->device; + event.event = IB_EVENT_CQ_ERR; + event.element.cq = ibcq; + ibcq->event_handler(&event, ibcq->cq_context); + } +} + +static void *get_cqe_from_buf(struct mlx4_ib_cq_buf *buf, int n) +{ + return mlx4_buf_offset(&buf->buf, n * buf->entry_size); +} + +static void *get_cqe(struct mlx4_ib_cq *cq, int n) +{ + return get_cqe_from_buf(&cq->buf, n); +} + +static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n) +{ + struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe); + struct mlx4_cqe *tcqe = ((cq->buf.entry_size == 64) ? (cqe + 1) : cqe); + + return (!!(tcqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^ + !!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe; +} + +static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq) +{ + return get_sw_cqe(cq, cq->mcq.cons_index); +} + +int mlx4_ib_modify_cq(struct ib_cq *cq, + struct ib_cq_attr *cq_attr, + int cq_attr_mask) +{ + int err = 0; + struct mlx4_ib_cq *mcq = to_mcq(cq); + struct mlx4_ib_dev *dev = to_mdev(cq->device); + + if (cq_attr_mask & IB_CQ_CAP_FLAGS) { + if (cq_attr->cq_cap_flags & IB_CQ_TIMESTAMP) + return -ENOTSUPP; + + if (cq_attr->cq_cap_flags & IB_CQ_IGNORE_OVERRUN) + return -ENOSYS; + } + + if (!err) + if (cq_attr_mask & IB_CQ_MODERATION) + err = mlx4_cq_modify(dev->dev, &mcq->mcq, + cq_attr->moderation.cq_count, + cq_attr->moderation.cq_period); + + return err; +} + +static int mlx4_ib_alloc_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *buf, int nent) +{ + int err; + + err = mlx4_buf_alloc(dev->dev, nent * dev->dev->caps.cqe_size, + PAGE_SIZE * 2, &buf->buf, GFP_KERNEL); + + if (err) + goto out; + + buf->entry_size = dev->dev->caps.cqe_size; + err = mlx4_mtt_init(dev->dev, buf->buf.npages, buf->buf.page_shift, + &buf->mtt); + if (err) + goto err_buf; + + err = mlx4_buf_write_mtt(dev->dev, &buf->mtt, &buf->buf, GFP_KERNEL); + if (err) + goto err_mtt; + + return 0; + +err_mtt: + mlx4_mtt_cleanup(dev->dev, &buf->mtt); + +err_buf: + mlx4_buf_free(dev->dev, nent * buf->entry_size, &buf->buf); + +out: + return err; +} + +static void mlx4_ib_free_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *buf, int cqe) +{ + mlx4_buf_free(dev->dev, (cqe + 1) * buf->entry_size, &buf->buf); +} + +static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev, struct ib_ucontext *context, + struct mlx4_ib_cq_buf *buf, struct ib_umem **umem, + u64 buf_addr, int cqe) +{ + int err; + int cqe_size = dev->dev->caps.cqe_size; + int shift; + int n; + + *umem = ib_umem_get(context, buf_addr, cqe * cqe_size, + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(*umem)) + return PTR_ERR(*umem); + + n = ib_umem_page_count(*umem); + shift = mlx4_ib_umem_calc_optimal_mtt_size(*umem, 0, &n); + err = mlx4_mtt_init(dev->dev, n, shift, &buf->mtt); + + if (err) + goto err_buf; + + err = mlx4_ib_umem_write_mtt(dev, &buf->mtt, *umem); + if (err) + goto err_mtt; + + return 0; + +err_mtt: + mlx4_mtt_cleanup(dev->dev, &buf->mtt); + +err_buf: + ib_umem_release(*umem); + + return err; +} + +/* we don't support system timestamping */ +#define CQ_CREATE_FLAGS_SUPPORTED IB_CQ_TIMESTAMP + +struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, + struct ib_cq_init_attr *attr, + struct ib_ucontext *context, + struct ib_udata *udata) +{ + struct mlx4_ib_dev *dev = to_mdev(ibdev); + struct mlx4_ib_cq *cq; + struct mlx4_uar *uar; + int err; + int entries = attr->cqe; + int vector = attr->comp_vector; + + if (entries < 1 || entries > dev->dev->caps.max_cqes) + return ERR_PTR(-EINVAL); + + if (attr->flags & ~CQ_CREATE_FLAGS_SUPPORTED) + return ERR_PTR(-EINVAL); + + cq = kzalloc(sizeof(*cq), GFP_KERNEL); + if (!cq) + return ERR_PTR(-ENOMEM); + + entries = roundup_pow_of_two(entries + 1); + cq->ibcq.cqe = entries - 1; + mutex_init(&cq->resize_mutex); + spin_lock_init(&cq->lock); + cq->resize_buf = NULL; + cq->resize_umem = NULL; + cq->create_flags = attr->flags; + + if (context) { + struct mlx4_ib_create_cq ucmd; + + if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { + err = -EFAULT; + goto err_cq; + } + + err = mlx4_ib_get_cq_umem(dev, context, &cq->buf, &cq->umem, + ucmd.buf_addr, entries); + if (err) + goto err_cq; + + err = mlx4_ib_db_map_user(to_mucontext(context), ucmd.db_addr, + &cq->db); + if (err) + goto err_mtt; + + uar = &to_mucontext(context)->uar; + } else { + err = mlx4_db_alloc(dev->dev, &cq->db, 1, GFP_KERNEL); + if (err) + goto err_cq; + + cq->mcq.set_ci_db = cq->db.db; + cq->mcq.arm_db = cq->db.db + 1; + *cq->mcq.set_ci_db = 0; + *cq->mcq.arm_db = 0; + + err = mlx4_ib_alloc_cq_buf(dev, &cq->buf, entries); + if (err) + goto err_db; + + uar = &dev->priv_uar; + } + + if (dev->eq_table) + vector = dev->eq_table[vector % ibdev->num_comp_vectors]; + + err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar, + cq->db.dma, &cq->mcq, vector, 0, + !!(cq->create_flags & IB_CQ_TIMESTAMP)); + if (err) + goto err_dbmap; + + cq->mcq.comp = mlx4_ib_cq_comp; + cq->mcq.event = mlx4_ib_cq_event; + + if (context) + if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) { + err = -EFAULT; + goto err_dbmap; + } + + return &cq->ibcq; + +err_dbmap: + if (context) + mlx4_ib_db_unmap_user(to_mucontext(context), &cq->db); + +err_mtt: + mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt); + + if (context) + ib_umem_release(cq->umem); + else + mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); + +err_db: + if (!context) + mlx4_db_free(dev->dev, &cq->db); + +err_cq: + kfree(cq); + + return ERR_PTR(err); +} + +static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq, + int entries) +{ + int err; + + if (cq->resize_buf) + return -EBUSY; + + cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC); + if (!cq->resize_buf) + return -ENOMEM; + + err = mlx4_ib_alloc_cq_buf(dev, &cq->resize_buf->buf, entries); + if (err) { + kfree(cq->resize_buf); + cq->resize_buf = NULL; + return err; + } + + cq->resize_buf->cqe = entries - 1; + + return 0; +} + +static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq, + int entries, struct ib_udata *udata) +{ + struct mlx4_ib_resize_cq ucmd; + int err; + + if (cq->resize_umem) + return -EBUSY; + + if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) + return -EFAULT; + + cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC); + if (!cq->resize_buf) + return -ENOMEM; + + err = mlx4_ib_get_cq_umem(dev, cq->umem->context, &cq->resize_buf->buf, + &cq->resize_umem, ucmd.buf_addr, entries); + if (err) { + kfree(cq->resize_buf); + cq->resize_buf = NULL; + return err; + } + + cq->resize_buf->cqe = entries - 1; + + return 0; +} + +static int mlx4_ib_get_outstanding_cqes(struct mlx4_ib_cq *cq) +{ + u32 i; + + i = cq->mcq.cons_index; + while (get_sw_cqe(cq, i)) + ++i; + + return i - cq->mcq.cons_index; +} + +static int mlx4_ib_cq_resize_copy_cqes(struct mlx4_ib_cq *cq) +{ + struct mlx4_cqe *cqe, *new_cqe; + int i; + int cqe_size = cq->buf.entry_size; + int cqe_inc = cqe_size == 64 ? 1 : 0; + struct mlx4_cqe *start_cqe; + + i = cq->mcq.cons_index; + cqe = get_cqe(cq, i & cq->ibcq.cqe); + start_cqe = cqe; + cqe += cqe_inc; + + while ((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) != MLX4_CQE_OPCODE_RESIZE) { + new_cqe = get_cqe_from_buf(&cq->resize_buf->buf, + (i + 1) & cq->resize_buf->cqe); + memcpy(new_cqe, get_cqe(cq, i & cq->ibcq.cqe), cqe_size); + new_cqe += cqe_inc; + + new_cqe->owner_sr_opcode = (cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK) | + (((i + 1) & (cq->resize_buf->cqe + 1)) ? MLX4_CQE_OWNER_MASK : 0); + cqe = get_cqe(cq, ++i & cq->ibcq.cqe); + if (cqe == start_cqe) { + pr_warn("resize CQ failed to get resize CQE, CQN 0x%x\n", cq->mcq.cqn); + return -ENOMEM; + } + cqe += cqe_inc; + + } + ++cq->mcq.cons_index; + return 0; +} + +int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) +{ + struct mlx4_ib_dev *dev = to_mdev(ibcq->device); + struct mlx4_ib_cq *cq = to_mcq(ibcq); + struct mlx4_mtt mtt; + int outst_cqe; + int err; + + if (dev->dev->caps.fw_ver < MLX4_FW_VER_RESIZE_CQ) + return -ENOSYS; + + mutex_lock(&cq->resize_mutex); + if (entries < 1 || entries > dev->dev->caps.max_cqes) { + err = -EINVAL; + goto out; + } + + entries = roundup_pow_of_two(entries + 1); + if (entries == ibcq->cqe + 1) { + err = 0; + goto out; + } + + if (entries > dev->dev->caps.max_cqes + 1) { + err = -EINVAL; + goto out; + } + + if (ibcq->uobject) { + err = mlx4_alloc_resize_umem(dev, cq, entries, udata); + if (err) + goto out; + } else { + /* Can't be smaller than the number of outstanding CQEs */ + outst_cqe = mlx4_ib_get_outstanding_cqes(cq); + if (entries < outst_cqe + 1) { + err = 0; + goto out; + } + + err = mlx4_alloc_resize_buf(dev, cq, entries); + if (err) + goto out; + } + + mtt = cq->buf.mtt; + + err = mlx4_cq_resize(dev->dev, &cq->mcq, entries, &cq->resize_buf->buf.mtt); + if (err) + goto err_buf; + + mlx4_mtt_cleanup(dev->dev, &mtt); + if (ibcq->uobject) { + cq->buf = cq->resize_buf->buf; + cq->ibcq.cqe = cq->resize_buf->cqe; + ib_umem_release(cq->umem); + cq->umem = cq->resize_umem; + + kfree(cq->resize_buf); + cq->resize_buf = NULL; + cq->resize_umem = NULL; + } else { + struct mlx4_ib_cq_buf tmp_buf; + int tmp_cqe = 0; + + spin_lock_irq(&cq->lock); + if (cq->resize_buf) { + err = mlx4_ib_cq_resize_copy_cqes(cq); + tmp_buf = cq->buf; + tmp_cqe = cq->ibcq.cqe; + cq->buf = cq->resize_buf->buf; + cq->ibcq.cqe = cq->resize_buf->cqe; + + kfree(cq->resize_buf); + cq->resize_buf = NULL; + } + spin_unlock_irq(&cq->lock); + + if (tmp_cqe) + mlx4_ib_free_cq_buf(dev, &tmp_buf, tmp_cqe); + } + + goto out; + +err_buf: + mlx4_mtt_cleanup(dev->dev, &cq->resize_buf->buf.mtt); + if (!ibcq->uobject) + mlx4_ib_free_cq_buf(dev, &cq->resize_buf->buf, + cq->resize_buf->cqe); + + kfree(cq->resize_buf); + cq->resize_buf = NULL; + + if (cq->resize_umem) { + ib_umem_release(cq->resize_umem); + cq->resize_umem = NULL; + } + +out: + mutex_unlock(&cq->resize_mutex); + + return err; +} + +int mlx4_ib_ignore_overrun_cq(struct ib_cq *ibcq) +{ + return -ENOSYS; +} + +int mlx4_ib_destroy_cq(struct ib_cq *cq) +{ + struct mlx4_ib_dev *dev = to_mdev(cq->device); + struct mlx4_ib_cq *mcq = to_mcq(cq); + + mlx4_cq_free(dev->dev, &mcq->mcq); + mlx4_mtt_cleanup(dev->dev, &mcq->buf.mtt); + + if (cq->uobject) { + mlx4_ib_db_unmap_user(to_mucontext(cq->uobject->context), &mcq->db); + ib_umem_release(mcq->umem); + } else { + mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe); + mlx4_db_free(dev->dev, &mcq->db); + } + + kfree(mcq); + + return 0; +} + +static void dump_cqe(void *cqe) +{ + __be32 *buf = cqe; + + pr_debug("CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n", + be32_to_cpu(buf[0]), be32_to_cpu(buf[1]), be32_to_cpu(buf[2]), + be32_to_cpu(buf[3]), be32_to_cpu(buf[4]), be32_to_cpu(buf[5]), + be32_to_cpu(buf[6]), be32_to_cpu(buf[7])); +} + +static void mlx4_ib_handle_error_cqe(struct mlx4_err_cqe *cqe, + struct ib_wc *wc) +{ + if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR) { + pr_debug("local QP operation err " + "(QPN %06x, WQE index %x, vendor syndrome %02x, " + "opcode = %02x)\n", + be32_to_cpu(cqe->my_qpn), be16_to_cpu(cqe->wqe_index), + cqe->vendor_err_syndrome, + cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK); + dump_cqe(cqe); + } + + switch (cqe->syndrome) { + case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR: + wc->status = IB_WC_LOC_LEN_ERR; + break; + case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR: + wc->status = IB_WC_LOC_QP_OP_ERR; + break; + case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR: + wc->status = IB_WC_LOC_PROT_ERR; + break; + case MLX4_CQE_SYNDROME_WR_FLUSH_ERR: + wc->status = IB_WC_WR_FLUSH_ERR; + break; + case MLX4_CQE_SYNDROME_MW_BIND_ERR: + wc->status = IB_WC_MW_BIND_ERR; + break; + case MLX4_CQE_SYNDROME_BAD_RESP_ERR: + wc->status = IB_WC_BAD_RESP_ERR; + break; + case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR: + wc->status = IB_WC_LOC_ACCESS_ERR; + break; + case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR: + wc->status = IB_WC_REM_INV_REQ_ERR; + break; + case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR: + wc->status = IB_WC_REM_ACCESS_ERR; + break; + case MLX4_CQE_SYNDROME_REMOTE_OP_ERR: + wc->status = IB_WC_REM_OP_ERR; + break; + case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR: + wc->status = IB_WC_RETRY_EXC_ERR; + break; + case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR: + wc->status = IB_WC_RNR_RETRY_EXC_ERR; + break; + case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR: + wc->status = IB_WC_REM_ABORT_ERR; + break; + default: + wc->status = IB_WC_GENERAL_ERR; + break; + } + + wc->vendor_err = cqe->vendor_err_syndrome; +} + +static int mlx4_ib_ipoib_csum_ok(__be16 status, __be16 checksum) +{ + return ((status & cpu_to_be16(MLX4_CQE_STATUS_IPV4 | + MLX4_CQE_STATUS_IPV4F | + MLX4_CQE_STATUS_IPV4OPT | + MLX4_CQE_STATUS_IPV6 | + MLX4_CQE_STATUS_IPOK)) == + cpu_to_be16(MLX4_CQE_STATUS_IPV4 | + MLX4_CQE_STATUS_IPOK)) && + (status & cpu_to_be16(MLX4_CQE_STATUS_UDP | + MLX4_CQE_STATUS_TCP)) && + checksum == cpu_to_be16(0xffff); +} + +static int use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct ib_wc *wc, + unsigned tail, struct mlx4_cqe *cqe, int is_eth) +{ + struct mlx4_ib_proxy_sqp_hdr *hdr; + + ib_dma_sync_single_for_cpu(qp->ibqp.device, + qp->sqp_proxy_rcv[tail].map, + sizeof (struct mlx4_ib_proxy_sqp_hdr), + DMA_FROM_DEVICE); + hdr = (struct mlx4_ib_proxy_sqp_hdr *) (qp->sqp_proxy_rcv[tail].addr); + wc->pkey_index = be16_to_cpu(hdr->tun.pkey_index); + wc->src_qp = be32_to_cpu(hdr->tun.flags_src_qp) & 0xFFFFFF; + wc->wc_flags |= (hdr->tun.g_ml_path & 0x80) ? (IB_WC_GRH) : 0; + wc->dlid_path_bits = 0; + + if (is_eth) { + wc->vlan_id = be16_to_cpu(hdr->tun.sl_vid); + memcpy(&(wc->smac[0]), (char *)&hdr->tun.mac_31_0, 4); + memcpy(&(wc->smac[4]), (char *)&hdr->tun.slid_mac_47_32, 2); + } else { + wc->slid = be16_to_cpu(hdr->tun.slid_mac_47_32); + wc->sl = (u8) (be16_to_cpu(hdr->tun.sl_vid) >> 12); + } + + return 0; +} + +static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq, + struct mlx4_ib_qp **cur_qp, + struct ib_wc *wc) +{ + struct mlx4_cqe *cqe; + struct mlx4_qp *mqp; + struct mlx4_ib_wq *wq; + struct mlx4_ib_srq *srq; + struct mlx4_srq *msrq = NULL; + int is_send; + int is_error; + u32 g_mlpath_rqpn; + u16 wqe_ctr; + unsigned tail = 0; + int timestamp_en = !!(cq->create_flags & IB_CQ_TIMESTAMP); + + +repoll: + cqe = next_cqe_sw(cq); + if (!cqe) + return -EAGAIN; + + if (cq->buf.entry_size == 64) + cqe++; + + ++cq->mcq.cons_index; + + /* + * Make sure we read CQ entry contents after we've checked the + * ownership bit. + */ + rmb(); + + is_send = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK; + is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == + MLX4_CQE_OPCODE_ERROR; + + if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_OPCODE_NOP && + is_send)) { + pr_warn("Completion for NOP opcode detected!\n"); + return -EINVAL; + } + + /* Resize CQ in progress */ + if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_CQE_OPCODE_RESIZE)) { + if (cq->resize_buf) { + struct mlx4_ib_dev *dev = to_mdev(cq->ibcq.device); + + mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); + cq->buf = cq->resize_buf->buf; + cq->ibcq.cqe = cq->resize_buf->cqe; + + kfree(cq->resize_buf); + cq->resize_buf = NULL; + } + + goto repoll; + } + + if (!*cur_qp || + (be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK) != (*cur_qp)->mqp.qpn) { + /* + * We do not have to take the QP table lock here, + * because CQs will be locked while QPs are removed + * from the table. + */ + mqp = __mlx4_qp_lookup(to_mdev(cq->ibcq.device)->dev, + be32_to_cpu(cqe->vlan_my_qpn)); + if (unlikely(!mqp)) { + pr_warn("CQ %06x with entry for unknown QPN %06x\n", + cq->mcq.cqn, be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK); + return -EINVAL; + } + + *cur_qp = to_mibqp(mqp); + } + + wc->qp = &(*cur_qp)->ibqp; + + if (wc->qp->qp_type == IB_QPT_XRC_TGT) { + u32 srq_num; + g_mlpath_rqpn = be32_to_cpu(cqe->g_mlpath_rqpn); + srq_num = g_mlpath_rqpn & 0xffffff; + /* SRQ is also in the radix tree */ + msrq = mlx4_srq_lookup(to_mdev(cq->ibcq.device)->dev, + srq_num); + if (unlikely(!msrq)) { + pr_warn("CQ %06x with entry for unknown SRQN %06x\n", + cq->mcq.cqn, srq_num); + return -EINVAL; + } + } + + if (is_send) { + wq = &(*cur_qp)->sq; + if (!(*cur_qp)->sq_signal_bits) { + wqe_ctr = be16_to_cpu(cqe->wqe_index); + wq->tail += (u16) (wqe_ctr - (u16) wq->tail); + } + wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; + ++wq->tail; + } else if ((*cur_qp)->ibqp.srq) { + srq = to_msrq((*cur_qp)->ibqp.srq); + wqe_ctr = be16_to_cpu(cqe->wqe_index); + wc->wr_id = srq->wrid[wqe_ctr]; + mlx4_ib_free_srq_wqe(srq, wqe_ctr); + } else if (msrq) { + srq = to_mibsrq(msrq); + wqe_ctr = be16_to_cpu(cqe->wqe_index); + wc->wr_id = srq->wrid[wqe_ctr]; + mlx4_ib_free_srq_wqe(srq, wqe_ctr); + } else { + wq = &(*cur_qp)->rq; + tail = wq->tail & (wq->wqe_cnt - 1); + wc->wr_id = wq->wrid[tail]; + ++wq->tail; + } + + if (unlikely(is_error)) { + mlx4_ib_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc); + return 0; + } + + wc->status = IB_WC_SUCCESS; + + if (is_send) { + wc->wc_flags = 0; + switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) { + case MLX4_OPCODE_RDMA_WRITE_IMM: + wc->wc_flags |= IB_WC_WITH_IMM; + /* fall through */ + case MLX4_OPCODE_RDMA_WRITE: + wc->opcode = IB_WC_RDMA_WRITE; + break; + case MLX4_OPCODE_SEND_IMM: + wc->wc_flags |= IB_WC_WITH_IMM; + case MLX4_OPCODE_SEND: + case MLX4_OPCODE_SEND_INVAL: + wc->opcode = IB_WC_SEND; + break; + case MLX4_OPCODE_RDMA_READ: + wc->opcode = IB_WC_RDMA_READ; + wc->byte_len = be32_to_cpu(cqe->byte_cnt); + break; + case MLX4_OPCODE_ATOMIC_CS: + wc->opcode = IB_WC_COMP_SWAP; + wc->byte_len = 8; + break; + case MLX4_OPCODE_ATOMIC_FA: + wc->opcode = IB_WC_FETCH_ADD; + wc->byte_len = 8; + break; + case MLX4_OPCODE_MASKED_ATOMIC_CS: + wc->opcode = IB_WC_MASKED_COMP_SWAP; + wc->byte_len = 8; + break; + case MLX4_OPCODE_MASKED_ATOMIC_FA: + wc->opcode = IB_WC_MASKED_FETCH_ADD; + wc->byte_len = 8; + break; + case MLX4_OPCODE_BIND_MW: + wc->opcode = IB_WC_BIND_MW; + break; + case MLX4_OPCODE_LSO: + wc->opcode = IB_WC_LSO; + break; + case MLX4_OPCODE_FMR: + wc->opcode = IB_WC_FAST_REG_MR; + break; + case MLX4_OPCODE_LOCAL_INVAL: + wc->opcode = IB_WC_LOCAL_INV; + break; + } + } else { + wc->byte_len = be32_to_cpu(cqe->byte_cnt); + + switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) { + case MLX4_RECV_OPCODE_RDMA_WRITE_IMM: + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; + wc->wc_flags = IB_WC_WITH_IMM; + wc->ex.imm_data = cqe->immed_rss_invalid; + break; + case MLX4_RECV_OPCODE_SEND_INVAL: + wc->opcode = IB_WC_RECV; + wc->wc_flags = IB_WC_WITH_INVALIDATE; + wc->ex.invalidate_rkey = be32_to_cpu(cqe->immed_rss_invalid); + break; + case MLX4_RECV_OPCODE_SEND: + wc->opcode = IB_WC_RECV; + wc->wc_flags = 0; + break; + case MLX4_RECV_OPCODE_SEND_IMM: + wc->opcode = IB_WC_RECV; + wc->wc_flags = IB_WC_WITH_IMM; + wc->ex.imm_data = cqe->immed_rss_invalid; + break; + } + + if (mlx4_is_mfunc(to_mdev(cq->ibcq.device)->dev)) { + if ((*cur_qp)->mlx4_ib_qp_type & + (MLX4_IB_QPT_PROXY_SMI_OWNER | + MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) + return use_tunnel_data + (*cur_qp, cq, wc, tail, cqe, + rdma_port_get_link_layer + (wc->qp->device, + (*cur_qp)->port) == + IB_LINK_LAYER_ETHERNET); + } + + if (timestamp_en) { + const struct mlx4_ts_cqe *ts_cqe = + (const struct mlx4_ts_cqe *)cqe; + /* currently, only CQ_CREATE_WITH_TIMESTAMPING_RAW is + * supported. CQ_CREATE_WITH_TIMESTAMPING_SYS isn't + * supported */ + if (cq->create_flags & IB_CQ_TIMESTAMP_TO_SYS_TIME) { + wc->ts.timestamp = 0; + } else { + wc->ts.timestamp = + ((u64)(be32_to_cpu(ts_cqe->timestamp_hi) + + !ts_cqe->timestamp_lo) << 16) + | be16_to_cpu(ts_cqe->timestamp_lo); + wc->wc_flags |= IB_WC_WITH_TIMESTAMP; + } + } else { + wc->wc_flags |= IB_WC_WITH_SLID; + wc->slid = be16_to_cpu(cqe->rlid); + } + g_mlpath_rqpn = be32_to_cpu(cqe->g_mlpath_rqpn); + wc->src_qp = g_mlpath_rqpn & 0xffffff; + wc->dlid_path_bits = (g_mlpath_rqpn >> 24) & 0x7f; + wc->wc_flags |= g_mlpath_rqpn & 0x80000000 ? IB_WC_GRH : 0; + wc->pkey_index = be32_to_cpu(cqe->immed_rss_invalid) & 0x7f; + wc->wc_flags |= mlx4_ib_ipoib_csum_ok(cqe->status, + cqe->checksum) ? IB_WC_IP_CSUM_OK : 0; + if (!timestamp_en) { + if (rdma_port_get_link_layer(wc->qp->device, + (*cur_qp)->port) == + IB_LINK_LAYER_ETHERNET) + wc->sl = be16_to_cpu(cqe->sl_vid) >> 13; + else + wc->sl = be16_to_cpu(cqe->sl_vid) >> 12; + wc->wc_flags |= IB_WC_WITH_SL; + } + if ((be32_to_cpu(cqe->vlan_my_qpn) & + MLX4_CQE_CVLAN_PRESENT_MASK) && !timestamp_en) { + wc->vlan_id = be16_to_cpu(cqe->sl_vid) & + MLX4_CQE_VID_MASK; + wc->wc_flags |= IB_WC_WITH_VLAN; + } else { + wc->vlan_id = 0xffff; + } + if (!timestamp_en) { + memcpy(wc->smac, cqe->smac, 6); + wc->wc_flags |= IB_WC_WITH_SMAC; + } + } + + return 0; +} + +int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) +{ + struct mlx4_ib_cq *cq = to_mcq(ibcq); + struct mlx4_ib_qp *cur_qp = NULL; + unsigned long flags; + int npolled; + int err = 0; + + spin_lock_irqsave(&cq->lock, flags); + + for (npolled = 0; npolled < num_entries; ++npolled) { + err = mlx4_ib_poll_one(cq, &cur_qp, wc + npolled); + if (err) + break; + } + + mlx4_cq_set_ci(&cq->mcq); + + spin_unlock_irqrestore(&cq->lock, flags); + + if (err == 0 || err == -EAGAIN) + return npolled; + else + return err; +} + +int mlx4_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) +{ + mlx4_cq_arm(&to_mcq(ibcq)->mcq, + (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? + MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT, + to_mdev(ibcq->device)->priv_uar.map, + MLX4_GET_DOORBELL_LOCK(&to_mdev(ibcq->device)->uar_lock)); + + return 0; +} + +void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) +{ + u32 prod_index; + int nfreed = 0; + struct mlx4_cqe *cqe, *dest; + u8 owner_bit; + int cqe_inc = cq->buf.entry_size == 64 ? 1 : 0; + + /* + * First we need to find the current producer index, so we + * know where to start cleaning from. It doesn't matter if HW + * adds new entries after this loop -- the QP we're worried + * about is already in RESET, so the new entries won't come + * from our QP and therefore don't need to be checked. + */ + for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index) + if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe) + break; + + /* + * Now sweep backwards through the CQ, removing CQ entries + * that match our QP by copying older entries on top of them. + */ + while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) { + cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); + cqe += cqe_inc; + + if ((be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK) == qpn) { + if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)) + mlx4_ib_free_srq_wqe(srq, be16_to_cpu(cqe->wqe_index)); + ++nfreed; + } else if (nfreed) { + dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe); + dest += cqe_inc; + + owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK; + memcpy(dest, cqe, sizeof *cqe); + dest->owner_sr_opcode = owner_bit | + (dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK); + } + } + + if (nfreed) { + cq->mcq.cons_index += nfreed; + /* + * Make sure update of buffer contents is done before + * updating consumer index. + */ + wmb(); + mlx4_cq_set_ci(&cq->mcq); + } +} + +void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) +{ + spin_lock_irq(&cq->lock); + __mlx4_ib_cq_clean(cq, qpn, srq); + spin_unlock_irq(&cq->lock); +} Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_cq.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_mad.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_mad.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_mad.c (revision 329159) @@ -0,0 +1,2190 @@ +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "mlx4_ib.h" + +enum { + MLX4_IB_VENDOR_CLASS1 = 0x9, + MLX4_IB_VENDOR_CLASS2 = 0xa +}; + +#define MLX4_TUN_SEND_WRID_SHIFT 34 +#define MLX4_TUN_QPN_SHIFT 32 +#define MLX4_TUN_WRID_RECV (((u64) 1) << MLX4_TUN_SEND_WRID_SHIFT) +#define MLX4_TUN_SET_WRID_QPN(a) (((u64) ((a) & 0x3)) << MLX4_TUN_QPN_SHIFT) + +#define MLX4_TUN_IS_RECV(a) (((a) >> MLX4_TUN_SEND_WRID_SHIFT) & 0x1) +#define MLX4_TUN_WRID_QPN(a) (((a) >> MLX4_TUN_QPN_SHIFT) & 0x3) + + /* Port mgmt change event handling */ + +#define GET_BLK_PTR_FROM_EQE(eqe) be32_to_cpu(eqe->event.port_mgmt_change.params.tbl_change_info.block_ptr) +#define GET_MASK_FROM_EQE(eqe) be32_to_cpu(eqe->event.port_mgmt_change.params.tbl_change_info.tbl_entries_mask) +#define NUM_IDX_IN_PKEY_TBL_BLK 32 +#define GUID_TBL_ENTRY_SIZE 8 /* size in bytes */ +#define GUID_TBL_BLK_NUM_ENTRIES 8 +#define GUID_TBL_BLK_SIZE (GUID_TBL_ENTRY_SIZE * GUID_TBL_BLK_NUM_ENTRIES) + +struct mlx4_mad_rcv_buf { + struct ib_grh grh; + u8 payload[256]; +} __packed; + +struct mlx4_mad_snd_buf { + u8 payload[256]; +} __packed; + +struct mlx4_tunnel_mad { + struct ib_grh grh; + struct mlx4_ib_tunnel_header hdr; + struct ib_mad mad; +} __packed; + +struct mlx4_rcv_tunnel_mad { + struct mlx4_rcv_tunnel_hdr hdr; + struct ib_grh grh; + struct ib_mad mad; +} __packed; + +static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u8 port_num); +static void handle_lid_change_event(struct mlx4_ib_dev *dev, u8 port_num); +static void __propagate_pkey_ev(struct mlx4_ib_dev *dev, int port_num, + int block, u32 change_bitmap); + +__be64 mlx4_ib_gen_node_guid(void) +{ +#define NODE_GUID_HI ((u64) (((u64)IB_OPENIB_OUI) << 40)) + return cpu_to_be64(NODE_GUID_HI | random()); +} + +__be64 mlx4_ib_get_new_demux_tid(struct mlx4_ib_demux_ctx *ctx) +{ + return cpu_to_be64(atomic_inc_return(&ctx->tid)) | + cpu_to_be64(0xff00000000000000LL); +} + +int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int mad_ifc_flags, + int port, struct ib_wc *in_wc, struct ib_grh *in_grh, + void *in_mad, void *response_mad) +{ + struct mlx4_cmd_mailbox *inmailbox, *outmailbox; + void *inbox; + int err; + u32 in_modifier = port; + u8 op_modifier = 0; + + inmailbox = mlx4_alloc_cmd_mailbox(dev->dev); + if (IS_ERR(inmailbox)) + return PTR_ERR(inmailbox); + inbox = inmailbox->buf; + + outmailbox = mlx4_alloc_cmd_mailbox(dev->dev); + if (IS_ERR(outmailbox)) { + mlx4_free_cmd_mailbox(dev->dev, inmailbox); + return PTR_ERR(outmailbox); + } + + memcpy(inbox, in_mad, 256); + + /* + * Key check traps can't be generated unless we have in_wc to + * tell us where to send the trap. + */ + if ((mad_ifc_flags & MLX4_MAD_IFC_IGNORE_MKEY) || !in_wc) + op_modifier |= 0x1; + if ((mad_ifc_flags & MLX4_MAD_IFC_IGNORE_BKEY) || !in_wc) + op_modifier |= 0x2; + if (mlx4_is_mfunc(dev->dev) && + (mad_ifc_flags & MLX4_MAD_IFC_NET_VIEW || in_wc)) + op_modifier |= 0x8; + + if (in_wc) { + struct { + __be32 my_qpn; + u32 reserved1; + __be32 rqpn; + u8 sl; + u8 g_path; + u16 reserved2[2]; + __be16 pkey; + u32 reserved3[11]; + u8 grh[40]; + } *ext_info; + + memset(inbox + 256, 0, 256); + ext_info = inbox + 256; + + ext_info->my_qpn = cpu_to_be32(in_wc->qp->qp_num); + ext_info->rqpn = cpu_to_be32(in_wc->src_qp); + ext_info->sl = in_wc->sl << 4; + ext_info->g_path = in_wc->dlid_path_bits | + (in_wc->wc_flags & IB_WC_GRH ? 0x80 : 0); + ext_info->pkey = cpu_to_be16(in_wc->pkey_index); + + if (in_grh) + memcpy(ext_info->grh, in_grh, 40); + + op_modifier |= 0x4; + + in_modifier |= in_wc->slid << 16; + } + + err = mlx4_cmd_box(dev->dev, inmailbox->dma, outmailbox->dma, in_modifier, + mlx4_is_master(dev->dev) ? (op_modifier & ~0x8) : op_modifier, + MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C, + (op_modifier & 0x8) ? MLX4_CMD_NATIVE : MLX4_CMD_WRAPPED); + + if (!err) + memcpy(response_mad, outmailbox->buf, 256); + + mlx4_free_cmd_mailbox(dev->dev, inmailbox); + mlx4_free_cmd_mailbox(dev->dev, outmailbox); + + return err; +} + +static void update_sm_ah(struct mlx4_ib_dev *dev, u8 port_num, u16 lid, u8 sl) +{ + struct ib_ah *new_ah; + struct ib_ah_attr ah_attr; + unsigned long flags; + + if (!dev->send_agent[port_num - 1][0]) + return; + + memset(&ah_attr, 0, sizeof ah_attr); + ah_attr.dlid = lid; + ah_attr.sl = sl; + ah_attr.port_num = port_num; + + new_ah = ib_create_ah(dev->send_agent[port_num - 1][0]->qp->pd, + &ah_attr); + if (IS_ERR(new_ah)) + return; + + spin_lock_irqsave(&dev->sm_lock, flags); + if (dev->sm_ah[port_num - 1]) + ib_destroy_ah(dev->sm_ah[port_num - 1]); + dev->sm_ah[port_num - 1] = new_ah; + spin_unlock_irqrestore(&dev->sm_lock, flags); +} + +/* + * Snoop SM MADs for port info, GUID info, and P_Key table sets, so we can + * synthesize LID change, Client-Rereg, GID change, and P_Key change events. + */ +static void smp_snoop(struct ib_device *ibdev, u8 port_num, struct ib_mad *mad, + u16 prev_lid) +{ + struct ib_port_info *pinfo; + u16 lid; + __be16 *base; + u32 bn, pkey_change_bitmap; + int i; + + + struct mlx4_ib_dev *dev = to_mdev(ibdev); + if ((mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || + mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && + mad->mad_hdr.method == IB_MGMT_METHOD_SET) + switch (mad->mad_hdr.attr_id) { + case IB_SMP_ATTR_PORT_INFO: + pinfo = (struct ib_port_info *) ((struct ib_smp *) mad)->data; + lid = be16_to_cpu(pinfo->lid); + + update_sm_ah(dev, port_num, + be16_to_cpu(pinfo->sm_lid), + pinfo->neighbormtu_mastersmsl & 0xf); + + if (pinfo->clientrereg_resv_subnetto & 0x80) + handle_client_rereg_event(dev, port_num); + + if (prev_lid != lid) + handle_lid_change_event(dev, port_num); + break; + + case IB_SMP_ATTR_PKEY_TABLE: + if (!mlx4_is_mfunc(dev->dev)) { + mlx4_ib_dispatch_event(dev, port_num, + IB_EVENT_PKEY_CHANGE); + break; + } + + /* at this point, we are running in the master. + * Slaves do not receive SMPs. + */ + bn = be32_to_cpu(((struct ib_smp *)mad)->attr_mod) & 0xFFFF; + base = (__be16 *) &(((struct ib_smp *)mad)->data[0]); + pkey_change_bitmap = 0; + for (i = 0; i < 32; i++) { + pr_debug("PKEY[%d] = x%x\n", + i + bn*32, be16_to_cpu(base[i])); + if (be16_to_cpu(base[i]) != + dev->pkeys.phys_pkey_cache[port_num - 1][i + bn*32]) { + pkey_change_bitmap |= (1 << i); + dev->pkeys.phys_pkey_cache[port_num - 1][i + bn*32] = + be16_to_cpu(base[i]); + } + } + pr_debug("PKEY Change event: port=%d, " + "block=0x%x, change_bitmap=0x%x\n", + port_num, bn, pkey_change_bitmap); + + if (pkey_change_bitmap) { + mlx4_ib_dispatch_event(dev, port_num, + IB_EVENT_PKEY_CHANGE); + if (!dev->sriov.is_going_down) + __propagate_pkey_ev(dev, port_num, bn, + pkey_change_bitmap); + } + break; + + case IB_SMP_ATTR_GUID_INFO: + /* paravirtualized master's guid is guid 0 -- does not change */ + if (!mlx4_is_master(dev->dev)) + mlx4_ib_dispatch_event(dev, port_num, + IB_EVENT_GID_CHANGE); + /*if master, notify relevant slaves*/ + if (mlx4_is_master(dev->dev) && + !dev->sriov.is_going_down) { + bn = be32_to_cpu(((struct ib_smp *)mad)->attr_mod); + mlx4_ib_update_cache_on_guid_change(dev, bn, port_num, + (u8 *)(&((struct ib_smp *)mad)->data)); + mlx4_ib_notify_slaves_on_guid_change(dev, bn, port_num, + (u8 *)(&((struct ib_smp *)mad)->data)); + } + break; + + default: + break; + } +} + +static void __propagate_pkey_ev(struct mlx4_ib_dev *dev, int port_num, + int block, u32 change_bitmap) +{ + int i, ix, slave, err; + int have_event = 0; + + for (slave = 0; slave < dev->dev->caps.sqp_demux; slave++) { + if (slave == mlx4_master_func_num(dev->dev)) + continue; + if (!mlx4_is_slave_active(dev->dev, slave)) + continue; + + have_event = 0; + for (i = 0; i < 32; i++) { + if (!(change_bitmap & (1 << i))) + continue; + for (ix = 0; + ix < dev->dev->caps.pkey_table_len[port_num]; ix++) { + if (dev->pkeys.virt2phys_pkey[slave][port_num - 1] + [ix] == i + 32 * block) { + err = mlx4_gen_pkey_eqe(dev->dev, slave, port_num); + pr_debug("propagate_pkey_ev: slave %d," + " port %d, ix %d (%d)\n", + slave, port_num, ix, err); + have_event = 1; + break; + } + } + if (have_event) + break; + } + } +} + +static void node_desc_override(struct ib_device *dev, + struct ib_mad *mad) +{ + unsigned long flags; + + if ((mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || + mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && + mad->mad_hdr.method == IB_MGMT_METHOD_GET_RESP && + mad->mad_hdr.attr_id == IB_SMP_ATTR_NODE_DESC) { + spin_lock_irqsave(&to_mdev(dev)->sm_lock, flags); + memcpy(((struct ib_smp *) mad)->data, dev->node_desc, 64); + spin_unlock_irqrestore(&to_mdev(dev)->sm_lock, flags); + } +} + +static void forward_trap(struct mlx4_ib_dev *dev, u8 port_num, struct ib_mad *mad) +{ + int qpn = mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED; + struct ib_mad_send_buf *send_buf; + struct ib_mad_agent *agent = dev->send_agent[port_num - 1][qpn]; + int ret; + unsigned long flags; + + if (agent) { + send_buf = ib_create_send_mad(agent, qpn, 0, 0, IB_MGMT_MAD_HDR, + IB_MGMT_MAD_DATA, GFP_ATOMIC); + if (IS_ERR(send_buf)) + return; + /* + * We rely here on the fact that MLX QPs don't use the + * address handle after the send is posted (this is + * wrong following the IB spec strictly, but we know + * it's OK for our devices). + */ + spin_lock_irqsave(&dev->sm_lock, flags); + memcpy(send_buf->mad, mad, sizeof *mad); + if ((send_buf->ah = dev->sm_ah[port_num - 1])) + ret = ib_post_send_mad(send_buf, NULL); + else + ret = -EINVAL; + spin_unlock_irqrestore(&dev->sm_lock, flags); + + if (ret) + ib_free_send_mad(send_buf); + } +} + +static int mlx4_ib_demux_sa_handler(struct ib_device *ibdev, int port, int slave, + struct ib_sa_mad *sa_mad) +{ + int ret = 0; + + /* dispatch to different sa handlers */ + switch (be16_to_cpu(sa_mad->mad_hdr.attr_id)) { + case IB_SA_ATTR_MC_MEMBER_REC: + ret = mlx4_ib_mcg_demux_handler(ibdev, port, slave, sa_mad); + break; + default: + break; + } + return ret; +} + +int mlx4_ib_find_real_gid(struct ib_device *ibdev, u8 port, __be64 guid) +{ + struct mlx4_ib_dev *dev = to_mdev(ibdev); + int i; + + for (i = 0; i < dev->dev->caps.sqp_demux; i++) { + if (dev->sriov.demux[port - 1].guid_cache[i] == guid) + return i; + } + return -1; +} + + +static int find_slave_port_pkey_ix(struct mlx4_ib_dev *dev, int slave, + u8 port, u16 pkey, u16 *ix) +{ + int i, ret; + u8 unassigned_pkey_ix, pkey_ix, partial_ix = 0xFF; + u16 slot_pkey; + + if (slave == mlx4_master_func_num(dev->dev)) + return ib_find_cached_pkey(&dev->ib_dev, port, pkey, ix); + + unassigned_pkey_ix = dev->dev->phys_caps.pkey_phys_table_len[port] - 1; + + for (i = 0; i < dev->dev->caps.pkey_table_len[port]; i++) { + if (dev->pkeys.virt2phys_pkey[slave][port - 1][i] == unassigned_pkey_ix) + continue; + + pkey_ix = dev->pkeys.virt2phys_pkey[slave][port - 1][i]; + + ret = ib_get_cached_pkey(&dev->ib_dev, port, pkey_ix, &slot_pkey); + if (ret) + continue; + if ((slot_pkey & 0x7FFF) == (pkey & 0x7FFF)) { + if (slot_pkey & 0x8000) { + *ix = (u16) pkey_ix; + return 0; + } else { + /* take first partial pkey index found */ + if (partial_ix == 0xFF) + partial_ix = pkey_ix; + } + } + } + + if (partial_ix < 0xFF) { + *ix = (u16) partial_ix; + return 0; + } + + return -EINVAL; +} + +int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port, + enum ib_qp_type dest_qpt, struct ib_wc *wc, + struct ib_grh *grh, struct ib_mad *mad) +{ + struct ib_sge list; + struct ib_send_wr wr, *bad_wr; + struct mlx4_ib_demux_pv_ctx *tun_ctx; + struct mlx4_ib_demux_pv_qp *tun_qp; + struct mlx4_rcv_tunnel_mad *tun_mad; + struct ib_ah_attr attr; + struct ib_ah *ah; + struct ib_qp *src_qp = NULL; + unsigned tun_tx_ix = 0; + int dqpn; + int ret = 0; + u16 tun_pkey_ix; + u16 cached_pkey; + u8 is_eth = dev->dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH; + + if (dest_qpt > IB_QPT_GSI) + return -EINVAL; + + tun_ctx = dev->sriov.demux[port-1].tun[slave]; + + /* check if proxy qp created */ + if (!tun_ctx || tun_ctx->state != DEMUX_PV_STATE_ACTIVE) + return -EAGAIN; + + /* QP0 forwarding only for Dom0 */ + if (!dest_qpt && (mlx4_master_func_num(dev->dev) != slave)) + return -EINVAL; + + if (!dest_qpt) + tun_qp = &tun_ctx->qp[0]; + else + tun_qp = &tun_ctx->qp[1]; + + /* compute P_Key index to put in tunnel header for slave */ + if (dest_qpt) { + u16 pkey_ix; + ret = ib_get_cached_pkey(&dev->ib_dev, port, wc->pkey_index, &cached_pkey); + if (ret) + return -EINVAL; + + ret = find_slave_port_pkey_ix(dev, slave, port, cached_pkey, &pkey_ix); + if (ret) + return -EINVAL; + tun_pkey_ix = pkey_ix; + } else + tun_pkey_ix = dev->pkeys.virt2phys_pkey[slave][port - 1][0]; + + dqpn = dev->dev->phys_caps.base_proxy_sqpn + 8 * slave + port + (dest_qpt * 2) - 1; + + /* get tunnel tx data buf for slave */ + src_qp = tun_qp->qp; + + /* create ah. Just need an empty one with the port num for the post send. + * The driver will set the force loopback bit in post_send */ + memset(&attr, 0, sizeof attr); + attr.port_num = port; + if (is_eth) { + memcpy(&attr.grh.dgid.raw[0], &grh->dgid.raw[0], 16); + attr.ah_flags = IB_AH_GRH; + } + ah = ib_create_ah(tun_ctx->pd, &attr); + if (IS_ERR(ah)) + return -ENOMEM; + + /* allocate tunnel tx buf after pass failure returns */ + spin_lock(&tun_qp->tx_lock); + if (tun_qp->tx_ix_head - tun_qp->tx_ix_tail >= + (MLX4_NUM_TUNNEL_BUFS - 1)) + ret = -EAGAIN; + else + tun_tx_ix = (++tun_qp->tx_ix_head) & (MLX4_NUM_TUNNEL_BUFS - 1); + spin_unlock(&tun_qp->tx_lock); + if (ret) + goto out; + + tun_mad = (struct mlx4_rcv_tunnel_mad *) (tun_qp->tx_ring[tun_tx_ix].buf.addr); + if (tun_qp->tx_ring[tun_tx_ix].ah) + ib_destroy_ah(tun_qp->tx_ring[tun_tx_ix].ah); + tun_qp->tx_ring[tun_tx_ix].ah = ah; + ib_dma_sync_single_for_cpu(&dev->ib_dev, + tun_qp->tx_ring[tun_tx_ix].buf.map, + sizeof (struct mlx4_rcv_tunnel_mad), + DMA_TO_DEVICE); + + /* copy over to tunnel buffer */ + if (grh) + memcpy(&tun_mad->grh, grh, sizeof *grh); + memcpy(&tun_mad->mad, mad, sizeof *mad); + + /* adjust tunnel data */ + tun_mad->hdr.pkey_index = cpu_to_be16(tun_pkey_ix); + tun_mad->hdr.flags_src_qp = cpu_to_be32(wc->src_qp & 0xFFFFFF); + tun_mad->hdr.g_ml_path = (grh && (wc->wc_flags & IB_WC_GRH)) ? 0x80 : 0; + + if (is_eth) { + u16 vlan = 0; + if (mlx4_get_slave_default_vlan(dev->dev, port, slave, &vlan, + NULL)) { + if (vlan != wc->vlan_id) + /* VST and default vlan is not the packet vlan drop the + * packet*/ + goto out; + else + /* VST , remove hide the vlan from the VF */ + vlan = 0; + } else { + vlan = wc->vlan_id; + } + + tun_mad->hdr.sl_vid = cpu_to_be16(vlan); + memcpy((char *)&tun_mad->hdr.mac_31_0, &(wc->smac[0]), 4); + memcpy((char *)&tun_mad->hdr.slid_mac_47_32, &(wc->smac[4]), 2); + } else { + tun_mad->hdr.sl_vid = cpu_to_be16(((u16)(wc->sl)) << 12); + tun_mad->hdr.slid_mac_47_32 = cpu_to_be16(wc->slid); + } + + ib_dma_sync_single_for_device(&dev->ib_dev, + tun_qp->tx_ring[tun_tx_ix].buf.map, + sizeof (struct mlx4_rcv_tunnel_mad), + DMA_TO_DEVICE); + + list.addr = tun_qp->tx_ring[tun_tx_ix].buf.map; + list.length = sizeof (struct mlx4_rcv_tunnel_mad); + list.lkey = tun_ctx->mr->lkey; + + wr.wr.ud.ah = ah; + wr.wr.ud.port_num = port; + wr.wr.ud.remote_qkey = IB_QP_SET_QKEY; + wr.wr.ud.remote_qpn = dqpn; + wr.next = NULL; + wr.wr_id = ((u64) tun_tx_ix) | MLX4_TUN_SET_WRID_QPN(dest_qpt); + wr.sg_list = &list; + wr.num_sge = 1; + wr.opcode = IB_WR_SEND; + wr.send_flags = IB_SEND_SIGNALED; + + ret = ib_post_send(src_qp, &wr, &bad_wr); +out: + if (ret) + ib_destroy_ah(ah); + return ret; +} + +static int mlx4_ib_demux_mad(struct ib_device *ibdev, u8 port, + struct ib_wc *wc, struct ib_grh *grh, + struct ib_mad *mad) +{ + struct mlx4_ib_dev *dev = to_mdev(ibdev); + int err; + int slave; + u8 *slave_id; + int is_eth = 0; + + if (rdma_port_get_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND) + is_eth = 0; + else + is_eth = 1; + + if (is_eth) { + if (!(wc->wc_flags & IB_WC_GRH)) { + mlx4_ib_warn(ibdev, "RoCE grh not present.\n"); + return -EINVAL; + } + if (mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_CM) { + mlx4_ib_warn(ibdev, "RoCE mgmt class is not CM\n"); + return -EINVAL; + } + if (mlx4_get_slave_from_roce_gid(dev->dev, port, grh->dgid.raw, &slave)) { + mlx4_ib_warn(ibdev, "failed matching grh\n"); + return -ENOENT; + } + if (slave >= dev->dev->caps.sqp_demux) { + mlx4_ib_warn(ibdev, "slave id: %d is bigger than allowed:%d\n", + slave, dev->dev->caps.sqp_demux); + return -ENOENT; + } + + if (mlx4_ib_demux_cm_handler(ibdev, port, &slave, mad, is_eth)) + return 0; + + err = mlx4_ib_send_to_slave(dev, slave, port, wc->qp->qp_type, wc, grh, mad); + if (err) + pr_debug("failed sending to slave %d via tunnel qp (%d)\n", + slave, err); + return 0; + } + + /* Initially assume that this mad is for us */ + slave = mlx4_master_func_num(dev->dev); + + /* See if the slave id is encoded in a response mad */ + if (mad->mad_hdr.method & 0x80) { + slave_id = (u8 *) &mad->mad_hdr.tid; + slave = *slave_id; + if (slave != 255) /*255 indicates the dom0*/ + *slave_id = 0; /* remap tid */ + } + + /* If a grh is present, we demux according to it */ + if (wc->wc_flags & IB_WC_GRH) { + slave = mlx4_ib_find_real_gid(ibdev, port, grh->dgid.global.interface_id); + if (slave < 0) { + mlx4_ib_warn(ibdev, "failed matching grh\n"); + return -ENOENT; + } + } + /* Class-specific handling */ + switch (mad->mad_hdr.mgmt_class) { + case IB_MGMT_CLASS_SUBN_ADM: + if (mlx4_ib_demux_sa_handler(ibdev, port, slave, + (struct ib_sa_mad *) mad)) + return 0; + break; + case IB_MGMT_CLASS_CM: + if (mlx4_ib_demux_cm_handler(ibdev, port, &slave, mad, is_eth)) + return 0; + break; + case IB_MGMT_CLASS_DEVICE_MGMT: + if (mad->mad_hdr.method != IB_MGMT_METHOD_GET_RESP) + return 0; + break; + default: + /* Drop unsupported classes for slaves in tunnel mode */ + if (slave != mlx4_master_func_num(dev->dev)) { + pr_debug("dropping unsupported ingress mad from class:%d " + "for slave:%d\n", mad->mad_hdr.mgmt_class, slave); + return 0; + } + } + /*make sure that no slave==255 was not handled yet.*/ + if (slave >= dev->dev->caps.sqp_demux) { + mlx4_ib_warn(ibdev, "slave id: %d is bigger than allowed:%d\n", + slave, dev->dev->caps.sqp_demux); + return -ENOENT; + } + + err = mlx4_ib_send_to_slave(dev, slave, port, wc->qp->qp_type, wc, grh, mad); + if (err) + pr_debug("failed sending to slave %d via tunnel qp (%d)\n", + slave, err); + return 0; +} + +static int ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, + struct ib_wc *in_wc, struct ib_grh *in_grh, + struct ib_mad *in_mad, struct ib_mad *out_mad) +{ + u16 slid, prev_lid = 0; + int err; + struct ib_port_attr pattr; + + if (in_wc && in_wc->qp->qp_num) { + pr_debug("received MAD: slid:%d sqpn:%d " + "dlid_bits:%d dqpn:%d wc_flags:0x%x, cls %x, mtd %x, atr %x\n", + in_wc->slid, in_wc->src_qp, + in_wc->dlid_path_bits, + in_wc->qp->qp_num, + in_wc->wc_flags, + in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.method, + be16_to_cpu(in_mad->mad_hdr.attr_id)); + if (in_wc->wc_flags & IB_WC_GRH) { + pr_debug("sgid_hi:0x%016llx sgid_lo:0x%016llx\n", + (unsigned long long)be64_to_cpu(in_grh->sgid.global.subnet_prefix), + (unsigned long long)be64_to_cpu(in_grh->sgid.global.interface_id)); + pr_debug("dgid_hi:0x%016llx dgid_lo:0x%016llx\n", + (unsigned long long)be64_to_cpu(in_grh->dgid.global.subnet_prefix), + (unsigned long long)be64_to_cpu(in_grh->dgid.global.interface_id)); + } + } + + slid = in_wc ? in_wc->slid : be16_to_cpu(IB_LID_PERMISSIVE); + + if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP && slid == 0) { + forward_trap(to_mdev(ibdev), port_num, in_mad); + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; + } + + if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || + in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) { + if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && + in_mad->mad_hdr.method != IB_MGMT_METHOD_SET && + in_mad->mad_hdr.method != IB_MGMT_METHOD_TRAP_REPRESS) + return IB_MAD_RESULT_SUCCESS; + + /* + * Don't process SMInfo queries -- the SMA can't handle them. + */ + if (in_mad->mad_hdr.attr_id == IB_SMP_ATTR_SM_INFO) + return IB_MAD_RESULT_SUCCESS; + } else if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT || + in_mad->mad_hdr.mgmt_class == MLX4_IB_VENDOR_CLASS1 || + in_mad->mad_hdr.mgmt_class == MLX4_IB_VENDOR_CLASS2 || + in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_CONG_MGMT) { + if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && + in_mad->mad_hdr.method != IB_MGMT_METHOD_SET) + return IB_MAD_RESULT_SUCCESS; + } else + return IB_MAD_RESULT_SUCCESS; + + if ((in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || + in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && + in_mad->mad_hdr.method == IB_MGMT_METHOD_SET && + in_mad->mad_hdr.attr_id == IB_SMP_ATTR_PORT_INFO && + !ib_query_port(ibdev, port_num, &pattr)) + prev_lid = pattr.lid; + + err = mlx4_MAD_IFC(to_mdev(ibdev), + (mad_flags & IB_MAD_IGNORE_MKEY ? MLX4_MAD_IFC_IGNORE_MKEY : 0) | + (mad_flags & IB_MAD_IGNORE_BKEY ? MLX4_MAD_IFC_IGNORE_BKEY : 0) | + MLX4_MAD_IFC_NET_VIEW, + port_num, in_wc, in_grh, in_mad, out_mad); + if (err) + return IB_MAD_RESULT_FAILURE; + + if (!out_mad->mad_hdr.status) { + if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV)) + smp_snoop(ibdev, port_num, in_mad, prev_lid); + /* slaves get node desc from FW */ + if (!mlx4_is_slave(to_mdev(ibdev)->dev)) + node_desc_override(ibdev, out_mad); + } + + /* set return bit in status of directed route responses */ + if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) + out_mad->mad_hdr.status |= cpu_to_be16(1 << 15); + + if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP_REPRESS) + /* no response for trap repress */ + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; + + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; +} + +static void edit_counter(struct mlx4_counter *cnt, void *counters, + __be16 attr_id) +{ + switch (attr_id) { + case IB_PMA_PORT_COUNTERS: + { + struct ib_pma_portcounters *pma_cnt = + (struct ib_pma_portcounters *)counters; + + ASSIGN_32BIT_COUNTER(pma_cnt->port_xmit_data, + (be64_to_cpu(cnt->tx_bytes) >> 2)); + ASSIGN_32BIT_COUNTER(pma_cnt->port_rcv_data, + (be64_to_cpu(cnt->rx_bytes) >> 2)); + ASSIGN_32BIT_COUNTER(pma_cnt->port_xmit_packets, + be64_to_cpu(cnt->tx_frames)); + ASSIGN_32BIT_COUNTER(pma_cnt->port_rcv_packets, + be64_to_cpu(cnt->rx_frames)); + break; + } + case IB_PMA_PORT_COUNTERS_EXT: + { + struct ib_pma_portcounters_ext *pma_cnt_ext = + (struct ib_pma_portcounters_ext *)counters; + + pma_cnt_ext->port_xmit_data = + cpu_to_be64(be64_to_cpu(cnt->tx_bytes) >> 2); + pma_cnt_ext->port_rcv_data = + cpu_to_be64(be64_to_cpu(cnt->rx_bytes) >> 2); + pma_cnt_ext->port_xmit_packets = cnt->tx_frames; + pma_cnt_ext->port_rcv_packets = cnt->rx_frames; + break; + } + default: + break; + } +} + +static int iboe_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, + const struct ib_wc *in_wc, const struct ib_grh *in_grh, + const struct ib_mad *in_mad, struct ib_mad *out_mad) +{ + struct mlx4_counter counter_stats; + struct mlx4_ib_dev *dev = to_mdev(ibdev); + int err; + u32 counter_index = dev->counters[port_num - 1].counter_index & 0xffff; + + if (in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_PERF_MGMT) + return -EINVAL; + + /* in case of default counter IB shares the counter with ETH */ + /* the state could be -EEXIST or -ENOSPC */ + if (dev->counters[port_num - 1].status) { + memset(out_mad->data, 0, sizeof out_mad->data); + err = IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; + } else { + memset(&counter_stats, 0, sizeof(counter_stats)); + err = mlx4_get_counter_stats(dev->dev, + counter_index, + &counter_stats, 0); + if (err) + return IB_MAD_RESULT_FAILURE; + + memset(out_mad->data, 0, sizeof(out_mad->data)); + err = IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; + switch (counter_stats.counter_mode & 0xf) { + case 0: + edit_counter(&counter_stats, + (void *)(out_mad->data + 40), + in_mad->mad_hdr.attr_id); + break; + default: + err = IB_MAD_RESULT_FAILURE; + } + } + + return err; +} + +int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, + struct ib_wc *in_wc, struct ib_grh *in_grh, + struct ib_mad *in_mad, struct ib_mad *out_mad) +{ + switch (rdma_port_get_link_layer(ibdev, port_num)) { + case IB_LINK_LAYER_INFINIBAND: + return ib_process_mad(ibdev, mad_flags, port_num, in_wc, + in_grh, in_mad, out_mad); + case IB_LINK_LAYER_ETHERNET: + return iboe_process_mad(ibdev, mad_flags, port_num, in_wc, + in_grh, in_mad, out_mad); + default: + return -EINVAL; + } +} + +static void send_handler(struct ib_mad_agent *agent, + struct ib_mad_send_wc *mad_send_wc) +{ + if (mad_send_wc->send_buf->context[0]) + ib_destroy_ah(mad_send_wc->send_buf->context[0]); + ib_free_send_mad(mad_send_wc->send_buf); +} + +int mlx4_ib_mad_init(struct mlx4_ib_dev *dev) +{ + struct ib_mad_agent *agent; + int p, q; + int ret; + enum rdma_link_layer ll; + + for (p = 0; p < dev->num_ports; ++p) { + ll = rdma_port_get_link_layer(&dev->ib_dev, p + 1); + for (q = 0; q <= 1; ++q) { + if (ll == IB_LINK_LAYER_INFINIBAND) { + agent = ib_register_mad_agent(&dev->ib_dev, p + 1, + q ? IB_QPT_GSI : IB_QPT_SMI, + NULL, 0, send_handler, + NULL, NULL); + if (IS_ERR(agent)) { + ret = PTR_ERR(agent); + goto err; + } + dev->send_agent[p][q] = agent; + } else + dev->send_agent[p][q] = NULL; + } + } + + return 0; + +err: + for (p = 0; p < dev->num_ports; ++p) + for (q = 0; q <= 1; ++q) + if (dev->send_agent[p][q]) + ib_unregister_mad_agent(dev->send_agent[p][q]); + + return ret; +} + +void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev) +{ + struct ib_mad_agent *agent; + int p, q; + + for (p = 0; p < dev->num_ports; ++p) { + for (q = 0; q <= 1; ++q) { + agent = dev->send_agent[p][q]; + if (agent) { + dev->send_agent[p][q] = NULL; + ib_unregister_mad_agent(agent); + } + } + + if (dev->sm_ah[p]) + ib_destroy_ah(dev->sm_ah[p]); + } +} + +static void handle_lid_change_event(struct mlx4_ib_dev *dev, u8 port_num) +{ + mlx4_ib_dispatch_event(dev, port_num, IB_EVENT_LID_CHANGE); + + if (mlx4_is_master(dev->dev) && !dev->sriov.is_going_down) + mlx4_gen_slaves_port_mgt_ev(dev->dev, port_num, + MLX4_EQ_PORT_INFO_LID_CHANGE_MASK); +} + +static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u8 port_num) +{ + /* re-configure the alias-guid and mcg's */ + if (mlx4_is_master(dev->dev)) { + mlx4_ib_invalidate_all_guid_record(dev, port_num); + + if (!dev->sriov.is_going_down) { + mlx4_ib_mcg_port_cleanup(&dev->sriov.demux[port_num - 1], 0); + mlx4_gen_slaves_port_mgt_ev(dev->dev, port_num, + MLX4_EQ_PORT_INFO_CLIENT_REREG_MASK); + } + } + mlx4_ib_dispatch_event(dev, port_num, IB_EVENT_CLIENT_REREGISTER); +} + +static void propagate_pkey_ev(struct mlx4_ib_dev *dev, int port_num, + struct mlx4_eqe *eqe) +{ + __propagate_pkey_ev(dev, port_num, GET_BLK_PTR_FROM_EQE(eqe), + GET_MASK_FROM_EQE(eqe)); +} + +static void handle_slaves_guid_change(struct mlx4_ib_dev *dev, u8 port_num, + u32 guid_tbl_blk_num, u32 change_bitmap) +{ + struct ib_smp *in_mad = NULL; + struct ib_smp *out_mad = NULL; + u16 i; + + if (!mlx4_is_mfunc(dev->dev) || !mlx4_is_master(dev->dev)) + return; + + in_mad = kmalloc(sizeof *in_mad, GFP_KERNEL); + out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); + if (!in_mad || !out_mad) { + mlx4_ib_warn(&dev->ib_dev, "failed to allocate memory for guid info mads\n"); + goto out; + } + + guid_tbl_blk_num *= 4; + + for (i = 0; i < 4; i++) { + if (change_bitmap && (!((change_bitmap >> (8 * i)) & 0xff))) + continue; + memset(in_mad, 0, sizeof *in_mad); + memset(out_mad, 0, sizeof *out_mad); + + in_mad->base_version = 1; + in_mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; + in_mad->class_version = 1; + in_mad->method = IB_MGMT_METHOD_GET; + in_mad->attr_id = IB_SMP_ATTR_GUID_INFO; + in_mad->attr_mod = cpu_to_be32(guid_tbl_blk_num + i); + + if (mlx4_MAD_IFC(dev, + MLX4_MAD_IFC_IGNORE_KEYS | MLX4_MAD_IFC_NET_VIEW, + port_num, NULL, NULL, in_mad, out_mad)) { + mlx4_ib_warn(&dev->ib_dev, "Failed in get GUID INFO MAD_IFC\n"); + goto out; + } + + mlx4_ib_update_cache_on_guid_change(dev, guid_tbl_blk_num + i, + port_num, + (u8 *)(&((struct ib_smp *)out_mad)->data)); + mlx4_ib_notify_slaves_on_guid_change(dev, guid_tbl_blk_num + i, + port_num, + (u8 *)(&((struct ib_smp *)out_mad)->data)); + } + +out: + kfree(in_mad); + kfree(out_mad); + return; +} + +void handle_port_mgmt_change_event(struct work_struct *work) +{ + struct ib_event_work *ew = container_of(work, struct ib_event_work, work); + struct mlx4_ib_dev *dev = ew->ib_dev; + struct mlx4_eqe *eqe = &(ew->ib_eqe); + u8 port = eqe->event.port_mgmt_change.port; + u32 changed_attr; + u32 tbl_block; + u32 change_bitmap; + + switch (eqe->subtype) { + case MLX4_DEV_PMC_SUBTYPE_PORT_INFO: + changed_attr = be32_to_cpu(eqe->event.port_mgmt_change.params.port_info.changed_attr); + + /* Update the SM ah - This should be done before handling + the other changed attributes so that MADs can be sent to the SM */ + if (changed_attr & MSTR_SM_CHANGE_MASK) { + u16 lid = be16_to_cpu(eqe->event.port_mgmt_change.params.port_info.mstr_sm_lid); + u8 sl = eqe->event.port_mgmt_change.params.port_info.mstr_sm_sl & 0xf; + update_sm_ah(dev, port, lid, sl); + } + + /* Check if it is a lid change event */ + if (changed_attr & MLX4_EQ_PORT_INFO_LID_CHANGE_MASK) + handle_lid_change_event(dev, port); + + /* Generate GUID changed event */ + if (changed_attr & MLX4_EQ_PORT_INFO_GID_PFX_CHANGE_MASK) { + mlx4_ib_dispatch_event(dev, port, IB_EVENT_GID_CHANGE); + /*if master, notify all slaves*/ + if (mlx4_is_master(dev->dev)) + mlx4_gen_slaves_port_mgt_ev(dev->dev, port, + MLX4_EQ_PORT_INFO_GID_PFX_CHANGE_MASK); + } + + if (changed_attr & MLX4_EQ_PORT_INFO_CLIENT_REREG_MASK) + handle_client_rereg_event(dev, port); + break; + + case MLX4_DEV_PMC_SUBTYPE_PKEY_TABLE: + mlx4_ib_dispatch_event(dev, port, IB_EVENT_PKEY_CHANGE); + if (mlx4_is_master(dev->dev) && !dev->sriov.is_going_down) + propagate_pkey_ev(dev, port, eqe); + break; + case MLX4_DEV_PMC_SUBTYPE_GUID_INFO: + /* paravirtualized master's guid is guid 0 -- does not change */ + if (!mlx4_is_master(dev->dev)) + mlx4_ib_dispatch_event(dev, port, IB_EVENT_GID_CHANGE); + /*if master, notify relevant slaves*/ + else if (!dev->sriov.is_going_down) { + tbl_block = GET_BLK_PTR_FROM_EQE(eqe); + change_bitmap = GET_MASK_FROM_EQE(eqe); + handle_slaves_guid_change(dev, port, tbl_block, change_bitmap); + } + break; + default: + pr_warn("Unsupported subtype 0x%x for " + "Port Management Change event\n", eqe->subtype); + } + + kfree(ew); +} + +void mlx4_ib_dispatch_event(struct mlx4_ib_dev *dev, u8 port_num, + enum ib_event_type type) +{ + struct ib_event event; + + event.device = &dev->ib_dev; + event.element.port_num = port_num; + event.event = type; + + ib_dispatch_event(&event); +} + +static void mlx4_ib_tunnel_comp_handler(struct ib_cq *cq, void *arg) +{ + unsigned long flags; + struct mlx4_ib_demux_pv_ctx *ctx = cq->cq_context; + struct mlx4_ib_dev *dev = to_mdev(ctx->ib_dev); + spin_lock_irqsave(&dev->sriov.going_down_lock, flags); + if (!dev->sriov.is_going_down && ctx->state == DEMUX_PV_STATE_ACTIVE) + queue_work(ctx->wq, &ctx->work); + spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); +} + +static int mlx4_ib_post_pv_qp_buf(struct mlx4_ib_demux_pv_ctx *ctx, + struct mlx4_ib_demux_pv_qp *tun_qp, + int index) +{ + struct ib_sge sg_list; + struct ib_recv_wr recv_wr, *bad_recv_wr; + int size; + + size = (tun_qp->qp->qp_type == IB_QPT_UD) ? + sizeof (struct mlx4_tunnel_mad) : sizeof (struct mlx4_mad_rcv_buf); + + sg_list.addr = tun_qp->ring[index].map; + sg_list.length = size; + sg_list.lkey = ctx->mr->lkey; + + recv_wr.next = NULL; + recv_wr.sg_list = &sg_list; + recv_wr.num_sge = 1; + recv_wr.wr_id = (u64) index | MLX4_TUN_WRID_RECV | + MLX4_TUN_SET_WRID_QPN(tun_qp->proxy_qpt); + ib_dma_sync_single_for_device(ctx->ib_dev, tun_qp->ring[index].map, + size, DMA_FROM_DEVICE); + return ib_post_recv(tun_qp->qp, &recv_wr, &bad_recv_wr); +} + +static int mlx4_ib_multiplex_sa_handler(struct ib_device *ibdev, int port, + int slave, struct ib_sa_mad *sa_mad) +{ + int ret = 0; + + /* dispatch to different sa handlers */ + switch (be16_to_cpu(sa_mad->mad_hdr.attr_id)) { + case IB_SA_ATTR_MC_MEMBER_REC: + ret = mlx4_ib_mcg_multiplex_handler(ibdev, port, slave, sa_mad); + break; + default: + break; + } + return ret; +} + +static int is_proxy_qp0(struct mlx4_ib_dev *dev, int qpn, int slave) +{ + int proxy_start = dev->dev->phys_caps.base_proxy_sqpn + 8 * slave; + + return (qpn >= proxy_start && qpn <= proxy_start + 1); +} + + +int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port, + enum ib_qp_type dest_qpt, u16 pkey_index, + u32 remote_qpn, u32 qkey, struct ib_ah_attr *attr, + u8 *s_mac, struct ib_mad *mad) +{ + struct ib_sge list; + struct ib_send_wr wr, *bad_wr; + struct mlx4_ib_demux_pv_ctx *sqp_ctx; + struct mlx4_ib_demux_pv_qp *sqp; + struct mlx4_mad_snd_buf *sqp_mad; + struct ib_ah *ah; + struct ib_qp *send_qp = NULL; + unsigned wire_tx_ix = 0; + int ret = 0; + u16 wire_pkey_ix; + int src_qpnum; + u8 sgid_index; + + + sqp_ctx = dev->sriov.sqps[port-1]; + + /* check if proxy qp created */ + if (!sqp_ctx || sqp_ctx->state != DEMUX_PV_STATE_ACTIVE) + return -EAGAIN; + + /* QP0 forwarding only for Dom0 */ + if (dest_qpt == IB_QPT_SMI && (mlx4_master_func_num(dev->dev) != slave)) + return -EINVAL; + + if (dest_qpt == IB_QPT_SMI) { + src_qpnum = 0; + sqp = &sqp_ctx->qp[0]; + wire_pkey_ix = dev->pkeys.virt2phys_pkey[slave][port - 1][0]; + } else { + src_qpnum = 1; + sqp = &sqp_ctx->qp[1]; + wire_pkey_ix = dev->pkeys.virt2phys_pkey[slave][port - 1][pkey_index]; + } + + send_qp = sqp->qp; + + /* create ah */ + sgid_index = attr->grh.sgid_index; + attr->grh.sgid_index = 0; + ah = ib_create_ah(sqp_ctx->pd, attr); + if (IS_ERR(ah)) + return -ENOMEM; + attr->grh.sgid_index = sgid_index; + to_mah(ah)->av.ib.gid_index = sgid_index; + /* get rid of force-loopback bit */ + to_mah(ah)->av.ib.port_pd &= cpu_to_be32(0x7FFFFFFF); + spin_lock(&sqp->tx_lock); + if (sqp->tx_ix_head - sqp->tx_ix_tail >= + (MLX4_NUM_TUNNEL_BUFS - 1)) + ret = -EAGAIN; + else + wire_tx_ix = (++sqp->tx_ix_head) & (MLX4_NUM_TUNNEL_BUFS - 1); + spin_unlock(&sqp->tx_lock); + if (ret) + goto out; + + sqp_mad = (struct mlx4_mad_snd_buf *) (sqp->tx_ring[wire_tx_ix].buf.addr); + if (sqp->tx_ring[wire_tx_ix].ah) + ib_destroy_ah(sqp->tx_ring[wire_tx_ix].ah); + sqp->tx_ring[wire_tx_ix].ah = ah; + ib_dma_sync_single_for_cpu(&dev->ib_dev, + sqp->tx_ring[wire_tx_ix].buf.map, + sizeof (struct mlx4_mad_snd_buf), + DMA_TO_DEVICE); + + memcpy(&sqp_mad->payload, mad, sizeof *mad); + + ib_dma_sync_single_for_device(&dev->ib_dev, + sqp->tx_ring[wire_tx_ix].buf.map, + sizeof (struct mlx4_mad_snd_buf), + DMA_TO_DEVICE); + + list.addr = sqp->tx_ring[wire_tx_ix].buf.map; + list.length = sizeof (struct mlx4_mad_snd_buf); + list.lkey = sqp_ctx->mr->lkey; + + wr.wr.ud.ah = ah; + wr.wr.ud.port_num = port; + wr.wr.ud.pkey_index = wire_pkey_ix; + wr.wr.ud.remote_qkey = qkey; + wr.wr.ud.remote_qpn = remote_qpn; + wr.next = NULL; + wr.wr_id = ((u64) wire_tx_ix) | MLX4_TUN_SET_WRID_QPN(src_qpnum); + wr.sg_list = &list; + wr.num_sge = 1; + wr.opcode = IB_WR_SEND; + wr.send_flags = IB_SEND_SIGNALED; + if (s_mac) + memcpy(to_mah(ah)->av.eth.s_mac, s_mac, 6); + + + ret = ib_post_send(send_qp, &wr, &bad_wr); +out: + if (ret) + ib_destroy_ah(ah); + return ret; +} + +static int get_slave_base_gid_ix(struct mlx4_ib_dev *dev, int slave, int port) +{ + int gids; + int vfs; + + if (rdma_port_get_link_layer(&dev->ib_dev, port) == IB_LINK_LAYER_INFINIBAND) + return slave; + + gids = MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS; + vfs = dev->dev->persist->num_vfs; + + if (slave == 0) + return 0; + if (slave <= gids % vfs) + return MLX4_ROCE_PF_GIDS + ((gids / vfs) + 1) * (slave - 1); + + return MLX4_ROCE_PF_GIDS + (gids % vfs) + ((gids / vfs) * (slave - 1)); +} + +static int get_real_sgid_index(struct mlx4_ib_dev *dev, int slave, int port, + struct ib_ah_attr *ah_attr) +{ + if (rdma_port_get_link_layer(&dev->ib_dev, port) == IB_LINK_LAYER_INFINIBAND) { + ah_attr->grh.sgid_index = slave; + return 0; + } + ah_attr->grh.sgid_index += get_slave_base_gid_ix(dev, slave, port); + return 0; +} + +static void mlx4_ib_multiplex_mad(struct mlx4_ib_demux_pv_ctx *ctx, struct ib_wc *wc) +{ + struct mlx4_ib_dev *dev = to_mdev(ctx->ib_dev); + struct mlx4_ib_demux_pv_qp *tun_qp = &ctx->qp[MLX4_TUN_WRID_QPN(wc->wr_id)]; + int wr_ix = wc->wr_id & (MLX4_NUM_TUNNEL_BUFS - 1); + struct mlx4_tunnel_mad *tunnel = tun_qp->ring[wr_ix].addr; + struct mlx4_ib_ah ah; + struct ib_ah_attr ah_attr; + u8 *slave_id; + int slave; + + /* Get slave that sent this packet */ + if (wc->src_qp < dev->dev->phys_caps.base_proxy_sqpn || + wc->src_qp >= dev->dev->phys_caps.base_proxy_sqpn + 8 * MLX4_MFUNC_MAX || + (wc->src_qp & 0x1) != ctx->port - 1 || + wc->src_qp & 0x4) { + mlx4_ib_warn(ctx->ib_dev, "can't multiplex bad sqp:%d\n", wc->src_qp); + return; + } + slave = ((wc->src_qp & ~0x7) - dev->dev->phys_caps.base_proxy_sqpn) / 8; + if (slave != ctx->slave) { + mlx4_ib_warn(ctx->ib_dev, "can't multiplex bad sqp:%d: " + "belongs to another slave\n", wc->src_qp); + return; + } + if (slave != mlx4_master_func_num(dev->dev) && !(wc->src_qp & 0x2)) { + mlx4_ib_warn(ctx->ib_dev, "can't multiplex bad sqp:%d: " + "non-master trying to send QP0 packets\n", wc->src_qp); + return; + } + + /* Map transaction ID */ + ib_dma_sync_single_for_cpu(ctx->ib_dev, tun_qp->ring[wr_ix].map, + sizeof (struct mlx4_tunnel_mad), + DMA_FROM_DEVICE); + switch (tunnel->mad.mad_hdr.method) { + case IB_MGMT_METHOD_SET: + case IB_MGMT_METHOD_GET: + case IB_MGMT_METHOD_REPORT: + case IB_SA_METHOD_GET_TABLE: + case IB_SA_METHOD_DELETE: + case IB_SA_METHOD_GET_MULTI: + case IB_SA_METHOD_GET_TRACE_TBL: + slave_id = (u8 *) &tunnel->mad.mad_hdr.tid; + if (*slave_id) { + mlx4_ib_warn(ctx->ib_dev, "egress mad has non-null tid msb:%d " + "class:%d slave:%d\n", *slave_id, + tunnel->mad.mad_hdr.mgmt_class, slave); + return; + } else + *slave_id = slave; + default: + /* nothing */; + } + + /* Class-specific handling */ + switch (tunnel->mad.mad_hdr.mgmt_class) { + case IB_MGMT_CLASS_SUBN_ADM: + if (mlx4_ib_multiplex_sa_handler(ctx->ib_dev, ctx->port, slave, + (struct ib_sa_mad *) &tunnel->mad)) + return; + break; + case IB_MGMT_CLASS_CM: + if (mlx4_ib_multiplex_cm_handler(ctx->ib_dev, ctx->port, slave, + (struct ib_mad *) &tunnel->mad)) + return; + break; + case IB_MGMT_CLASS_DEVICE_MGMT: + if (tunnel->mad.mad_hdr.method != IB_MGMT_METHOD_GET && + tunnel->mad.mad_hdr.method != IB_MGMT_METHOD_SET) + return; + break; + default: + /* Drop unsupported classes for slaves in tunnel mode */ + if (slave != mlx4_master_func_num(dev->dev)) { + mlx4_ib_warn(ctx->ib_dev, "dropping unsupported egress mad from class:%d " + "for slave:%d\n", tunnel->mad.mad_hdr.mgmt_class, slave); + return; + } + } + + /* We are using standard ib_core services to send the mad, so generate a + * stadard address handle by decoding the tunnelled mlx4_ah fields */ + memcpy(&ah.av, &tunnel->hdr.av, sizeof (struct mlx4_av)); + ah.ibah.device = ctx->ib_dev; + mlx4_ib_query_ah(&ah.ibah, &ah_attr); + if (ah_attr.ah_flags & IB_AH_GRH) + if (get_real_sgid_index(dev, slave, ctx->port, &ah_attr)) + return; + memcpy(ah_attr.dmac, tunnel->hdr.mac, 6); + ah_attr.vlan_id = tunnel->hdr.vlan; + /* if slave have default vlan use it */ + mlx4_get_slave_default_vlan(dev->dev, ctx->port, slave, + &ah_attr.vlan_id, &ah_attr.sl); + + mlx4_ib_send_to_wire(dev, slave, ctx->port, + is_proxy_qp0(dev, wc->src_qp, slave) ? + IB_QPT_SMI : IB_QPT_GSI, + be16_to_cpu(tunnel->hdr.pkey_index), + be32_to_cpu(tunnel->hdr.remote_qpn), + be32_to_cpu(tunnel->hdr.qkey), + &ah_attr, wc->smac, &tunnel->mad); +} + +static int mlx4_ib_alloc_pv_bufs(struct mlx4_ib_demux_pv_ctx *ctx, + enum ib_qp_type qp_type, int is_tun) +{ + int i; + struct mlx4_ib_demux_pv_qp *tun_qp; + int rx_buf_size, tx_buf_size; + + if (qp_type > IB_QPT_GSI) + return -EINVAL; + + tun_qp = &ctx->qp[qp_type]; + + tun_qp->ring = kzalloc(sizeof (struct mlx4_ib_buf) * MLX4_NUM_TUNNEL_BUFS, + GFP_KERNEL); + if (!tun_qp->ring) + return -ENOMEM; + + tun_qp->tx_ring = kcalloc(MLX4_NUM_TUNNEL_BUFS, + sizeof (struct mlx4_ib_tun_tx_buf), + GFP_KERNEL); + if (!tun_qp->tx_ring) { + kfree(tun_qp->ring); + tun_qp->ring = NULL; + return -ENOMEM; + } + + if (is_tun) { + rx_buf_size = sizeof (struct mlx4_tunnel_mad); + tx_buf_size = sizeof (struct mlx4_rcv_tunnel_mad); + } else { + rx_buf_size = sizeof (struct mlx4_mad_rcv_buf); + tx_buf_size = sizeof (struct mlx4_mad_snd_buf); + } + + for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { + tun_qp->ring[i].addr = kmalloc(rx_buf_size, GFP_KERNEL); + if (!tun_qp->ring[i].addr) + goto err; + tun_qp->ring[i].map = ib_dma_map_single(ctx->ib_dev, + tun_qp->ring[i].addr, + rx_buf_size, + DMA_FROM_DEVICE); + if (unlikely(ib_dma_mapping_error(ctx->ib_dev, + tun_qp->ring[i].map))) { + mlx4_ib_warn(ctx->ib_dev, "ib_dma_map_single failed\n"); + kfree(tun_qp->ring[i].addr); + goto err; + } + } + + for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { + tun_qp->tx_ring[i].buf.addr = + kmalloc(tx_buf_size, GFP_KERNEL); + if (!tun_qp->tx_ring[i].buf.addr) + goto tx_err; + tun_qp->tx_ring[i].buf.map = + ib_dma_map_single(ctx->ib_dev, + tun_qp->tx_ring[i].buf.addr, + tx_buf_size, + DMA_TO_DEVICE); + if (unlikely(ib_dma_mapping_error(ctx->ib_dev, + tun_qp->tx_ring[i].buf.map))) { + mlx4_ib_warn(ctx->ib_dev, "ib_dma_map_single failed\n"); + kfree(tun_qp->tx_ring[i].buf.addr); + goto tx_err; + } + tun_qp->tx_ring[i].ah = NULL; + } + spin_lock_init(&tun_qp->tx_lock); + tun_qp->tx_ix_head = 0; + tun_qp->tx_ix_tail = 0; + tun_qp->proxy_qpt = qp_type; + + return 0; + +tx_err: + while (i > 0) { + --i; + ib_dma_unmap_single(ctx->ib_dev, tun_qp->tx_ring[i].buf.map, + tx_buf_size, DMA_TO_DEVICE); + kfree(tun_qp->tx_ring[i].buf.addr); + } + kfree(tun_qp->tx_ring); + tun_qp->tx_ring = NULL; + i = MLX4_NUM_TUNNEL_BUFS; +err: + while (i > 0) { + --i; + ib_dma_unmap_single(ctx->ib_dev, tun_qp->ring[i].map, + rx_buf_size, DMA_FROM_DEVICE); + kfree(tun_qp->ring[i].addr); + } + kfree(tun_qp->ring); + tun_qp->ring = NULL; + return -ENOMEM; +} + +static void mlx4_ib_free_pv_qp_bufs(struct mlx4_ib_demux_pv_ctx *ctx, + enum ib_qp_type qp_type, int is_tun) +{ + int i; + struct mlx4_ib_demux_pv_qp *tun_qp; + int rx_buf_size, tx_buf_size; + + if (qp_type > IB_QPT_GSI) + return; + + tun_qp = &ctx->qp[qp_type]; + if (is_tun) { + rx_buf_size = sizeof (struct mlx4_tunnel_mad); + tx_buf_size = sizeof (struct mlx4_rcv_tunnel_mad); + } else { + rx_buf_size = sizeof (struct mlx4_mad_rcv_buf); + tx_buf_size = sizeof (struct mlx4_mad_snd_buf); + } + + + for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { + ib_dma_unmap_single(ctx->ib_dev, tun_qp->ring[i].map, + rx_buf_size, DMA_FROM_DEVICE); + kfree(tun_qp->ring[i].addr); + } + + for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { + ib_dma_unmap_single(ctx->ib_dev, tun_qp->tx_ring[i].buf.map, + tx_buf_size, DMA_TO_DEVICE); + kfree(tun_qp->tx_ring[i].buf.addr); + if (tun_qp->tx_ring[i].ah) + ib_destroy_ah(tun_qp->tx_ring[i].ah); + } + kfree(tun_qp->tx_ring); + kfree(tun_qp->ring); +} + +static void mlx4_ib_tunnel_comp_worker(struct work_struct *work) +{ + struct mlx4_ib_demux_pv_ctx *ctx; + struct mlx4_ib_demux_pv_qp *tun_qp; + struct ib_wc wc; + int ret; + ctx = container_of(work, struct mlx4_ib_demux_pv_ctx, work); + ib_req_notify_cq(ctx->cq, IB_CQ_NEXT_COMP); + + while (ib_poll_cq(ctx->cq, 1, &wc) == 1) { + tun_qp = &ctx->qp[MLX4_TUN_WRID_QPN(wc.wr_id)]; + if (wc.status == IB_WC_SUCCESS) { + switch (wc.opcode) { + case IB_WC_RECV: + mlx4_ib_multiplex_mad(ctx, &wc); + ret = mlx4_ib_post_pv_qp_buf(ctx, tun_qp, + wc.wr_id & + (MLX4_NUM_TUNNEL_BUFS - 1)); + if (ret) + pr_err("Failed reposting tunnel " + "buf:%lld\n", (unsigned long long)wc.wr_id); + break; + case IB_WC_SEND: + pr_debug("received tunnel send completion:" + "wrid=0x%llx, status=0x%x\n", + (unsigned long long)wc.wr_id, wc.status); + ib_destroy_ah(tun_qp->tx_ring[wc.wr_id & + (MLX4_NUM_TUNNEL_BUFS - 1)].ah); + tun_qp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah + = NULL; + spin_lock(&tun_qp->tx_lock); + tun_qp->tx_ix_tail++; + spin_unlock(&tun_qp->tx_lock); + + break; + default: + break; + } + } else { + pr_debug("mlx4_ib: completion error in tunnel: %d." + " status = %d, wrid = 0x%llx\n", + ctx->slave, wc.status, (unsigned long long)wc.wr_id); + if (!MLX4_TUN_IS_RECV(wc.wr_id)) { + ib_destroy_ah(tun_qp->tx_ring[wc.wr_id & + (MLX4_NUM_TUNNEL_BUFS - 1)].ah); + tun_qp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah + = NULL; + spin_lock(&tun_qp->tx_lock); + tun_qp->tx_ix_tail++; + spin_unlock(&tun_qp->tx_lock); + } + } + } +} + +static void pv_qp_event_handler(struct ib_event *event, void *qp_context) +{ + struct mlx4_ib_demux_pv_ctx *sqp = qp_context; + + /* It's worse than that! He's dead, Jim! */ + pr_err("Fatal error (%d) on a MAD QP on port %d\n", + event->event, sqp->port); +} + +static int create_pv_sqp(struct mlx4_ib_demux_pv_ctx *ctx, + enum ib_qp_type qp_type, int create_tun) +{ + int i, ret; + struct mlx4_ib_demux_pv_qp *tun_qp; + struct mlx4_ib_qp_tunnel_init_attr qp_init_attr; + struct ib_qp_attr attr; + int qp_attr_mask_INIT; + + if (qp_type > IB_QPT_GSI) + return -EINVAL; + + tun_qp = &ctx->qp[qp_type]; + + memset(&qp_init_attr, 0, sizeof qp_init_attr); + qp_init_attr.init_attr.send_cq = ctx->cq; + qp_init_attr.init_attr.recv_cq = ctx->cq; + qp_init_attr.init_attr.sq_sig_type = IB_SIGNAL_ALL_WR; + qp_init_attr.init_attr.cap.max_send_wr = MLX4_NUM_TUNNEL_BUFS; + qp_init_attr.init_attr.cap.max_recv_wr = MLX4_NUM_TUNNEL_BUFS; + qp_init_attr.init_attr.cap.max_send_sge = 1; + qp_init_attr.init_attr.cap.max_recv_sge = 1; + if (create_tun) { + qp_init_attr.init_attr.qp_type = IB_QPT_UD; + qp_init_attr.init_attr.create_flags = (enum ib_qp_create_flags)MLX4_IB_SRIOV_TUNNEL_QP; + qp_init_attr.port = ctx->port; + qp_init_attr.slave = ctx->slave; + qp_init_attr.proxy_qp_type = qp_type; + qp_attr_mask_INIT = IB_QP_STATE | IB_QP_PKEY_INDEX | + IB_QP_QKEY | IB_QP_PORT; + } else { + qp_init_attr.init_attr.qp_type = qp_type; + qp_init_attr.init_attr.create_flags = (enum ib_qp_create_flags)MLX4_IB_SRIOV_SQP; + qp_attr_mask_INIT = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_QKEY; + } + qp_init_attr.init_attr.port_num = ctx->port; + qp_init_attr.init_attr.qp_context = ctx; + qp_init_attr.init_attr.event_handler = pv_qp_event_handler; + tun_qp->qp = ib_create_qp(ctx->pd, &qp_init_attr.init_attr); + if (IS_ERR(tun_qp->qp)) { + ret = PTR_ERR(tun_qp->qp); + tun_qp->qp = NULL; + pr_err("Couldn't create %s QP (%d)\n", + create_tun ? "tunnel" : "special", ret); + return ret; + } + + memset(&attr, 0, sizeof attr); + attr.qp_state = IB_QPS_INIT; + ret = 0; + if (create_tun) + ret = find_slave_port_pkey_ix(to_mdev(ctx->ib_dev), ctx->slave, + ctx->port, 0xFFFF, &attr.pkey_index); + if (ret || !create_tun) + attr.pkey_index = + to_mdev(ctx->ib_dev)->pkeys.virt2phys_pkey[ctx->slave][ctx->port - 1][0]; + attr.qkey = IB_QP1_QKEY; + attr.port_num = ctx->port; + ret = ib_modify_qp(tun_qp->qp, &attr, qp_attr_mask_INIT); + if (ret) { + pr_err("Couldn't change %s qp state to INIT (%d)\n", + create_tun ? "tunnel" : "special", ret); + goto err_qp; + } + attr.qp_state = IB_QPS_RTR; + ret = ib_modify_qp(tun_qp->qp, &attr, IB_QP_STATE); + if (ret) { + pr_err("Couldn't change %s qp state to RTR (%d)\n", + create_tun ? "tunnel" : "special", ret); + goto err_qp; + } + attr.qp_state = IB_QPS_RTS; + attr.sq_psn = 0; + ret = ib_modify_qp(tun_qp->qp, &attr, IB_QP_STATE | IB_QP_SQ_PSN); + if (ret) { + pr_err("Couldn't change %s qp state to RTS (%d)\n", + create_tun ? "tunnel" : "special", ret); + goto err_qp; + } + + for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { + ret = mlx4_ib_post_pv_qp_buf(ctx, tun_qp, i); + if (ret) { + pr_err(" mlx4_ib_post_pv_buf error" + " (err = %d, i = %d)\n", ret, i); + goto err_qp; + } + } + return 0; + +err_qp: + ib_destroy_qp(tun_qp->qp); + tun_qp->qp = NULL; + return ret; +} + +/* + * IB MAD completion callback for real SQPs + */ +static void mlx4_ib_sqp_comp_worker(struct work_struct *work) +{ + struct mlx4_ib_demux_pv_ctx *ctx; + struct mlx4_ib_demux_pv_qp *sqp; + struct ib_wc wc; + struct ib_grh *grh; + struct ib_mad *mad; + + ctx = container_of(work, struct mlx4_ib_demux_pv_ctx, work); + ib_req_notify_cq(ctx->cq, IB_CQ_NEXT_COMP); + + while (mlx4_ib_poll_cq(ctx->cq, 1, &wc) == 1) { + sqp = &ctx->qp[MLX4_TUN_WRID_QPN(wc.wr_id)]; + if (wc.status == IB_WC_SUCCESS) { + switch (wc.opcode) { + case IB_WC_SEND: + ib_destroy_ah(sqp->tx_ring[wc.wr_id & + (MLX4_NUM_TUNNEL_BUFS - 1)].ah); + sqp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah + = NULL; + spin_lock(&sqp->tx_lock); + sqp->tx_ix_tail++; + spin_unlock(&sqp->tx_lock); + break; + case IB_WC_RECV: + mad = (struct ib_mad *) &(((struct mlx4_mad_rcv_buf *) + (sqp->ring[wc.wr_id & + (MLX4_NUM_TUNNEL_BUFS - 1)].addr))->payload); + grh = &(((struct mlx4_mad_rcv_buf *) + (sqp->ring[wc.wr_id & + (MLX4_NUM_TUNNEL_BUFS - 1)].addr))->grh); + mlx4_ib_demux_mad(ctx->ib_dev, ctx->port, &wc, grh, mad); + if (mlx4_ib_post_pv_qp_buf(ctx, sqp, wc.wr_id & + (MLX4_NUM_TUNNEL_BUFS - 1))) + pr_err("Failed reposting SQP " + "buf:%lld\n", (unsigned long long)wc.wr_id); + break; + default: + BUG_ON(1); + break; + } + } else { + pr_debug("mlx4_ib: completion error in tunnel: %d." + " status = %d, wrid = 0x%llx\n", + ctx->slave, wc.status, (unsigned long long)wc.wr_id); + if (!MLX4_TUN_IS_RECV(wc.wr_id)) { + ib_destroy_ah(sqp->tx_ring[wc.wr_id & + (MLX4_NUM_TUNNEL_BUFS - 1)].ah); + sqp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah + = NULL; + spin_lock(&sqp->tx_lock); + sqp->tx_ix_tail++; + spin_unlock(&sqp->tx_lock); + } + } + } +} + +static int alloc_pv_object(struct mlx4_ib_dev *dev, int slave, int port, + struct mlx4_ib_demux_pv_ctx **ret_ctx) +{ + struct mlx4_ib_demux_pv_ctx *ctx; + + *ret_ctx = NULL; + ctx = kzalloc(sizeof (struct mlx4_ib_demux_pv_ctx), GFP_KERNEL); + if (!ctx) { + pr_err("failed allocating pv resource context " + "for port %d, slave %d\n", port, slave); + return -ENOMEM; + } + + ctx->ib_dev = &dev->ib_dev; + ctx->port = port; + ctx->slave = slave; + *ret_ctx = ctx; + return 0; +} + +static void free_pv_object(struct mlx4_ib_dev *dev, int slave, int port) +{ + if (dev->sriov.demux[port - 1].tun[slave]) { + kfree(dev->sriov.demux[port - 1].tun[slave]); + dev->sriov.demux[port - 1].tun[slave] = NULL; + } +} + +static int create_pv_resources(struct ib_device *ibdev, int slave, int port, + int create_tun, struct mlx4_ib_demux_pv_ctx *ctx) +{ + int ret, cq_size; + + if (ctx->state != DEMUX_PV_STATE_DOWN) + return -EEXIST; + + ctx->state = DEMUX_PV_STATE_STARTING; + /* have QP0 only on port owner, and only if link layer is IB */ + if (ctx->slave == mlx4_master_func_num(to_mdev(ctx->ib_dev)->dev) && + rdma_port_get_link_layer(ibdev, ctx->port) == IB_LINK_LAYER_INFINIBAND) + ctx->has_smi = 1; + + if (ctx->has_smi) { + ret = mlx4_ib_alloc_pv_bufs(ctx, IB_QPT_SMI, create_tun); + if (ret) { + pr_err("Failed allocating qp0 tunnel bufs (%d)\n", ret); + goto err_out; + } + } + + ret = mlx4_ib_alloc_pv_bufs(ctx, IB_QPT_GSI, create_tun); + if (ret) { + pr_err("Failed allocating qp1 tunnel bufs (%d)\n", ret); + goto err_out_qp0; + } + + cq_size = 2 * MLX4_NUM_TUNNEL_BUFS; + if (ctx->has_smi) + cq_size *= 2; + + ctx->cq = ib_create_cq(ctx->ib_dev, mlx4_ib_tunnel_comp_handler, + NULL, ctx, cq_size, 0); + if (IS_ERR(ctx->cq)) { + ret = PTR_ERR(ctx->cq); + pr_err("Couldn't create tunnel CQ (%d)\n", ret); + goto err_buf; + } + + ctx->pd = ib_alloc_pd(ctx->ib_dev); + if (IS_ERR(ctx->pd)) { + ret = PTR_ERR(ctx->pd); + pr_err("Couldn't create tunnel PD (%d)\n", ret); + goto err_cq; + } + + ctx->mr = ib_get_dma_mr(ctx->pd, IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(ctx->mr)) { + ret = PTR_ERR(ctx->mr); + pr_err("Couldn't get tunnel DMA MR (%d)\n", ret); + goto err_pd; + } + + if (ctx->has_smi) { + ret = create_pv_sqp(ctx, IB_QPT_SMI, create_tun); + if (ret) { + pr_err("Couldn't create %s QP0 (%d)\n", + create_tun ? "tunnel for" : "", ret); + goto err_mr; + } + } + + ret = create_pv_sqp(ctx, IB_QPT_GSI, create_tun); + if (ret) { + pr_err("Couldn't create %s QP1 (%d)\n", + create_tun ? "tunnel for" : "", ret); + goto err_qp0; + } + + if (create_tun) + INIT_WORK(&ctx->work, mlx4_ib_tunnel_comp_worker); + else + INIT_WORK(&ctx->work, mlx4_ib_sqp_comp_worker); + + ctx->wq = to_mdev(ibdev)->sriov.demux[port - 1].wq; + + ret = ib_req_notify_cq(ctx->cq, IB_CQ_NEXT_COMP); + if (ret) { + pr_err("Couldn't arm tunnel cq (%d)\n", ret); + goto err_wq; + } + ctx->state = DEMUX_PV_STATE_ACTIVE; + return 0; + +err_wq: + ctx->wq = NULL; + ib_destroy_qp(ctx->qp[1].qp); + ctx->qp[1].qp = NULL; + + +err_qp0: + if (ctx->has_smi) + ib_destroy_qp(ctx->qp[0].qp); + ctx->qp[0].qp = NULL; + +err_mr: + ib_dereg_mr(ctx->mr); + ctx->mr = NULL; + +err_pd: + ib_dealloc_pd(ctx->pd); + ctx->pd = NULL; + +err_cq: + ib_destroy_cq(ctx->cq); + ctx->cq = NULL; + +err_buf: + mlx4_ib_free_pv_qp_bufs(ctx, IB_QPT_GSI, create_tun); + +err_out_qp0: + if (ctx->has_smi) + mlx4_ib_free_pv_qp_bufs(ctx, IB_QPT_SMI, create_tun); +err_out: + ctx->state = DEMUX_PV_STATE_DOWN; + return ret; +} + +static void destroy_pv_resources(struct mlx4_ib_dev *dev, int slave, int port, + struct mlx4_ib_demux_pv_ctx *ctx, int flush) +{ + if (!ctx) + return; + if (ctx->state > DEMUX_PV_STATE_DOWN) { + ctx->state = DEMUX_PV_STATE_DOWNING; + if (flush) + flush_workqueue(ctx->wq); + if (ctx->has_smi) { + ib_destroy_qp(ctx->qp[0].qp); + ctx->qp[0].qp = NULL; + mlx4_ib_free_pv_qp_bufs(ctx, IB_QPT_SMI, 1); + } + ib_destroy_qp(ctx->qp[1].qp); + ctx->qp[1].qp = NULL; + mlx4_ib_free_pv_qp_bufs(ctx, IB_QPT_GSI, 1); + ib_dereg_mr(ctx->mr); + ctx->mr = NULL; + ib_dealloc_pd(ctx->pd); + ctx->pd = NULL; + ib_destroy_cq(ctx->cq); + ctx->cq = NULL; + ctx->state = DEMUX_PV_STATE_DOWN; + } +} + +static int mlx4_ib_tunnels_update(struct mlx4_ib_dev *dev, int slave, + int port, int do_init) +{ + int ret = 0; + + if (!do_init) { + clean_vf_mcast(&dev->sriov.demux[port - 1], slave); + /* for master, destroy real sqp resources */ + if (slave == mlx4_master_func_num(dev->dev)) + destroy_pv_resources(dev, slave, port, + dev->sriov.sqps[port - 1], 1); + /* destroy the tunnel qp resources */ + destroy_pv_resources(dev, slave, port, + dev->sriov.demux[port - 1].tun[slave], 1); + return 0; + } + + /* create the tunnel qp resources */ + ret = create_pv_resources(&dev->ib_dev, slave, port, 1, + dev->sriov.demux[port - 1].tun[slave]); + + /* for master, create the real sqp resources */ + if (!ret && slave == mlx4_master_func_num(dev->dev)) + ret = create_pv_resources(&dev->ib_dev, slave, port, 0, + dev->sriov.sqps[port - 1]); + return ret; +} + +void mlx4_ib_tunnels_update_work(struct work_struct *work) +{ + struct mlx4_ib_demux_work *dmxw; + + dmxw = container_of(work, struct mlx4_ib_demux_work, work); + mlx4_ib_tunnels_update(dmxw->dev, dmxw->slave, (int) dmxw->port, + dmxw->do_init); + kfree(dmxw); + return; +} + +static int mlx4_ib_alloc_demux_ctx(struct mlx4_ib_dev *dev, + struct mlx4_ib_demux_ctx *ctx, + int port) +{ + char name[12]; + int ret = 0; + int i; + + ctx->tun = kcalloc(dev->dev->caps.sqp_demux, + sizeof (struct mlx4_ib_demux_pv_ctx *), GFP_KERNEL); + if (!ctx->tun) + return -ENOMEM; + + ctx->dev = dev; + ctx->port = port; + ctx->ib_dev = &dev->ib_dev; + + for (i = 0; i < dev->dev->caps.sqp_demux; i++) { + ret = alloc_pv_object(dev, i, port, &ctx->tun[i]); + if (ret) { + ret = -ENOMEM; + goto err_mcg; + } + } + + ret = mlx4_ib_mcg_port_init(ctx); + if (ret) { + pr_err("Failed initializing mcg para-virt (%d)\n", ret); + goto err_mcg; + } + + snprintf(name, sizeof name, "mlx4_ibt%d", port); + ctx->wq = create_singlethread_workqueue(name); + if (!ctx->wq) { + pr_err("Failed to create tunnelling WQ for port %d\n", port); + ret = -ENOMEM; + goto err_wq; + } + + snprintf(name, sizeof name, "mlx4_ibud%d", port); + ctx->ud_wq = create_singlethread_workqueue(name); + if (!ctx->ud_wq) { + pr_err("Failed to create up/down WQ for port %d\n", port); + ret = -ENOMEM; + goto err_udwq; + } + + return 0; + +err_udwq: + destroy_workqueue(ctx->wq); + ctx->wq = NULL; + +err_wq: + mlx4_ib_mcg_port_cleanup(ctx, 1); +err_mcg: + for (i = 0; i < dev->dev->caps.sqp_demux; i++) + free_pv_object(dev, i, port); + kfree(ctx->tun); + ctx->tun = NULL; + return ret; +} + +static void mlx4_ib_free_sqp_ctx(struct mlx4_ib_demux_pv_ctx *sqp_ctx) +{ + if (sqp_ctx->state > DEMUX_PV_STATE_DOWN) { + sqp_ctx->state = DEMUX_PV_STATE_DOWNING; + flush_workqueue(sqp_ctx->wq); + if (sqp_ctx->has_smi) { + ib_destroy_qp(sqp_ctx->qp[0].qp); + sqp_ctx->qp[0].qp = NULL; + mlx4_ib_free_pv_qp_bufs(sqp_ctx, IB_QPT_SMI, 0); + } + ib_destroy_qp(sqp_ctx->qp[1].qp); + sqp_ctx->qp[1].qp = NULL; + mlx4_ib_free_pv_qp_bufs(sqp_ctx, IB_QPT_GSI, 0); + ib_dereg_mr(sqp_ctx->mr); + sqp_ctx->mr = NULL; + ib_dealloc_pd(sqp_ctx->pd); + sqp_ctx->pd = NULL; + ib_destroy_cq(sqp_ctx->cq); + sqp_ctx->cq = NULL; + sqp_ctx->state = DEMUX_PV_STATE_DOWN; + } +} + +static void mlx4_ib_free_demux_ctx(struct mlx4_ib_demux_ctx *ctx) +{ + int i; + if (ctx) { + struct mlx4_ib_dev *dev = to_mdev(ctx->ib_dev); + mlx4_ib_mcg_port_cleanup(ctx, 1); + for (i = 0; i < dev->dev->caps.sqp_demux; i++) { + if (!ctx->tun[i]) + continue; + if (ctx->tun[i]->state > DEMUX_PV_STATE_DOWN) + ctx->tun[i]->state = DEMUX_PV_STATE_DOWNING; + } + flush_workqueue(ctx->wq); + for (i = 0; i < dev->dev->caps.sqp_demux; i++) { + destroy_pv_resources(dev, i, ctx->port, ctx->tun[i], 0); + free_pv_object(dev, i, ctx->port); + } + kfree(ctx->tun); + destroy_workqueue(ctx->ud_wq); + destroy_workqueue(ctx->wq); + } +} + +static void mlx4_ib_master_tunnels(struct mlx4_ib_dev *dev, int do_init) +{ + int i; + + if (!mlx4_is_master(dev->dev)) + return; + /* initialize or tear down tunnel QPs for the master */ + for (i = 0; i < dev->dev->caps.num_ports; i++) + mlx4_ib_tunnels_update(dev, mlx4_master_func_num(dev->dev), i + 1, do_init); + return; +} + +int mlx4_ib_init_sriov(struct mlx4_ib_dev *dev) +{ + int i = 0; + int err; + + if (!mlx4_is_mfunc(dev->dev)) + return 0; + + dev->sriov.is_going_down = 0; + spin_lock_init(&dev->sriov.going_down_lock); + mlx4_ib_cm_paravirt_init(dev); + + mlx4_ib_warn(&dev->ib_dev, "multi-function enabled\n"); + + if (mlx4_is_slave(dev->dev)) { + mlx4_ib_warn(&dev->ib_dev, "operating in qp1 tunnel mode\n"); + return 0; + } + + for (i = 0; i < dev->dev->caps.sqp_demux; i++) { + if (i == mlx4_master_func_num(dev->dev)) + mlx4_put_slave_node_guid(dev->dev, i, dev->ib_dev.node_guid); + else + mlx4_put_slave_node_guid(dev->dev, i, mlx4_ib_gen_node_guid()); + } + + err = mlx4_ib_init_alias_guid_service(dev); + if (err) { + mlx4_ib_warn(&dev->ib_dev, "Failed init alias guid process.\n"); + goto paravirt_err; + } + err = mlx4_ib_device_register_sysfs(dev); + if (err) { + mlx4_ib_warn(&dev->ib_dev, "Failed to register sysfs\n"); + goto sysfs_err; + } + + mlx4_ib_warn(&dev->ib_dev, "initializing demux service for %d qp1 clients\n", + dev->dev->caps.sqp_demux); + for (i = 0; i < dev->num_ports; i++) { + union ib_gid gid; + err = __mlx4_ib_query_gid(&dev->ib_dev, i + 1, 0, &gid, 1); + if (err) + goto demux_err; + dev->sriov.demux[i].guid_cache[0] = gid.global.interface_id; + err = alloc_pv_object(dev, mlx4_master_func_num(dev->dev), i + 1, + &dev->sriov.sqps[i]); + if (err) + goto demux_err; + err = mlx4_ib_alloc_demux_ctx(dev, &dev->sriov.demux[i], i + 1); + if (err) + goto demux_err; + } + mlx4_ib_master_tunnels(dev, 1); + return 0; + +demux_err: + while (i > 0) { + free_pv_object(dev, mlx4_master_func_num(dev->dev), i + 1); + mlx4_ib_free_demux_ctx(&dev->sriov.demux[i]); + --i; + } + mlx4_ib_device_unregister_sysfs(dev); + +sysfs_err: + mlx4_ib_destroy_alias_guid_service(dev); + +paravirt_err: + mlx4_ib_cm_paravirt_clean(dev, -1); + + return err; +} + +void mlx4_ib_close_sriov(struct mlx4_ib_dev *dev) +{ + int i; + unsigned long flags; + + if (!mlx4_is_mfunc(dev->dev)) + return; + + spin_lock_irqsave(&dev->sriov.going_down_lock, flags); + dev->sriov.is_going_down = 1; + spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); + if (mlx4_is_master(dev->dev)) { + for (i = 0; i < dev->num_ports; i++) { + flush_workqueue(dev->sriov.demux[i].ud_wq); + mlx4_ib_free_sqp_ctx(dev->sriov.sqps[i]); + kfree(dev->sriov.sqps[i]); + dev->sriov.sqps[i] = NULL; + mlx4_ib_free_demux_ctx(&dev->sriov.demux[i]); + } + + mlx4_ib_cm_paravirt_clean(dev, -1); + mlx4_ib_destroy_alias_guid_service(dev); + mlx4_ib_device_unregister_sysfs(dev); + } +} Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_mad.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_main.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_main.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_main.c (revision 329159) @@ -0,0 +1,2785 @@ +/* + * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#define LINUXKPI_PARAM_PREFIX mlx4_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include "mlx4_ib.h" +#include "mlx4_exp.h" +#include "user.h" +#include "wc.h" + +#define DRV_NAME MLX4_IB_DRV_NAME +#define DRV_VERSION "1.0" + +#define MLX4_IB_DRIVER_PROC_DIR_NAME "driver/mlx4_ib" +#define MLX4_IB_MRS_PROC_DIR_NAME "mrs" +#define MLX4_IB_FLOW_MAX_PRIO 0xFFF +#define MLX4_IB_FLOW_QPN_MASK 0xFFFFFF + +MODULE_AUTHOR("Roland Dreier"); +MODULE_DESCRIPTION("Mellanox ConnectX HCA InfiniBand driver"); +MODULE_LICENSE("Dual BSD/GPL"); +#ifdef __linux__ +MODULE_VERSION(DRV_VERSION); +#endif + +int mlx4_ib_sm_guid_assign = 1; + +module_param_named(sm_guid_assign, mlx4_ib_sm_guid_assign, int, 0444); +MODULE_PARM_DESC(sm_guid_assign, "Enable SM alias_GUID assignment if sm_guid_assign > 0 (Default: 1)"); + +static const char mlx4_ib_version[] = + DRV_NAME ": Mellanox ConnectX InfiniBand driver v" + DRV_VERSION "\n"; + +struct update_gid_work { + struct work_struct work; + union ib_gid gids[128]; + struct mlx4_ib_dev *dev; + int port; +}; + +struct dev_rec { + int bus; + int dev; + int func; + int nr; +}; + +static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init); + +static void mlx4_ib_scan_netdevs(struct mlx4_ib_dev *ibdev, struct net_device*, + unsigned long); + +static u8 mlx4_ib_get_dev_port(struct net_device *dev, + struct mlx4_ib_dev *ibdev); + +static struct workqueue_struct *wq; + +static void init_query_mad(struct ib_smp *mad) +{ + mad->base_version = 1; + mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; + mad->class_version = 1; + mad->method = IB_MGMT_METHOD_GET; +} + +static union ib_gid zgid; + +static int check_flow_steering_support(struct mlx4_dev *dev) +{ + int eth_num_ports = 0; + int ib_num_ports = 0; + int dmfs = dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED; + + if (dmfs) { + int i; + mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) + eth_num_ports++; + mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB) + ib_num_ports++; + dmfs &= (!ib_num_ports || + (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_DMFS_IPOIB)) && + (!eth_num_ports || + (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FS_EN)); + if (ib_num_ports && mlx4_is_mfunc(dev)) { + dmfs = 0; + } + } + return dmfs; +} + +int mlx4_ib_query_device(struct ib_device *ibdev, + struct ib_device_attr *props) +{ + struct mlx4_ib_dev *dev = to_mdev(ibdev); + struct ib_smp *in_mad = NULL; + struct ib_smp *out_mad = NULL; + int err = -ENOMEM; + struct mlx4_clock_params clock_params; + + in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); + out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); + if (!in_mad || !out_mad) + goto out; + + init_query_mad(in_mad); + in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; + + err = mlx4_MAD_IFC(to_mdev(ibdev), MLX4_MAD_IFC_IGNORE_KEYS, + 1, NULL, NULL, in_mad, out_mad); + if (err) + goto out; + + memset(props, 0, sizeof *props); + + props->fw_ver = dev->dev->caps.fw_ver; + props->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT | + IB_DEVICE_PORT_ACTIVE_EVENT | + IB_DEVICE_SYS_IMAGE_GUID | + IB_DEVICE_RC_RNR_NAK_GEN | + IB_DEVICE_BLOCK_MULTICAST_LOOPBACK | + IB_DEVICE_SHARED_MR; + + if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BAD_PKEY_CNTR) + props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR; + if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BAD_QKEY_CNTR) + props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR; + if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_APM) + props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG; + if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UD_AV_PORT) + props->device_cap_flags |= IB_DEVICE_UD_AV_PORT_ENFORCE; + if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_IPOIB_CSUM) + props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM; + if (dev->dev->caps.max_gso_sz && dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BLH) + props->device_cap_flags |= IB_DEVICE_UD_TSO; + if (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_RESERVED_LKEY) + props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY; + if ((dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_LOCAL_INV) && + (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_REMOTE_INV) && + (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_FAST_REG_WR)) + props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; + if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) + props->device_cap_flags |= IB_DEVICE_XRC; + + if (check_flow_steering_support(dev->dev)) + props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING; + + + props->device_cap_flags |= IB_DEVICE_QPG; + if (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS) { + props->device_cap_flags |= IB_DEVICE_UD_RSS; + props->max_rss_tbl_sz = dev->dev->caps.max_rss_tbl_sz; + } + if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW) + props->device_cap_flags |= IB_DEVICE_MEM_WINDOW; + if (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN) { + if (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_WIN_TYPE_2B) + props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2B; + else + props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2A; + } + props->vendor_id = be32_to_cpup((__be32 *) (out_mad->data + 36)) & + 0xffffff; + props->vendor_part_id = dev->dev->persist->pdev->device; + props->hw_ver = be32_to_cpup((__be32 *) (out_mad->data + 32)); + memcpy(&props->sys_image_guid, out_mad->data + 4, 8); + + props->max_mr_size = ~0ull; + props->page_size_cap = dev->dev->caps.page_size_cap; + props->max_qp = dev->dev->quotas.qp; + props->max_qp_wr = dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE; + props->max_sge = min(dev->dev->caps.max_sq_sg, + dev->dev->caps.max_rq_sg); + props->max_cq = dev->dev->quotas.cq; + props->max_cqe = dev->dev->caps.max_cqes; + props->max_mr = dev->dev->quotas.mpt; + props->max_pd = dev->dev->caps.num_pds - dev->dev->caps.reserved_pds; + props->max_qp_rd_atom = dev->dev->caps.max_qp_dest_rdma; + props->max_qp_init_rd_atom = dev->dev->caps.max_qp_init_rdma; + props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; + props->max_srq = dev->dev->quotas.srq; + props->max_srq_wr = dev->dev->caps.max_srq_wqes - 1; + props->max_srq_sge = dev->dev->caps.max_srq_sge; + props->max_fast_reg_page_list_len = MLX4_MAX_FAST_REG_PAGES; + props->local_ca_ack_delay = dev->dev->caps.local_ca_ack_delay; + props->atomic_cap = dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_ATOMIC ? + IB_ATOMIC_HCA : IB_ATOMIC_NONE; + props->masked_atomic_cap = props->atomic_cap; + props->max_pkeys = dev->dev->caps.pkey_table_len[1]; + props->max_mcast_grp = dev->dev->caps.num_mgms + dev->dev->caps.num_amgms; + props->max_mcast_qp_attach = dev->dev->caps.num_qp_per_mgm; + props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * + props->max_mcast_grp; + props->max_map_per_fmr = dev->dev->caps.max_fmr_maps; + props->hca_core_clock = dev->dev->caps.hca_core_clock; + + if (!mlx4_is_slave(dev->dev)) + err = mlx4_get_internal_clock_params(dev->dev, &clock_params); + if (dev->dev->caps.hca_core_clock > 0) + props->comp_mask |= IB_DEVICE_ATTR_WITH_HCA_CORE_CLOCK; + if (!err && !mlx4_is_slave(dev->dev)) { + props->timestamp_mask = 0xFFFFFFFFFFFF; + props->comp_mask |= IB_DEVICE_ATTR_WITH_TIMESTAMP_MASK; + } + +out: + kfree(in_mad); + kfree(out_mad); + + return err; +} + +static enum rdma_link_layer +mlx4_ib_port_link_layer(struct ib_device *device, u8 port_num) +{ + struct mlx4_dev *dev = to_mdev(device)->dev; + + return dev->caps.port_mask[port_num] == MLX4_PORT_TYPE_IB ? + IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET; +} + +static int ib_link_query_port(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props, int netw_view) +{ + struct ib_smp *in_mad = NULL; + struct ib_smp *out_mad = NULL; + int ext_active_speed; + int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS; + int err = -ENOMEM; + + in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); + out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); + if (!in_mad || !out_mad) + goto out; + + init_query_mad(in_mad); + in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; + in_mad->attr_mod = cpu_to_be32(port); + + if (mlx4_is_mfunc(to_mdev(ibdev)->dev) && netw_view) + mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW; + + err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, NULL, NULL, + in_mad, out_mad); + if (err) + goto out; + + + props->lid = be16_to_cpup((__be16 *) (out_mad->data + 16)); + props->lmc = out_mad->data[34] & 0x7; + props->sm_lid = be16_to_cpup((__be16 *) (out_mad->data + 18)); + props->sm_sl = out_mad->data[36] & 0xf; + props->state = out_mad->data[32] & 0xf; + props->phys_state = out_mad->data[33] >> 4; + props->port_cap_flags = be32_to_cpup((__be32 *) (out_mad->data + 20)); + if (netw_view) + props->gid_tbl_len = out_mad->data[50]; + else + props->gid_tbl_len = to_mdev(ibdev)->dev->caps.gid_table_len[port]; + props->max_msg_sz = to_mdev(ibdev)->dev->caps.max_msg_sz; + props->pkey_tbl_len = to_mdev(ibdev)->dev->caps.pkey_table_len[port]; + props->bad_pkey_cntr = be16_to_cpup((__be16 *) (out_mad->data + 46)); + props->qkey_viol_cntr = be16_to_cpup((__be16 *) (out_mad->data + 48)); + props->active_width = out_mad->data[31] & 0xf; + props->active_speed = out_mad->data[35] >> 4; + props->max_mtu = out_mad->data[41] & 0xf; + props->active_mtu = out_mad->data[36] >> 4; + props->subnet_timeout = out_mad->data[51] & 0x1f; + props->max_vl_num = out_mad->data[37] >> 4; + props->init_type_reply = out_mad->data[41] >> 4; + + /* Check if extended speeds (EDR/FDR/...) are supported */ + if (props->port_cap_flags & IB_PORT_EXTENDED_SPEEDS_SUP) { + ext_active_speed = out_mad->data[62] >> 4; + + switch (ext_active_speed) { + case 1: + props->active_speed = IB_SPEED_FDR; + break; + case 2: + props->active_speed = IB_SPEED_EDR; + break; + } + } + + /* If reported active speed is QDR, check if is FDR-10 */ + if (props->active_speed == IB_SPEED_QDR) { + init_query_mad(in_mad); + in_mad->attr_id = MLX4_ATTR_EXTENDED_PORT_INFO; + in_mad->attr_mod = cpu_to_be32(port); + + err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, + NULL, NULL, in_mad, out_mad); + if (err) + goto out; + + /* Checking LinkSpeedActive for FDR-10 */ + if (out_mad->data[15] & 0x1) + props->active_speed = IB_SPEED_FDR10; + } + + /* Avoid wrong speed value returned by FW if the IB link is down. */ + if (props->state == IB_PORT_DOWN) + props->active_speed = IB_SPEED_SDR; + +out: + kfree(in_mad); + kfree(out_mad); + return err; +} + +static u8 state_to_phys_state(enum ib_port_state state) +{ + return state == IB_PORT_ACTIVE ? 5 : 3; +} + +static int eth_link_query_port(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props, int netw_view) +{ + + struct mlx4_ib_dev *mdev = to_mdev(ibdev); + struct mlx4_ib_iboe *iboe = &mdev->iboe; + struct net_device *ndev; + enum ib_mtu tmp; + struct mlx4_cmd_mailbox *mailbox; + unsigned long flags; + int err = 0; + + mailbox = mlx4_alloc_cmd_mailbox(mdev->dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + err = mlx4_cmd_box(mdev->dev, 0, mailbox->dma, port, 0, + MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); + if (err) + goto out; + + props->active_width = (((u8 *)mailbox->buf)[5] == 0x40) ? + IB_WIDTH_4X : IB_WIDTH_1X; + props->active_speed = IB_SPEED_QDR; + props->port_cap_flags = IB_PORT_CM_SUP; + if (netw_view) + props->gid_tbl_len = MLX4_ROCE_MAX_GIDS; + else + props->gid_tbl_len = mdev->dev->caps.gid_table_len[port]; + + props->max_msg_sz = mdev->dev->caps.max_msg_sz; + props->pkey_tbl_len = 1; + props->max_mtu = IB_MTU_4096; + props->max_vl_num = 2; + props->state = IB_PORT_DOWN; + props->phys_state = state_to_phys_state(props->state); + props->active_mtu = IB_MTU_256; + spin_lock_irqsave(&iboe->lock, flags); + ndev = iboe->netdevs[port - 1]; + if (!ndev) + goto out_unlock; + + tmp = iboe_get_mtu(ndev->if_mtu); + props->active_mtu = tmp ? min(props->max_mtu, tmp) : IB_MTU_256; + + props->state = (netif_running(ndev) && netif_carrier_ok(ndev)) ? + IB_PORT_ACTIVE : IB_PORT_DOWN; + props->phys_state = state_to_phys_state(props->state); +out_unlock: + spin_unlock_irqrestore(&iboe->lock, flags); +out: + mlx4_free_cmd_mailbox(mdev->dev, mailbox); + return err; +} + +int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props, int netw_view) +{ + int err; + + memset(props, 0, sizeof *props); + + err = mlx4_ib_port_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND ? + ib_link_query_port(ibdev, port, props, netw_view) : + eth_link_query_port(ibdev, port, props, netw_view); + + return err; +} + +static int mlx4_ib_query_port(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props) +{ + /* returns host view */ + return __mlx4_ib_query_port(ibdev, port, props, 0); +} + +int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index, + union ib_gid *gid, int netw_view) +{ + struct ib_smp *in_mad = NULL; + struct ib_smp *out_mad = NULL; + int err = -ENOMEM; + struct mlx4_ib_dev *dev = to_mdev(ibdev); + int clear = 0; + int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS; + + in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); + out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); + if (!in_mad || !out_mad) + goto out; + + init_query_mad(in_mad); + in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; + in_mad->attr_mod = cpu_to_be32(port); + + if (mlx4_is_mfunc(dev->dev) && netw_view) + mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW; + + err = mlx4_MAD_IFC(dev, mad_ifc_flags, port, NULL, NULL, in_mad, out_mad); + if (err) + goto out; + + memcpy(gid->raw, out_mad->data + 8, 8); + + if (mlx4_is_mfunc(dev->dev) && !netw_view) { + if (index) { + /* For any index > 0, return the null guid */ + err = 0; + clear = 1; + goto out; + } + } + + init_query_mad(in_mad); + in_mad->attr_id = IB_SMP_ATTR_GUID_INFO; + in_mad->attr_mod = cpu_to_be32(index / 8); + + err = mlx4_MAD_IFC(dev, mad_ifc_flags, port, + NULL, NULL, in_mad, out_mad); + if (err) + goto out; + + memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8); + +out: + if (clear) + memset(gid->raw + 8, 0, 8); + kfree(in_mad); + kfree(out_mad); + return err; +} + +static int iboe_query_gid(struct ib_device *ibdev, u8 port, int index, + union ib_gid *gid) +{ + struct mlx4_ib_dev *dev = to_mdev(ibdev); + + *gid = dev->iboe.gid_table[port - 1][index]; + + return 0; +} + +static int mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index, + union ib_gid *gid) +{ + if (rdma_port_get_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND) + return __mlx4_ib_query_gid(ibdev, port, index, gid, 0); + else + return iboe_query_gid(ibdev, port, index, gid); +} + +int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, + u16 *pkey, int netw_view) +{ + struct ib_smp *in_mad = NULL; + struct ib_smp *out_mad = NULL; + int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS; + int err = -ENOMEM; + + in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); + out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); + if (!in_mad || !out_mad) + goto out; + + init_query_mad(in_mad); + in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE; + in_mad->attr_mod = cpu_to_be32(index / 32); + + if (mlx4_is_mfunc(to_mdev(ibdev)->dev) && netw_view) + mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW; + + err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, NULL, NULL, + in_mad, out_mad); + if (err) + goto out; + + *pkey = be16_to_cpu(((__be16 *) out_mad->data)[index % 32]); + +out: + kfree(in_mad); + kfree(out_mad); + return err; +} + +static int mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) +{ + return __mlx4_ib_query_pkey(ibdev, port, index, pkey, 0); +} + +static int mlx4_ib_modify_device(struct ib_device *ibdev, int mask, + struct ib_device_modify *props) +{ + struct mlx4_cmd_mailbox *mailbox; + unsigned long flags; + + if (mask & ~IB_DEVICE_MODIFY_NODE_DESC) + return -EOPNOTSUPP; + + if (!(mask & IB_DEVICE_MODIFY_NODE_DESC)) + return 0; + + if (mlx4_is_slave(to_mdev(ibdev)->dev)) + return -EOPNOTSUPP; + + spin_lock_irqsave(&to_mdev(ibdev)->sm_lock, flags); + memcpy(ibdev->node_desc, props->node_desc, 64); + spin_unlock_irqrestore(&to_mdev(ibdev)->sm_lock, flags); + + /* + * If possible, pass node desc to FW, so it can generate + * a 144 trap. If cmd fails, just ignore. + */ + mailbox = mlx4_alloc_cmd_mailbox(to_mdev(ibdev)->dev); + if (IS_ERR(mailbox)) + return 0; + + memset(mailbox->buf, 0, 256); + memcpy(mailbox->buf, props->node_desc, 64); + mlx4_cmd(to_mdev(ibdev)->dev, mailbox->dma, 1, 0, + MLX4_CMD_SET_NODE, MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(to_mdev(ibdev)->dev, mailbox); + + return 0; +} + +static int mlx4_SET_PORT(struct mlx4_ib_dev *dev, u8 port, int reset_qkey_viols, + u32 cap_mask) +{ + struct mlx4_cmd_mailbox *mailbox; + int err; + u8 is_eth = dev->dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH; + + mailbox = mlx4_alloc_cmd_mailbox(dev->dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + memset(mailbox->buf, 0, 256); + + if (dev->dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { + *(u8 *) mailbox->buf = !!reset_qkey_viols << 6; + ((__be32 *) mailbox->buf)[2] = cpu_to_be32(cap_mask); + } else { + ((u8 *) mailbox->buf)[3] = !!reset_qkey_viols; + ((__be32 *) mailbox->buf)[1] = cpu_to_be32(cap_mask); + } + + err = mlx4_cmd(dev->dev, mailbox->dma, port, is_eth, MLX4_CMD_SET_PORT, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev->dev, mailbox); + return err; +} + +static int mlx4_ib_modify_port(struct ib_device *ibdev, u8 port, int mask, + struct ib_port_modify *props) +{ + struct ib_port_attr attr; + u32 cap_mask; + int err; + + mutex_lock(&to_mdev(ibdev)->cap_mask_mutex); + + err = mlx4_ib_query_port(ibdev, port, &attr); + if (err) + goto out; + + cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) & + ~props->clr_port_cap_mask; + + err = mlx4_SET_PORT(to_mdev(ibdev), port, + !!(mask & IB_PORT_RESET_QKEY_CNTR), + cap_mask); + +out: + mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex); + return err; +} + +static struct ib_ucontext *mlx4_ib_alloc_ucontext(struct ib_device *ibdev, + struct ib_udata *udata) +{ + struct mlx4_ib_dev *dev = to_mdev(ibdev); + struct mlx4_ib_ucontext *context; + struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3; + struct mlx4_ib_alloc_ucontext_resp resp; + int err; + + if (!dev->ib_active) + return ERR_PTR(-EAGAIN); + + if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) { + resp_v3.qp_tab_size = dev->dev->caps.num_qps; + if (mlx4_wc_enabled()) { + resp_v3.bf_reg_size = dev->dev->caps.bf_reg_size; + resp_v3.bf_regs_per_page = dev->dev->caps.bf_regs_per_page; + } else { + resp_v3.bf_reg_size = 0; + resp_v3.bf_regs_per_page = 0; + } + } else { + resp.dev_caps = dev->dev->caps.userspace_caps; + resp.qp_tab_size = dev->dev->caps.num_qps; + if (mlx4_wc_enabled()) { + resp.bf_reg_size = dev->dev->caps.bf_reg_size; + resp.bf_regs_per_page = dev->dev->caps.bf_regs_per_page; + } else { + resp.bf_reg_size = 0; + resp.bf_regs_per_page = 0; + } + resp.cqe_size = dev->dev->caps.cqe_size; + } + + context = kmalloc(sizeof *context, GFP_KERNEL); + if (!context) + return ERR_PTR(-ENOMEM); + + err = mlx4_uar_alloc(to_mdev(ibdev)->dev, &context->uar); + if (err) { + kfree(context); + return ERR_PTR(err); + } + + INIT_LIST_HEAD(&context->db_page_list); + mutex_init(&context->db_page_mutex); + + if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) + err = ib_copy_to_udata(udata, &resp_v3, sizeof(resp_v3)); + else + err = ib_copy_to_udata(udata, &resp, sizeof(resp)); + + if (err) { + mlx4_uar_free(to_mdev(ibdev)->dev, &context->uar); + kfree(context); + return ERR_PTR(-EFAULT); + } + + return &context->ibucontext; +} + +static int mlx4_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) +{ + struct mlx4_ib_ucontext *context = to_mucontext(ibcontext); + + mlx4_uar_free(to_mdev(ibcontext->device)->dev, &context->uar); + kfree(context); + + return 0; +} + +/* XXX FBSD has no support for get_unmapped_area function */ +#if 0 +static unsigned long mlx4_ib_get_unmapped_area(struct file *file, + unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long start_addr; + unsigned long page_size_order; + unsigned long command; + + mm = current->mm; + if (addr) + return current->mm->get_unmapped_area(file, addr, len, + pgoff, flags); + + /* Last 8 bits hold the command others are data per that command */ + command = pgoff & MLX4_IB_MMAP_CMD_MASK; + if (command != MLX4_IB_MMAP_GET_CONTIGUOUS_PAGES) + return current->mm->get_unmapped_area(file, addr, len, + pgoff, flags); + + page_size_order = pgoff >> MLX4_IB_MMAP_CMD_BITS; + /* code is based on the huge-pages get_unmapped_area code */ + start_addr = mm->free_area_cache; + + if (len <= mm->cached_hole_size) + start_addr = TASK_UNMAPPED_BASE; + + +full_search: + addr = ALIGN(start_addr, 1 << page_size_order); + + for (vma = find_vma(mm, addr); ; vma = vma->vm_next) { + /* At this point: (!vma || addr < vma->vm_end). */ + if (TASK_SIZE - len < addr) { + /* + * Start a new search - just in case we missed + * some holes. + */ + if (start_addr != TASK_UNMAPPED_BASE) { + start_addr = TASK_UNMAPPED_BASE; + goto full_search; + } + return -ENOMEM; + } + + if (!vma || addr + len <= vma->vm_start) + return addr; + addr = ALIGN(vma->vm_end, 1 << page_size_order); + } +} +#endif + +static int mlx4_ib_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) +{ + struct mlx4_ib_dev *dev = to_mdev(context->device); + + /* Last 8 bits hold the command others are data per that command */ + unsigned long command = vma->vm_pgoff & MLX4_IB_MMAP_CMD_MASK; + + if (command < MLX4_IB_MMAP_GET_CONTIGUOUS_PAGES) { + /* compatibility handling for commands 0 & 1*/ + if (vma->vm_end - vma->vm_start != PAGE_SIZE) + return -EINVAL; + } + if (command == MLX4_IB_MMAP_UAR_PAGE) { + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + if (io_remap_pfn_range(vma, vma->vm_start, + to_mucontext(context)->uar.pfn, + PAGE_SIZE, vma->vm_page_prot)) + return -EAGAIN; + } else if (command == MLX4_IB_MMAP_BLUE_FLAME_PAGE && + dev->dev->caps.bf_reg_size != 0) { + vma->vm_page_prot = pgprot_wc(vma->vm_page_prot); + + if (io_remap_pfn_range(vma, vma->vm_start, + to_mucontext(context)->uar.pfn + + dev->dev->caps.num_uars, + PAGE_SIZE, vma->vm_page_prot)) + return -EAGAIN; + } else if (command == MLX4_IB_MMAP_GET_HW_CLOCK) { + struct mlx4_clock_params params; + int ret; + + ret = mlx4_get_internal_clock_params(dev->dev, ¶ms); + if (ret) + return ret; + + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + if (io_remap_pfn_range(vma, vma->vm_start, + (pci_resource_start(dev->dev->persist->pdev, + params.bar) + + params.offset) + >> PAGE_SHIFT, + PAGE_SIZE, vma->vm_page_prot)) + return -EAGAIN; + } else + return -EINVAL; + + return 0; +} + +static int mlx4_ib_ioctl(struct ib_ucontext *context, unsigned int cmd, + unsigned long arg) +{ + struct mlx4_ib_dev *dev = to_mdev(context->device); + int ret; + int offset; + + switch (cmd) { + case MLX4_IOCHWCLOCKOFFSET: { + struct mlx4_clock_params params; + int ret; + ret = mlx4_get_internal_clock_params(dev->dev, ¶ms); + if (!ret) { + offset = params.offset % PAGE_SIZE; + ret = put_user(offset, + (int *)arg); + return sizeof(int); + } else { + return ret; + } + } + default: { + pr_err("mlx4_ib: invalid ioctl %u command with arg %lX\n", + cmd, arg); + return -ENOTTY; + } + } + + return ret; +} + +static int mlx4_ib_query_values(struct ib_device *device, int q_values, + struct ib_device_values *values) +{ + struct mlx4_ib_dev *dev = to_mdev(device); + s64 cycles; + + values->values_mask = 0; + if (q_values & IBV_VALUES_HW_CLOCK) { + cycles = mlx4_read_clock(dev->dev); + if (cycles >= 0) { + values->hwclock = cycles & CORE_CLOCK_MASK; + values->values_mask |= IBV_VALUES_HW_CLOCK; + } + q_values &= ~IBV_VALUES_HW_CLOCK; + } + + if (q_values) + return -ENOTTY; + + return 0; +} + +static struct ib_pd *mlx4_ib_alloc_pd(struct ib_device *ibdev, + struct ib_ucontext *context, + struct ib_udata *udata) +{ + struct mlx4_ib_pd *pd; + int err; + + pd = kmalloc(sizeof *pd, GFP_KERNEL); + if (!pd) + return ERR_PTR(-ENOMEM); + + err = mlx4_pd_alloc(to_mdev(ibdev)->dev, &pd->pdn); + if (err) { + kfree(pd); + return ERR_PTR(err); + } + + if (context) + if (ib_copy_to_udata(udata, &pd->pdn, sizeof (__u32))) { + mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn); + kfree(pd); + return ERR_PTR(-EFAULT); + } + + return &pd->ibpd; +} + +static int mlx4_ib_dealloc_pd(struct ib_pd *pd) +{ + mlx4_pd_free(to_mdev(pd->device)->dev, to_mpd(pd)->pdn); + kfree(pd); + + return 0; +} + +static struct ib_xrcd *mlx4_ib_alloc_xrcd(struct ib_device *ibdev, + struct ib_ucontext *context, + struct ib_udata *udata) +{ + struct mlx4_ib_xrcd *xrcd; + int err; + + if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)) + return ERR_PTR(-ENOSYS); + + xrcd = kmalloc(sizeof *xrcd, GFP_KERNEL); + if (!xrcd) + return ERR_PTR(-ENOMEM); + + err = mlx4_xrcd_alloc(to_mdev(ibdev)->dev, &xrcd->xrcdn); + if (err) + goto err1; + + xrcd->pd = ib_alloc_pd(ibdev); + if (IS_ERR(xrcd->pd)) { + err = PTR_ERR(xrcd->pd); + goto err2; + } + + xrcd->cq = ib_create_cq(ibdev, NULL, NULL, xrcd, 1, 0); + if (IS_ERR(xrcd->cq)) { + err = PTR_ERR(xrcd->cq); + goto err3; + } + + return &xrcd->ibxrcd; + +err3: + ib_dealloc_pd(xrcd->pd); +err2: + mlx4_xrcd_free(to_mdev(ibdev)->dev, xrcd->xrcdn); +err1: + kfree(xrcd); + return ERR_PTR(err); +} + +static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd) +{ + ib_destroy_cq(to_mxrcd(xrcd)->cq); + ib_dealloc_pd(to_mxrcd(xrcd)->pd); + mlx4_xrcd_free(to_mdev(xrcd->device)->dev, to_mxrcd(xrcd)->xrcdn); + kfree(xrcd); + + return 0; +} + +static int add_gid_entry(struct ib_qp *ibqp, union ib_gid *gid) +{ + struct mlx4_ib_qp *mqp = to_mqp(ibqp); + struct mlx4_ib_dev *mdev = to_mdev(ibqp->device); + struct mlx4_ib_gid_entry *ge; + + ge = kzalloc(sizeof *ge, GFP_KERNEL); + if (!ge) + return -ENOMEM; + + ge->gid = *gid; + if (mlx4_ib_add_mc(mdev, mqp, gid)) { + ge->port = mqp->port; + ge->added = 1; + } + + mutex_lock(&mqp->mutex); + list_add_tail(&ge->list, &mqp->gid_list); + mutex_unlock(&mqp->mutex); + + return 0; +} + +int mlx4_ib_add_mc(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp, + union ib_gid *gid) +{ + u8 mac[6]; + struct net_device *ndev; + int ret = 0; + + if (!mqp->port) + return 0; + + spin_lock(&mdev->iboe.lock); + ndev = mdev->iboe.netdevs[mqp->port - 1]; + if (ndev) + dev_hold(ndev); + spin_unlock(&mdev->iboe.lock); + + if (ndev) { + rdma_get_mcast_mac((struct in6_addr *)gid, mac); + rtnl_lock(); + dev_mc_add(mdev->iboe.netdevs[mqp->port - 1], mac, 6, 0); + ret = 1; + rtnl_unlock(); + dev_put(ndev); + } + + return ret; +} + +struct mlx4_ib_steering { + struct list_head list; + u64 reg_id; + union ib_gid gid; +}; + +static int parse_flow_attr(struct mlx4_dev *dev, + union ib_flow_spec *ib_spec, + struct _rule_hw *mlx4_spec) +{ + enum mlx4_net_trans_rule_id type; + + switch (ib_spec->type) { + case IB_FLOW_SPEC_ETH: + type = MLX4_NET_TRANS_RULE_ID_ETH; + memcpy(mlx4_spec->eth.dst_mac, ib_spec->eth.val.dst_mac, + ETH_ALEN); + memcpy(mlx4_spec->eth.dst_mac_msk, ib_spec->eth.mask.dst_mac, + ETH_ALEN); + mlx4_spec->eth.vlan_tag = ib_spec->eth.val.vlan_tag; + mlx4_spec->eth.vlan_tag_msk = ib_spec->eth.mask.vlan_tag; + break; + + case IB_FLOW_SPEC_IB: + type = MLX4_NET_TRANS_RULE_ID_IB; + mlx4_spec->ib.l3_qpn = ib_spec->ib.val.l3_type_qpn; + mlx4_spec->ib.qpn_mask = ib_spec->ib.mask.l3_type_qpn; + memcpy(&mlx4_spec->ib.dst_gid, ib_spec->ib.val.dst_gid, 16); + memcpy(&mlx4_spec->ib.dst_gid_msk, + ib_spec->ib.mask.dst_gid, 16); + break; + + case IB_FLOW_SPEC_IPV4: + type = MLX4_NET_TRANS_RULE_ID_IPV4; + mlx4_spec->ipv4.src_ip = ib_spec->ipv4.val.src_ip; + mlx4_spec->ipv4.src_ip_msk = ib_spec->ipv4.mask.src_ip; + mlx4_spec->ipv4.dst_ip = ib_spec->ipv4.val.dst_ip; + mlx4_spec->ipv4.dst_ip_msk = ib_spec->ipv4.mask.dst_ip; + break; + + case IB_FLOW_SPEC_TCP: + case IB_FLOW_SPEC_UDP: + type = ib_spec->type == IB_FLOW_SPEC_TCP ? + MLX4_NET_TRANS_RULE_ID_TCP : + MLX4_NET_TRANS_RULE_ID_UDP; + mlx4_spec->tcp_udp.dst_port = ib_spec->tcp_udp.val.dst_port; + mlx4_spec->tcp_udp.dst_port_msk = + ib_spec->tcp_udp.mask.dst_port; + mlx4_spec->tcp_udp.src_port = ib_spec->tcp_udp.val.src_port; + mlx4_spec->tcp_udp.src_port_msk = + ib_spec->tcp_udp.mask.src_port; + break; + + default: + return -EINVAL; + } + if (mlx4_map_sw_to_hw_steering_id(dev, type) < 0 || + mlx4_hw_rule_sz(dev, type) < 0) + return -EINVAL; + mlx4_spec->id = cpu_to_be16(mlx4_map_sw_to_hw_steering_id(dev, type)); + mlx4_spec->size = mlx4_hw_rule_sz(dev, type) >> 2; + return mlx4_hw_rule_sz(dev, type); +} + +static int __mlx4_ib_create_flow(struct ib_qp *qp, struct ib_flow_attr *flow_attr, + int domain, + enum mlx4_net_trans_promisc_mode flow_type, + u64 *reg_id) +{ + int ret, i; + int size = 0; + void *ib_flow; + struct mlx4_ib_dev *mdev = to_mdev(qp->device); + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_net_trans_rule_hw_ctrl *ctrl; + size_t rule_size = sizeof(struct mlx4_net_trans_rule_hw_ctrl) + + (sizeof(struct _rule_hw) * flow_attr->num_of_specs); + + static const u16 __mlx4_domain[] = { + [IB_FLOW_DOMAIN_USER] = MLX4_DOMAIN_UVERBS, + [IB_FLOW_DOMAIN_ETHTOOL] = MLX4_DOMAIN_ETHTOOL, + [IB_FLOW_DOMAIN_RFS] = MLX4_DOMAIN_RFS, + [IB_FLOW_DOMAIN_NIC] = MLX4_DOMAIN_NIC, + }; + + if (flow_attr->priority > MLX4_IB_FLOW_MAX_PRIO) { + pr_err("Invalid priority value.\n"); + return -EINVAL; + } + if (domain >= IB_FLOW_DOMAIN_NUM) { + pr_err("Invalid domain value.\n"); + return -EINVAL; + } + + if (mlx4_map_sw_to_hw_steering_mode(mdev->dev, flow_type) < 0) + return -EINVAL; + + mailbox = mlx4_alloc_cmd_mailbox(mdev->dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + memset(mailbox->buf, 0, rule_size); + ctrl = mailbox->buf; + + ctrl->prio = cpu_to_be16(__mlx4_domain[domain] | + flow_attr->priority); + ctrl->type = mlx4_map_sw_to_hw_steering_mode(mdev->dev, flow_type); + ctrl->port = flow_attr->port; + ctrl->qpn = cpu_to_be32(qp->qp_num); + + if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_ALLOW_LOOP_BACK) + ctrl->flags = (1 << 3); + + ib_flow = flow_attr + 1; + size += sizeof(struct mlx4_net_trans_rule_hw_ctrl); + for (i = 0; i < flow_attr->num_of_specs; i++) { + ret = parse_flow_attr(mdev->dev, ib_flow, mailbox->buf + size); + if (ret < 0) { + mlx4_free_cmd_mailbox(mdev->dev, mailbox); + return -EINVAL; + } + ib_flow += ((union ib_flow_spec *)ib_flow)->size; + size += ret; + } + + ret = mlx4_cmd_imm(mdev->dev, mailbox->dma, reg_id, size >> 2, 0, + MLX4_QP_FLOW_STEERING_ATTACH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (ret == -ENOMEM) + pr_err("mcg table is full. Fail to register network rule.\n"); + else if (ret == -ENXIO) + pr_err("Device managed flow steering is disabled. Fail to register network rule.\n"); + else if (ret) + pr_err("Invalid argumant. Fail to register network rule.\n"); + mlx4_free_cmd_mailbox(mdev->dev, mailbox); + return ret; +} + +static int __mlx4_ib_destroy_flow(struct mlx4_dev *dev, u64 reg_id) +{ + int err; + err = mlx4_cmd(dev, reg_id, 0, 0, + MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + pr_err("Fail to detach network rule. registration id = 0x%llx\n", + (unsigned long long)reg_id); + return err; +} + +static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp, + struct ib_flow_attr *flow_attr, + int domain) +{ + int err = 0, i = 0; + struct mlx4_ib_flow *mflow; + enum mlx4_net_trans_promisc_mode type[2]; + + memset(type, 0, sizeof(type)); + + mflow = kzalloc(sizeof(struct mlx4_ib_flow), GFP_KERNEL); + if (!mflow) { + err = -ENOMEM; + goto err_free; + } + + switch (flow_attr->type) { + case IB_FLOW_ATTR_NORMAL: + type[0] = MLX4_FS_REGULAR; + break; + + case IB_FLOW_ATTR_ALL_DEFAULT: + type[0] = MLX4_FS_ALL_DEFAULT; + break; + + case IB_FLOW_ATTR_MC_DEFAULT: + type[0] = MLX4_FS_MC_DEFAULT; + break; + + case IB_FLOW_ATTR_SNIFFER: + type[0] = MLX4_FS_UC_SNIFFER; + type[1] = MLX4_FS_MC_SNIFFER; + break; + + default: + err = -EINVAL; + goto err_free; + } + + while (i < ARRAY_SIZE(type) && type[i]) { + err = __mlx4_ib_create_flow(qp, flow_attr, domain, type[i], + &mflow->reg_id[i]); + if (err) + goto err_free; + i++; + } + + return &mflow->ibflow; + +err_free: + kfree(mflow); + return ERR_PTR(err); +} + +static int mlx4_ib_destroy_flow(struct ib_flow *flow_id) +{ + int err, ret = 0; + int i = 0; + struct mlx4_ib_dev *mdev = to_mdev(flow_id->qp->device); + struct mlx4_ib_flow *mflow = to_mflow(flow_id); + + while (i < ARRAY_SIZE(mflow->reg_id) && mflow->reg_id[i]) { + err = __mlx4_ib_destroy_flow(mdev->dev, mflow->reg_id[i]); + if (err) + ret = err; + i++; + } + + kfree(mflow); + return ret; +} + +static struct mlx4_ib_gid_entry *find_gid_entry(struct mlx4_ib_qp *qp, u8 *raw) +{ + struct mlx4_ib_gid_entry *ge; + struct mlx4_ib_gid_entry *tmp; + struct mlx4_ib_gid_entry *ret = NULL; + + list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) { + if (!memcmp(raw, ge->gid.raw, 16)) { + ret = ge; + break; + } + } + + return ret; +} + + +static int del_gid_entry(struct ib_qp *ibqp, union ib_gid *gid) +{ + struct mlx4_ib_dev *mdev = to_mdev(ibqp->device); + struct mlx4_ib_qp *mqp = to_mqp(ibqp); + struct mlx4_ib_gid_entry *ge; + struct net_device *ndev; + u8 mac[6]; + + mutex_lock(&mqp->mutex); + ge = find_gid_entry(mqp, gid->raw); + if (ge) { + spin_lock(&mdev->iboe.lock); + ndev = ge->added ? mdev->iboe.netdevs[ge->port - 1] : NULL; + if (ndev) + dev_hold(ndev); + spin_unlock(&mdev->iboe.lock); + rdma_get_mcast_mac((struct in6_addr *)gid, mac); + if (ndev) { + rtnl_lock(); + dev_mc_delete(mdev->iboe.netdevs[ge->port - 1], mac, 6, 0); + rtnl_unlock(); + dev_put(ndev); + } + list_del(&ge->list); + kfree(ge); + } else + pr_warn("could not find mgid entry\n"); + + mutex_unlock(&mqp->mutex); + return ge != NULL ? 0 : -EINVAL; +} + +static int _mlx4_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid, + int count) +{ + int err; + struct mlx4_ib_dev *mdev = to_mdev(ibqp->device); + struct mlx4_ib_qp *mqp = to_mqp(ibqp); + u64 reg_id = 0; + int record_err = 0; + + if (mdev->dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) { + struct mlx4_ib_steering *ib_steering; + struct mlx4_ib_steering *tmp; + LIST_HEAD(temp); + + mutex_lock(&mqp->mutex); + list_for_each_entry_safe(ib_steering, tmp, &mqp->steering_rules, + list) { + if (memcmp(ib_steering->gid.raw, gid->raw, 16)) + continue; + + if (--count < 0) + break; + + list_del(&ib_steering->list); + list_add(&ib_steering->list, &temp); + } + mutex_unlock(&mqp->mutex); + list_for_each_entry_safe(ib_steering, tmp, &temp, + list) { + reg_id = ib_steering->reg_id; + + err = mlx4_multicast_detach(mdev->dev, &mqp->mqp, + gid->raw, + (ibqp->qp_type == IB_QPT_RAW_PACKET) ? + MLX4_PROT_ETH : MLX4_PROT_IB_IPV6, + reg_id); + if (err) { + record_err = record_err ?: err; + continue; + } + + err = del_gid_entry(ibqp, gid); + if (err) { + record_err = record_err ?: err; + continue; + } + + list_del(&ib_steering->list); + kfree(ib_steering); + } + mutex_lock(&mqp->mutex); + list_for_each_entry(ib_steering, &temp, list) { + list_add(&ib_steering->list, &mqp->steering_rules); + } + mutex_unlock(&mqp->mutex); + if (count) { + pr_warn("Couldn't release all reg_ids for mgid. Steering rule is left attached\n"); + return -EINVAL; + } + + } else { + if (mdev->dev->caps.steering_mode == MLX4_STEERING_MODE_B0 && + ibqp->qp_type == IB_QPT_RAW_PACKET) + gid->raw[5] = mqp->port; + + err = mlx4_multicast_detach(mdev->dev, &mqp->mqp, gid->raw, + (ibqp->qp_type == IB_QPT_RAW_PACKET) ? + MLX4_PROT_ETH : MLX4_PROT_IB_IPV6, + reg_id); + if (err) + return err; + + err = del_gid_entry(ibqp, gid); + + if (err) + return err; + } + + return record_err; +} + +static int mlx4_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) +{ + struct mlx4_ib_dev *mdev = to_mdev(ibqp->device); + int count = (mdev->dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) ? + mdev->dev->caps.num_ports : 1; + + return _mlx4_ib_mcg_detach(ibqp, gid, lid, count); +} + +static int mlx4_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) +{ + int err = -ENODEV; + struct mlx4_ib_dev *mdev = to_mdev(ibqp->device); + struct mlx4_ib_qp *mqp = to_mqp(ibqp); + DECLARE_BITMAP(ports, MLX4_MAX_PORTS); + int i = 0; + + if (mdev->dev->caps.steering_mode == MLX4_STEERING_MODE_B0 && + ibqp->qp_type == IB_QPT_RAW_PACKET) + gid->raw[5] = mqp->port; + + if (mdev->dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) { + bitmap_fill(ports, mdev->dev->caps.num_ports); + } else { + if (mqp->port <= mdev->dev->caps.num_ports) { + bitmap_zero(ports, mdev->dev->caps.num_ports); + set_bit(0, ports); + } else { + return -EINVAL; + } + } + + for (; i < mdev->dev->caps.num_ports; i++) { + u64 reg_id; + struct mlx4_ib_steering *ib_steering = NULL; + if (!test_bit(i, ports)) + continue; + if (mdev->dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) { + ib_steering = kmalloc(sizeof(*ib_steering), GFP_KERNEL); + if (!ib_steering) + goto err_add; + } + + err = mlx4_multicast_attach(mdev->dev, &mqp->mqp, + gid->raw, i + 1, + !!(mqp->flags & + MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK), + (ibqp->qp_type == IB_QPT_RAW_PACKET) ? + MLX4_PROT_ETH : MLX4_PROT_IB_IPV6, + ®_id); + if (err) { + kfree(ib_steering); + goto err_add; + } + + err = add_gid_entry(ibqp, gid); + if (err) { + mlx4_multicast_detach(mdev->dev, &mqp->mqp, gid->raw, + MLX4_PROT_IB_IPV6, reg_id); + kfree(ib_steering); + goto err_add; + } + + if (ib_steering) { + memcpy(ib_steering->gid.raw, gid->raw, 16); + mutex_lock(&mqp->mutex); + list_add(&ib_steering->list, &mqp->steering_rules); + mutex_unlock(&mqp->mutex); + ib_steering->reg_id = reg_id; + } + } + + + return 0; + +err_add: + if (i > 0) + _mlx4_ib_mcg_detach(ibqp, gid, lid, i); + + return err; +} + +static int init_node_data(struct mlx4_ib_dev *dev) +{ + struct ib_smp *in_mad = NULL; + struct ib_smp *out_mad = NULL; + int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS; + int err = -ENOMEM; + + in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); + out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); + if (!in_mad || !out_mad) + goto out; + + init_query_mad(in_mad); + in_mad->attr_id = IB_SMP_ATTR_NODE_DESC; + if (mlx4_is_master(dev->dev)) + mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW; + + err = mlx4_MAD_IFC(dev, mad_ifc_flags, 1, NULL, NULL, in_mad, out_mad); + if (err) + goto out; + + memcpy(dev->ib_dev.node_desc, out_mad->data, 64); + + in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; + + err = mlx4_MAD_IFC(dev, mad_ifc_flags, 1, NULL, NULL, in_mad, out_mad); + if (err) + goto out; + + dev->dev->rev_id = be32_to_cpup((__be32 *) (out_mad->data + 32)); + memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8); + +out: + kfree(in_mad); + kfree(out_mad); + return err; +} + +static ssize_t show_hca(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct mlx4_ib_dev *dev = + container_of(device, struct mlx4_ib_dev, ib_dev.dev); + return sprintf(buf, "MT%d\n", dev->dev->persist->pdev->device); +} + +static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct mlx4_ib_dev *dev = + container_of(device, struct mlx4_ib_dev, ib_dev.dev); + return sprintf(buf, "%d.%d.%d\n", (int) (dev->dev->caps.fw_ver >> 32), + (int) (dev->dev->caps.fw_ver >> 16) & 0xffff, + (int) dev->dev->caps.fw_ver & 0xffff); +} + +static ssize_t show_rev(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct mlx4_ib_dev *dev = + container_of(device, struct mlx4_ib_dev, ib_dev.dev); + return sprintf(buf, "%x\n", dev->dev->rev_id); +} + +static ssize_t show_board(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct mlx4_ib_dev *dev = + container_of(device, struct mlx4_ib_dev, ib_dev.dev); + return sprintf(buf, "%.*s\n", MLX4_BOARD_ID_LEN, + dev->dev->board_id); +} + +static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL); +static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL); +static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL); +static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL); + +static struct device_attribute *mlx4_class_attributes[] = { + &dev_attr_hw_rev, + &dev_attr_fw_ver, + &dev_attr_hca_type, + &dev_attr_board_id +}; + +static void mlx4_addrconf_ifid_eui48(u8 *eui, u16 vlan_id, struct net_device *dev, u8 port) +{ + memcpy(eui, IF_LLADDR(dev), 3); + memcpy(eui + 5, IF_LLADDR(dev) + 3, 3); + if (vlan_id < 0x1000) { + eui[3] = vlan_id >> 8; + eui[4] = vlan_id & 0xff; + } else { + eui[3] = 0xff; + eui[4] = 0xfe; + } + eui[0] ^= 2; +} + +static void update_gids_task(struct work_struct *work) +{ + struct update_gid_work *gw = container_of(work, struct update_gid_work, work); + struct mlx4_cmd_mailbox *mailbox; + union ib_gid *gids; + int err; + struct mlx4_dev *dev = gw->dev->dev; + + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + pr_warn("update gid table failed %ld\n", PTR_ERR(mailbox)); + goto free; + } + + gids = mailbox->buf; + memcpy(gids, gw->gids, sizeof gw->gids); + + if (mlx4_ib_port_link_layer(&gw->dev->ib_dev, gw->port) == + IB_LINK_LAYER_ETHERNET) { + err = mlx4_cmd(dev, mailbox->dma, + MLX4_SET_PORT_GID_TABLE << 8 | gw->port, + 1, MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); + + if (err) + pr_warn("set port command failed\n"); + else + mlx4_ib_dispatch_event(gw->dev, gw->port, + IB_EVENT_GID_CHANGE); + } + + mlx4_free_cmd_mailbox(dev, mailbox); +free: + kfree(gw); +} + +static struct net_device *mlx4_ib_get_netdev(struct ib_device *device, u8 port_num) +{ + struct mlx4_ib_dev *ibdev = to_mdev(device); + return mlx4_get_protocol_dev(ibdev->dev, MLX4_PROT_ETH, port_num); +} + +static void reset_gids_task(struct work_struct *work) +{ + struct update_gid_work *gw = + container_of(work, struct update_gid_work, work); + struct mlx4_cmd_mailbox *mailbox; + union ib_gid *gids; + int err; + struct mlx4_dev *dev = gw->dev->dev; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + pr_warn("reset gid table failed\n"); + goto free; + } + + gids = mailbox->buf; + memcpy(gids, gw->gids, sizeof(gw->gids)); + + if (mlx4_ib_port_link_layer(&gw->dev->ib_dev, 1) == + IB_LINK_LAYER_ETHERNET && + dev->caps.num_ports > 0) { + err = mlx4_cmd(dev, mailbox->dma, + MLX4_SET_PORT_GID_TABLE << 8 | 1, + 1, MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); + if (err) + pr_warn("set port 1 command failed\n"); + } + + if (mlx4_ib_port_link_layer(&gw->dev->ib_dev, 2) == + IB_LINK_LAYER_ETHERNET && + dev->caps.num_ports > 1) { + err = mlx4_cmd(dev, mailbox->dma, + MLX4_SET_PORT_GID_TABLE << 8 | 2, + 1, MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); + if (err) + pr_warn("set port 2 command failed\n"); + } + + mlx4_free_cmd_mailbox(dev, mailbox); +free: + kfree(gw); +} + +static int update_gid_table(struct mlx4_ib_dev *dev, int port, + union ib_gid *gid, int clear, int default_gid) +{ + struct update_gid_work *work; + int i; + int need_update = 0; + int free = -1; + int found = -1; + int max_gids; + int start_index = !default_gid; + + max_gids = dev->dev->caps.gid_table_len[port]; + for (i = start_index; i < max_gids; ++i) { + if (!memcmp(&dev->iboe.gid_table[port - 1][i], gid, + sizeof(*gid))) + found = i; + + if (clear) { + if (found >= 0) { + need_update = 1; + dev->iboe.gid_table[port - 1][found] = zgid; + break; + } + } else { + if (found >= 0) + break; + + if (free < 0 && + !memcmp(&dev->iboe.gid_table[port - 1][i], + &zgid, sizeof(*gid))) + free = i; + } + } + + if (found == -1 && !clear && free < 0) { + pr_err("GID table of port %d is full. Can't add "GID_PRINT_FMT"\n", + port, GID_PRINT_ARGS(gid)); + return -ENOMEM; + } + if (found == -1 && clear) { + pr_err(GID_PRINT_FMT" is not in GID table of port %d\n", GID_PRINT_ARGS(gid), port); + return -EINVAL; + } + if (found == -1 && !clear && free >= 0) { + dev->iboe.gid_table[port - 1][free] = *gid; + need_update = 1; + } + + if (!need_update) + return 0; + + work = kzalloc(sizeof *work, GFP_ATOMIC); + if (!work) + return -ENOMEM; + + memcpy(work->gids, dev->iboe.gid_table[port - 1], sizeof(work->gids)); + INIT_WORK(&work->work, update_gids_task); + work->port = port; + work->dev = dev; + queue_work(wq, &work->work); + + return 0; +} + +static int reset_gid_table(struct mlx4_ib_dev *dev) +{ + struct update_gid_work *work; + + + work = kzalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return -ENOMEM; + + memset(dev->iboe.gid_table, 0, sizeof(dev->iboe.gid_table)); + memset(work->gids, 0, sizeof(work->gids)); + INIT_WORK(&work->work, reset_gids_task); + work->dev = dev; + queue_work(wq, &work->work); + return 0; +} + +/* XXX BOND Related - stub (no support for these flags in FBSD)*/ +static inline int netif_is_bond_master(struct net_device *dev) +{ +#if 0 + return (dev->flags & IFF_MASTER) && (dev->priv_flags & IFF_BONDING); +#endif + return 0; +} + +static void mlx4_make_default_gid(struct net_device *dev, union ib_gid *gid, u8 port) +{ + gid->global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL); + mlx4_addrconf_ifid_eui48(&gid->raw[8], 0xffff, dev, port); +} + +static u8 mlx4_ib_get_dev_port(struct net_device *dev, struct mlx4_ib_dev *ibdev) +{ + u8 port = 0; + struct mlx4_ib_iboe *iboe; + struct net_device *real_dev = rdma_vlan_dev_real_dev(dev) ? + rdma_vlan_dev_real_dev(dev) : dev; + + iboe = &ibdev->iboe; + + for (port = 1; port <= MLX4_MAX_PORTS; ++port) + if ((netif_is_bond_master(real_dev) && (real_dev == iboe->masters[port - 1])) || + (!netif_is_bond_master(real_dev) && (real_dev == iboe->netdevs[port - 1]))) + break; + + return port > MLX4_MAX_PORTS ? 0 : port; +} + +static void mlx4_ib_get_dev_addr(struct net_device *dev, struct mlx4_ib_dev *ibdev, u8 port) +{ + struct ifaddr *ifa; +#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) + struct inet6_dev *in6_dev; + union ib_gid *pgid; + struct inet6_ifaddr *ifp; +#endif + union ib_gid gid; + + + if ((port == 0) || (port > MLX4_MAX_PORTS)) + return; + + /* IPv4 gids */ + TAILQ_FOREACH(ifa, &dev->if_addrhead, ifa_link) { + if (ifa->ifa_addr && ifa->ifa_addr->sa_family == AF_INET){ + ipv6_addr_set_v4mapped( + ((struct sockaddr_in *) ifa->ifa_addr)->sin_addr.s_addr, + (struct in6_addr *)&gid); + update_gid_table(ibdev, port, &gid, 0, 0); + } + + } +#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) + /* IPv6 gids */ + in6_dev = in6_dev_get(dev); + if (in6_dev) { + read_lock_bh(&in6_dev->lock); + list_for_each_entry(ifp, &in6_dev->addr_list, if_list) { + pgid = (union ib_gid *)&ifp->addr; + update_gid_table(ibdev, port, pgid, 0, 0); + } + read_unlock_bh(&in6_dev->lock); + in6_dev_put(in6_dev); + } +#endif +} + +static void mlx4_set_default_gid(struct mlx4_ib_dev *ibdev, + struct net_device *dev, u8 port) +{ + union ib_gid gid; + mlx4_make_default_gid(dev, &gid, port); + update_gid_table(ibdev, port, &gid, 0, 1); +} + +static int mlx4_ib_init_gid_table(struct mlx4_ib_dev *ibdev) +{ + struct net_device *dev; + + if (reset_gid_table(ibdev)) + return -1; + + IFNET_RLOCK_NOSLEEP(); + TAILQ_FOREACH(dev, &V_ifnet, if_link) { + u8 port = mlx4_ib_get_dev_port(dev, ibdev); + if (port) { + if (!rdma_vlan_dev_real_dev(dev) && + !netif_is_bond_master(dev)) + mlx4_set_default_gid(ibdev, dev, port); + mlx4_ib_get_dev_addr(dev, ibdev, port); + } + } + + IFNET_RUNLOCK_NOSLEEP(); + + return 0; +} + +static void mlx4_ib_scan_netdevs(struct mlx4_ib_dev *ibdev, + struct net_device *dev, unsigned long event) +{ + struct mlx4_ib_iboe *iboe; + int port; + int init = 0; + unsigned long flags; + + iboe = &ibdev->iboe; + + spin_lock_irqsave(&iboe->lock, flags); + mlx4_foreach_ib_transport_port(port, ibdev->dev) { + struct net_device *old_netdev = iboe->netdevs[port - 1]; +/* XXX BOND related */ +#if 0 + struct net_device *old_master = iboe->masters[port - 1]; +#endif + iboe->masters[port - 1] = NULL; + iboe->netdevs[port - 1] = + mlx4_get_protocol_dev(ibdev->dev, MLX4_PROT_ETH, port); + + + if (old_netdev != iboe->netdevs[port - 1]) + init = 1; + if (dev == iboe->netdevs[port - 1] && + event == NETDEV_CHANGEADDR) + init = 1; +/* XXX BOND related */ +#if 0 + if (iboe->netdevs[port - 1] && netif_is_bond_slave(iboe->netdevs[port - 1])) + iboe->masters[port - 1] = iboe->netdevs[port - 1]->master; + + /* if bonding is used it is possible that we add it to masters only after + IP address is assigned to the net bonding interface */ + if (old_master != iboe->masters[port - 1]) + init = 1; +#endif + } + + spin_unlock_irqrestore(&iboe->lock, flags); + + if (init) + if (mlx4_ib_init_gid_table(ibdev)) + pr_warn("Fail to reset gid table\n"); +} + +static int mlx4_ib_netdev_event(struct notifier_block *this, unsigned long event, + void *ptr) +{ + struct net_device *dev = ptr; + struct mlx4_ib_dev *ibdev; + + ibdev = container_of(this, struct mlx4_ib_dev, iboe.nb); + + mlx4_ib_scan_netdevs(ibdev, dev, event); + + return NOTIFY_DONE; +} + +/* This function initializes the gid table only if the event_netdev real device is an iboe + * device, will be invoked by the inet/inet6 events */ +static int mlx4_ib_inet_event(struct notifier_block *this, unsigned long event, + void *ptr) +{ + struct net_device *event_netdev = ptr; + struct mlx4_ib_dev *ibdev; + struct mlx4_ib_iboe *ibdev_iboe; + int port = 0; + + ibdev = container_of(this, struct mlx4_ib_dev, iboe.nb_inet); + + struct net_device *real_dev = rdma_vlan_dev_real_dev(event_netdev) ? + rdma_vlan_dev_real_dev(event_netdev) : + event_netdev; + + ibdev_iboe = &ibdev->iboe; + + port = mlx4_ib_get_dev_port(real_dev, ibdev); + + /* Perform init_gid_table if the event real_dev is the net_device which represents this port, + * otherwise this event is not related and would be ignored.*/ + if(port && (real_dev == ibdev_iboe->netdevs[port - 1])) + if (mlx4_ib_init_gid_table(ibdev)) + pr_warn("Fail to reset gid table\n"); + + return NOTIFY_DONE; +} + + +static void init_pkeys(struct mlx4_ib_dev *ibdev) +{ + int port; + int slave; + int i; + + if (mlx4_is_master(ibdev->dev)) { + for (slave = 0; slave <= ibdev->dev->persist->num_vfs; + ++slave) { + for (port = 1; port <= ibdev->dev->caps.num_ports; ++port) { + for (i = 0; + i < ibdev->dev->phys_caps.pkey_phys_table_len[port]; + ++i) { + ibdev->pkeys.virt2phys_pkey[slave][port - 1][i] = + /* master has the identity virt2phys pkey mapping */ + (slave == mlx4_master_func_num(ibdev->dev) || !i) ? i : + ibdev->dev->phys_caps.pkey_phys_table_len[port] - 1; + mlx4_sync_pkey_table(ibdev->dev, slave, port, i, + ibdev->pkeys.virt2phys_pkey[slave][port - 1][i]); + } + } + } + /* initialize pkey cache */ + for (port = 1; port <= ibdev->dev->caps.num_ports; ++port) { + for (i = 0; + i < ibdev->dev->phys_caps.pkey_phys_table_len[port]; + ++i) + ibdev->pkeys.phys_pkey_cache[port-1][i] = + (i) ? 0 : 0xFFFF; + } + } +} + +static void mlx4_ib_alloc_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev) +{ + int i, j, eq = 0, total_eqs = 0; + + ibdev->eq_table = kcalloc(dev->caps.num_comp_vectors, + sizeof(ibdev->eq_table[0]), GFP_KERNEL); + if (!ibdev->eq_table) + return; + + for (i = 1; i <= dev->caps.num_ports; i++) { + for (j = 0; j < mlx4_get_eqs_per_port(dev, i); + j++, total_eqs++) { + if (i > 1 && mlx4_is_eq_shared(dev, total_eqs)) + continue; + ibdev->eq_table[eq] = total_eqs; + if (!mlx4_assign_eq(dev, i, + &ibdev->eq_table[eq])) + eq++; + else + ibdev->eq_table[eq] = -1; + } + } + + for (i = eq; i < dev->caps.num_comp_vectors; + ibdev->eq_table[i++] = -1) + ; + + /* Advertise the new number of EQs to clients */ + ibdev->ib_dev.num_comp_vectors = eq; +} + +static void mlx4_ib_free_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev) +{ + int i; + int total_eqs = ibdev->ib_dev.num_comp_vectors; + + /* no eqs were allocated */ + if (!ibdev->eq_table) + return; + + /* Reset the advertised EQ number */ + ibdev->ib_dev.num_comp_vectors = 0; + + for (i = 0; i < total_eqs; i++) + mlx4_release_eq(dev, ibdev->eq_table[i]); + + kfree(ibdev->eq_table); + ibdev->eq_table = NULL; +} + +/* + * create show function and a device_attribute struct pointing to + * the function for _name + */ +#define DEVICE_DIAG_RPRT_ATTR(_name, _offset, _op_mod) \ +static ssize_t show_rprt_##_name(struct device *dev, \ + struct device_attribute *attr, \ + char *buf){ \ + return show_diag_rprt(dev, buf, _offset, _op_mod); \ +} \ +static DEVICE_ATTR(_name, S_IRUGO, show_rprt_##_name, NULL); + +#define MLX4_DIAG_RPRT_CLEAR_DIAGS 3 + +static size_t show_diag_rprt(struct device *device, char *buf, + u32 offset, u8 op_modifier) +{ + size_t ret; + u32 counter_offset = offset; + u32 diag_counter = 0; + struct mlx4_ib_dev *dev = container_of(device, struct mlx4_ib_dev, + ib_dev.dev); + + ret = mlx4_query_diag_counters(dev->dev, op_modifier, + &counter_offset, &diag_counter, 1, 0); + if (ret) + return ret; + + return sprintf(buf, "%d\n", diag_counter); +} + +static ssize_t clear_diag_counters(struct device *device, + struct device_attribute *attr, + const char *buf, size_t length) +{ + size_t ret; + struct mlx4_ib_dev *dev = container_of(device, struct mlx4_ib_dev, + ib_dev.dev); + + ret = mlx4_query_diag_counters(dev->dev, MLX4_DIAG_RPRT_CLEAR_DIAGS, + NULL, NULL, 0, 0); + if (ret) + return ret; + + return length; +} + +DEVICE_DIAG_RPRT_ATTR(rq_num_lle , 0x00, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_lle , 0x04, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_lqpoe , 0x08, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_lqpoe , 0x0C, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_lpe , 0x18, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_lpe , 0x1C, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_wrfe , 0x20, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_wrfe , 0x24, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_mwbe , 0x2C, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_bre , 0x34, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_lae , 0x38, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_rire , 0x44, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_rire , 0x48, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_rae , 0x4C, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_rae , 0x50, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_roe , 0x54, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_tree , 0x5C, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_rree , 0x64, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_rnr , 0x68, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_rnr , 0x6C, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_oos , 0x100, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_oos , 0x104, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_mce , 0x108, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_udsdprd , 0x118, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_ucsdprd , 0x120, 2); +DEVICE_DIAG_RPRT_ATTR(num_cqovf , 0x1A0, 2); +DEVICE_DIAG_RPRT_ATTR(num_eqovf , 0x1A4, 2); +DEVICE_DIAG_RPRT_ATTR(num_baddb , 0x1A8, 2); + +static DEVICE_ATTR(clear_diag, S_IWUSR, NULL, clear_diag_counters); + +static struct attribute *diag_rprt_attrs[] = { + &dev_attr_rq_num_lle.attr, + &dev_attr_sq_num_lle.attr, + &dev_attr_rq_num_lqpoe.attr, + &dev_attr_sq_num_lqpoe.attr, + &dev_attr_rq_num_lpe.attr, + &dev_attr_sq_num_lpe.attr, + &dev_attr_rq_num_wrfe.attr, + &dev_attr_sq_num_wrfe.attr, + &dev_attr_sq_num_mwbe.attr, + &dev_attr_sq_num_bre.attr, + &dev_attr_rq_num_lae.attr, + &dev_attr_sq_num_rire.attr, + &dev_attr_rq_num_rire.attr, + &dev_attr_sq_num_rae.attr, + &dev_attr_rq_num_rae.attr, + &dev_attr_sq_num_roe.attr, + &dev_attr_sq_num_tree.attr, + &dev_attr_sq_num_rree.attr, + &dev_attr_rq_num_rnr.attr, + &dev_attr_sq_num_rnr.attr, + &dev_attr_rq_num_oos.attr, + &dev_attr_sq_num_oos.attr, + &dev_attr_rq_num_mce.attr, + &dev_attr_rq_num_udsdprd.attr, + &dev_attr_rq_num_ucsdprd.attr, + &dev_attr_num_cqovf.attr, + &dev_attr_num_eqovf.attr, + &dev_attr_num_baddb.attr, + &dev_attr_clear_diag.attr, + NULL +}; + +static struct attribute_group diag_counters_group = { + .name = "diag_counters", + .attrs = diag_rprt_attrs +}; + +static int mlx4_port_immutable(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable) +{ + struct ib_port_attr attr; + struct mlx4_ib_dev *mdev = to_mdev(ibdev); + int err; + + if (mlx4_ib_port_link_layer(ibdev, port_num) == IB_LINK_LAYER_INFINIBAND) { + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB; + immutable->max_mad_size = IB_MGMT_MAD_SIZE; + } else { + if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_IBOE) + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; + if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE | + RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; + immutable->core_cap_flags |= RDMA_CORE_PORT_RAW_PACKET; + if (immutable->core_cap_flags & (RDMA_CORE_PORT_IBA_ROCE | + RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP)) + immutable->max_mad_size = IB_MGMT_MAD_SIZE; + } + + err = ib_query_port(ibdev, port_num, &attr); + if (err) + return err; + + immutable->pkey_tbl_len = attr.pkey_tbl_len; + immutable->gid_tbl_len = attr.gid_tbl_len; + + return 0; +} + +static void *mlx4_ib_add(struct mlx4_dev *dev) +{ + struct mlx4_ib_dev *ibdev; + int num_ports = 0; + int i, j; + int err; + struct mlx4_ib_iboe *iboe; + + pr_info_once("%s", mlx4_ib_version); + + mlx4_foreach_ib_transport_port(i, dev) + num_ports++; + + /* No point in registering a device with no ports... */ + if (num_ports == 0) + return NULL; + + ibdev = (struct mlx4_ib_dev *) ib_alloc_device(sizeof *ibdev); + if (!ibdev) { + dev_err(&dev->persist->pdev->dev, + "Device struct alloc failed\n"); + return NULL; + } + + iboe = &ibdev->iboe; + + if (mlx4_pd_alloc(dev, &ibdev->priv_pdn)) + goto err_dealloc; + + if (mlx4_uar_alloc(dev, &ibdev->priv_uar)) + goto err_pd; + + ibdev->priv_uar.map = ioremap(ibdev->priv_uar.pfn << PAGE_SHIFT, + PAGE_SIZE); + + if (!ibdev->priv_uar.map) + goto err_uar; + + MLX4_INIT_DOORBELL_LOCK(&ibdev->uar_lock); + + ibdev->dev = dev; + + strlcpy(ibdev->ib_dev.name, "mlx4_%d", IB_DEVICE_NAME_MAX); + + ibdev->ib_dev.owner = THIS_MODULE; + ibdev->ib_dev.node_type = RDMA_NODE_IB_CA; + ibdev->ib_dev.local_dma_lkey = dev->caps.reserved_lkey; + ibdev->num_ports = num_ports; + ibdev->ib_dev.phys_port_cnt = ibdev->num_ports; + ibdev->ib_dev.num_comp_vectors = dev->caps.num_comp_vectors; + ibdev->ib_dev.dma_device = &dev->persist->pdev->dev; + + if (dev->caps.userspace_caps) + ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION; + else + ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION; + + ibdev->ib_dev.uverbs_cmd_mask = + (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | + (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | + (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | + (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | + (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | + (1ull << IB_USER_VERBS_CMD_REG_MR) | + (1ull << IB_USER_VERBS_CMD_DEREG_MR) | + (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | + (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | + (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | + (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | + (1ull << IB_USER_VERBS_CMD_CREATE_QP) | + (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | + (1ull << IB_USER_VERBS_CMD_QUERY_QP) | + (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | + (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) | + (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) | + (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | + (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | + (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | + (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | + (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ) | + (1ull << IB_USER_VERBS_CMD_OPEN_QP); + + ibdev->ib_dev.query_device = mlx4_ib_query_device; + ibdev->ib_dev.query_port = mlx4_ib_query_port; + ibdev->ib_dev.get_link_layer = mlx4_ib_port_link_layer; + ibdev->ib_dev.query_gid = mlx4_ib_query_gid; + ibdev->ib_dev.query_pkey = mlx4_ib_query_pkey; + ibdev->ib_dev.modify_device = mlx4_ib_modify_device; + ibdev->ib_dev.modify_port = mlx4_ib_modify_port; + ibdev->ib_dev.alloc_ucontext = mlx4_ib_alloc_ucontext; + ibdev->ib_dev.dealloc_ucontext = mlx4_ib_dealloc_ucontext; + ibdev->ib_dev.mmap = mlx4_ib_mmap; +/* XXX FBSD has no support for get_unmapped_area function */ +#if 0 + ibdev->ib_dev.get_unmapped_area = mlx4_ib_get_unmapped_area; +#endif + ibdev->ib_dev.alloc_pd = mlx4_ib_alloc_pd; + ibdev->ib_dev.dealloc_pd = mlx4_ib_dealloc_pd; + ibdev->ib_dev.create_ah = mlx4_ib_create_ah; + ibdev->ib_dev.query_ah = mlx4_ib_query_ah; + ibdev->ib_dev.destroy_ah = mlx4_ib_destroy_ah; + ibdev->ib_dev.create_srq = mlx4_ib_create_srq; + ibdev->ib_dev.modify_srq = mlx4_ib_modify_srq; + ibdev->ib_dev.query_srq = mlx4_ib_query_srq; + ibdev->ib_dev.destroy_srq = mlx4_ib_destroy_srq; + ibdev->ib_dev.post_srq_recv = mlx4_ib_post_srq_recv; + ibdev->ib_dev.create_qp = mlx4_ib_create_qp; + ibdev->ib_dev.modify_qp = mlx4_ib_modify_qp; + ibdev->ib_dev.query_qp = mlx4_ib_query_qp; + ibdev->ib_dev.destroy_qp = mlx4_ib_destroy_qp; + ibdev->ib_dev.post_send = mlx4_ib_post_send; + ibdev->ib_dev.post_recv = mlx4_ib_post_recv; + ibdev->ib_dev.create_cq = mlx4_ib_create_cq; + ibdev->ib_dev.modify_cq = mlx4_ib_modify_cq; + ibdev->ib_dev.resize_cq = mlx4_ib_resize_cq; + ibdev->ib_dev.destroy_cq = mlx4_ib_destroy_cq; + ibdev->ib_dev.poll_cq = mlx4_ib_poll_cq; + ibdev->ib_dev.req_notify_cq = mlx4_ib_arm_cq; + ibdev->ib_dev.get_dma_mr = mlx4_ib_get_dma_mr; + ibdev->ib_dev.reg_user_mr = mlx4_ib_reg_user_mr; + ibdev->ib_dev.dereg_mr = mlx4_ib_dereg_mr; + ibdev->ib_dev.alloc_fast_reg_mr = mlx4_ib_alloc_fast_reg_mr; + ibdev->ib_dev.alloc_fast_reg_page_list = mlx4_ib_alloc_fast_reg_page_list; + ibdev->ib_dev.free_fast_reg_page_list = mlx4_ib_free_fast_reg_page_list; + ibdev->ib_dev.attach_mcast = mlx4_ib_mcg_attach; + ibdev->ib_dev.detach_mcast = mlx4_ib_mcg_detach; + ibdev->ib_dev.process_mad = mlx4_ib_process_mad; + ibdev->ib_dev.get_port_immutable = mlx4_port_immutable; + ibdev->ib_dev.get_netdev = mlx4_ib_get_netdev; + ibdev->ib_dev.ioctl = mlx4_ib_ioctl; + ibdev->ib_dev.query_values = mlx4_ib_query_values; + + if (!mlx4_is_slave(ibdev->dev)) { + ibdev->ib_dev.alloc_fmr = mlx4_ib_fmr_alloc; + ibdev->ib_dev.map_phys_fmr = mlx4_ib_map_phys_fmr; + ibdev->ib_dev.unmap_fmr = mlx4_ib_unmap_fmr; + ibdev->ib_dev.dealloc_fmr = mlx4_ib_fmr_dealloc; + } + + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW) { + ibdev->ib_dev.alloc_mw = mlx4_ib_alloc_mw; + ibdev->ib_dev.bind_mw = mlx4_ib_bind_mw; + ibdev->ib_dev.dealloc_mw = mlx4_ib_dealloc_mw; + + ibdev->ib_dev.uverbs_cmd_mask |= + (1ull << IB_USER_VERBS_CMD_ALLOC_MW) | + (1ull << IB_USER_VERBS_CMD_DEALLOC_MW); + } + + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) { + ibdev->ib_dev.alloc_xrcd = mlx4_ib_alloc_xrcd; + ibdev->ib_dev.dealloc_xrcd = mlx4_ib_dealloc_xrcd; + ibdev->ib_dev.uverbs_cmd_mask |= + (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) | + (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD); + } + + /* + * Set experimental data + */ + ibdev->ib_dev.uverbs_exp_cmd_mask = + (1ull << IB_USER_VERBS_EXP_CMD_CREATE_QP) | + (1ull << IB_USER_VERBS_EXP_CMD_MODIFY_CQ) | + (1ull << IB_USER_VERBS_EXP_CMD_QUERY_DEVICE) | + (1ull << IB_USER_VERBS_EXP_CMD_CREATE_CQ); + ibdev->ib_dev.exp_create_qp = mlx4_ib_exp_create_qp; + ibdev->ib_dev.exp_query_device = mlx4_ib_exp_query_device; + if (check_flow_steering_support(dev)) { + ibdev->ib_dev.uverbs_ex_cmd_mask |= + (1ull << IB_USER_VERBS_EX_CMD_CREATE_FLOW) | + (1ull << IB_USER_VERBS_EX_CMD_DESTROY_FLOW); + ibdev->ib_dev.create_flow = mlx4_ib_create_flow; + ibdev->ib_dev.destroy_flow = mlx4_ib_destroy_flow; + } else { + pr_debug("Device managed flow steering is unavailable for this configuration.\n"); + } + /* + * End of experimental data + */ + + mlx4_ib_alloc_eqs(dev, ibdev); + + spin_lock_init(&iboe->lock); + + if (init_node_data(ibdev)) + goto err_map; + + for (i = 0; i < ibdev->num_ports; ++i) { + if (mlx4_ib_port_link_layer(&ibdev->ib_dev, i + 1) == + IB_LINK_LAYER_ETHERNET) { + if (mlx4_is_slave(dev)) { + ibdev->counters[i].status = mlx4_counter_alloc(ibdev->dev, + &ibdev->counters[i].counter_index); + if (ibdev->counters[i].status) + ibdev->counters[i].counter_index = mlx4_get_default_counter_index(dev, + i + 1); + } else {/* allocating the PF IB default counter indices reserved in mlx4_init_counters_table */ + ibdev->counters[i].counter_index = ((i + 1) << 1) - 1; + ibdev->counters[i].status = 0; + } + + dev_info(&dev->persist->pdev->dev, + "%s: allocated counter index %d for port %d\n", + __func__, ibdev->counters[i].counter_index, i+1); + } else { + ibdev->counters[i].counter_index = MLX4_SINK_COUNTER_INDEX(dev); + ibdev->counters[i].status = -ENOSPC; + } + } + + spin_lock_init(&ibdev->sm_lock); + mutex_init(&ibdev->cap_mask_mutex); + + if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED && + !mlx4_is_mfunc(dev)) { + ibdev->steer_qpn_count = MLX4_IB_UC_MAX_NUM_QPS; + err = mlx4_qp_reserve_range(dev, ibdev->steer_qpn_count, + MLX4_IB_UC_STEER_QPN_ALIGN, &ibdev->steer_qpn_base, 0); + if (err) + goto err_counter; + + ibdev->ib_uc_qpns_bitmap = + kmalloc(BITS_TO_LONGS(ibdev->steer_qpn_count) * + sizeof(long), + GFP_KERNEL); + if (!ibdev->ib_uc_qpns_bitmap) { + dev_err(&dev->persist->pdev->dev, + "bit map alloc failed\n"); + goto err_steer_qp_release; + } + + bitmap_zero(ibdev->ib_uc_qpns_bitmap, ibdev->steer_qpn_count); + + err = mlx4_FLOW_STEERING_IB_UC_QP_RANGE(dev, ibdev->steer_qpn_base, + ibdev->steer_qpn_base + ibdev->steer_qpn_count - 1); + if (err) + goto err_steer_free_bitmap; + } + + if (ib_register_device(&ibdev->ib_dev, NULL)) + goto err_steer_free_bitmap; + + if (mlx4_ib_mad_init(ibdev)) + goto err_reg; + + if (mlx4_ib_init_sriov(ibdev)) + goto err_mad; + + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_IBOE) { + if (!iboe->nb.notifier_call) { + iboe->nb.notifier_call = mlx4_ib_netdev_event; + err = register_netdevice_notifier(&iboe->nb); + if (err) { + iboe->nb.notifier_call = NULL; + goto err_notify; + } + } + if (!iboe->nb_inet.notifier_call) { + iboe->nb_inet.notifier_call = mlx4_ib_inet_event; + err = register_inetaddr_notifier(&iboe->nb_inet); + if (err) { + iboe->nb_inet.notifier_call = NULL; + goto err_notify; + } + } + mlx4_ib_scan_netdevs(ibdev, NULL, 0); + } + for (j = 0; j < ARRAY_SIZE(mlx4_class_attributes); ++j) { + if (device_create_file(&ibdev->ib_dev.dev, + mlx4_class_attributes[j])) + goto err_notify; + } + if (sysfs_create_group(&ibdev->ib_dev.dev.kobj, &diag_counters_group)) + goto err_notify; + + ibdev->ib_active = true; + + if (mlx4_is_mfunc(ibdev->dev)) + init_pkeys(ibdev); + + /* create paravirt contexts for any VFs which are active */ + if (mlx4_is_master(ibdev->dev)) { + for (j = 0; j < MLX4_MFUNC_MAX; j++) { + if (j == mlx4_master_func_num(ibdev->dev)) + continue; + if (mlx4_is_slave_active(ibdev->dev, j)) + do_slave_init(ibdev, j, 1); + } + } + return ibdev; + +err_notify: + for (j = 0; j < ARRAY_SIZE(mlx4_class_attributes); ++j) { + device_remove_file(&ibdev->ib_dev.dev, + mlx4_class_attributes[j]); + } + + if (ibdev->iboe.nb.notifier_call) { + if (unregister_netdevice_notifier(&ibdev->iboe.nb)) + pr_warn("failure unregistering notifier\n"); + ibdev->iboe.nb.notifier_call = NULL; + } + if (ibdev->iboe.nb_inet.notifier_call) { + if (unregister_inetaddr_notifier(&ibdev->iboe.nb_inet)) + pr_warn("failure unregistering notifier\n"); + ibdev->iboe.nb_inet.notifier_call = NULL; + } + flush_workqueue(wq); + + mlx4_ib_close_sriov(ibdev); + +err_mad: + mlx4_ib_mad_cleanup(ibdev); + +err_reg: + ib_unregister_device(&ibdev->ib_dev); + +err_steer_free_bitmap: + kfree(ibdev->ib_uc_qpns_bitmap); + +err_steer_qp_release: + if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) + mlx4_qp_release_range(dev, ibdev->steer_qpn_base, + ibdev->steer_qpn_count); +err_counter: + for (; i; --i) { + if (mlx4_ib_port_link_layer(&ibdev->ib_dev, i) == + IB_LINK_LAYER_ETHERNET) { + mlx4_counter_free(ibdev->dev, + ibdev->counters[i - 1].counter_index); + } + } + +err_map: + iounmap(ibdev->priv_uar.map); + mlx4_ib_free_eqs(dev, ibdev); + +err_uar: + mlx4_uar_free(dev, &ibdev->priv_uar); + +err_pd: + mlx4_pd_free(dev, ibdev->priv_pdn); + +err_dealloc: + ib_dealloc_device(&ibdev->ib_dev); + + return NULL; +} + +int mlx4_ib_steer_qp_alloc(struct mlx4_ib_dev *dev, int count, int *qpn) +{ + int offset; + + WARN_ON(!dev->ib_uc_qpns_bitmap); + + offset = bitmap_find_free_region(dev->ib_uc_qpns_bitmap, + dev->steer_qpn_count, + get_count_order(count)); + if (offset < 0) + return offset; + + *qpn = dev->steer_qpn_base + offset; + return 0; +} + +void mlx4_ib_steer_qp_free(struct mlx4_ib_dev *dev, u32 qpn, int count) +{ + if (!qpn || + dev->dev->caps.steering_mode != MLX4_STEERING_MODE_DEVICE_MANAGED) + return; + + BUG_ON(qpn < dev->steer_qpn_base); + + bitmap_release_region(dev->ib_uc_qpns_bitmap, + qpn - dev->steer_qpn_base, get_count_order(count)); +} + +int mlx4_ib_steer_qp_reg(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp, + int is_attach) +{ + int err; + size_t flow_size; + struct ib_flow_attr *flow = NULL; + struct ib_flow_spec_ib *ib_spec; + + if (is_attach) { + flow_size = sizeof(struct ib_flow_attr) + + sizeof(struct ib_flow_spec_ib); + flow = kzalloc(flow_size, GFP_KERNEL); + if (!flow) + return -ENOMEM; + flow->port = mqp->port; + flow->num_of_specs = 1; + flow->size = flow_size; + ib_spec = (struct ib_flow_spec_ib *)(flow + 1); + ib_spec->type = IB_FLOW_SPEC_IB; + ib_spec->size = sizeof(struct ib_flow_spec_ib); + ib_spec->val.l3_type_qpn = mqp->ibqp.qp_num; + ib_spec->mask.l3_type_qpn = MLX4_IB_FLOW_QPN_MASK; + + err = __mlx4_ib_create_flow(&mqp->ibqp, flow, + IB_FLOW_DOMAIN_NIC, + MLX4_FS_REGULAR, + &mqp->reg_id); + } else { + err = __mlx4_ib_destroy_flow(mdev->dev, mqp->reg_id); + } + kfree(flow); + return err; +} + +static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr) +{ + struct mlx4_ib_dev *ibdev = ibdev_ptr; + int p, j; + + if (ibdev->iboe.nb_inet.notifier_call) { + if (unregister_inetaddr_notifier(&ibdev->iboe.nb_inet)) + pr_warn("failure unregistering notifier\n"); + ibdev->iboe.nb_inet.notifier_call = NULL; + } + + mlx4_ib_close_sriov(ibdev); + sysfs_remove_group(&ibdev->ib_dev.dev.kobj, &diag_counters_group); + mlx4_ib_mad_cleanup(ibdev); + + for (j = 0; j < ARRAY_SIZE(mlx4_class_attributes); ++j) { + device_remove_file(&ibdev->ib_dev.dev, + mlx4_class_attributes[j]); + } + + ib_unregister_device(&ibdev->ib_dev); + + if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) { + mlx4_qp_release_range(dev, ibdev->steer_qpn_base, + ibdev->steer_qpn_count); + kfree(ibdev->ib_uc_qpns_bitmap); + } + + if (ibdev->iboe.nb.notifier_call) { + if (unregister_netdevice_notifier(&ibdev->iboe.nb)) + pr_warn("failure unregistering notifier\n"); + ibdev->iboe.nb.notifier_call = NULL; + } + iounmap(ibdev->priv_uar.map); + + for (p = 0; p < ibdev->num_ports; ++p) { + if (mlx4_ib_port_link_layer(&ibdev->ib_dev, p + 1) == + IB_LINK_LAYER_ETHERNET) { + mlx4_counter_free(ibdev->dev, + ibdev->counters[p].counter_index); + } + } + + mlx4_foreach_port(p, dev, MLX4_PORT_TYPE_IB) + mlx4_CLOSE_PORT(dev, p); + + mlx4_ib_free_eqs(dev, ibdev); + + mlx4_uar_free(dev, &ibdev->priv_uar); + mlx4_pd_free(dev, ibdev->priv_pdn); + ib_dealloc_device(&ibdev->ib_dev); +} + +static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init) +{ + struct mlx4_ib_demux_work **dm = NULL; + struct mlx4_dev *dev = ibdev->dev; + int i; + unsigned long flags; + + if (!mlx4_is_master(dev)) + return; + + dm = kcalloc(dev->caps.num_ports, sizeof *dm, GFP_ATOMIC); + if (!dm) { + pr_err("failed to allocate memory for tunneling qp update\n"); + goto out; + } + + for (i = 0; i < dev->caps.num_ports; i++) { + dm[i] = kmalloc(sizeof (struct mlx4_ib_demux_work), GFP_ATOMIC); + if (!dm[i]) { + pr_err("failed to allocate memory for tunneling qp update work struct\n"); + for (i = 0; i < dev->caps.num_ports; i++) { + if (dm[i]) + kfree(dm[i]); + } + goto out; + } + } + /* initialize or tear down tunnel QPs for the slave */ + for (i = 0; i < dev->caps.num_ports; i++) { + INIT_WORK(&dm[i]->work, mlx4_ib_tunnels_update_work); + dm[i]->port = i + 1; + dm[i]->slave = slave; + dm[i]->do_init = do_init; + dm[i]->dev = ibdev; + spin_lock_irqsave(&ibdev->sriov.going_down_lock, flags); + if (!ibdev->sriov.is_going_down) + queue_work(ibdev->sriov.demux[i].ud_wq, &dm[i]->work); + spin_unlock_irqrestore(&ibdev->sriov.going_down_lock, flags); + } +out: + if (dm) + kfree(dm); + return; +} + +static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr, + enum mlx4_dev_event event, unsigned long param) +{ + struct ib_event ibev; + struct mlx4_ib_dev *ibdev = to_mdev((struct ib_device *) ibdev_ptr); + struct mlx4_eqe *eqe = NULL; + struct ib_event_work *ew; + int p = 0; + + if (event == MLX4_DEV_EVENT_PORT_MGMT_CHANGE) + eqe = (struct mlx4_eqe *)param; + else + p = (int) param; + + switch (event) { + case MLX4_DEV_EVENT_PORT_UP: + if (p > ibdev->num_ports) + return; + if (mlx4_is_master(dev) && + rdma_port_get_link_layer(&ibdev->ib_dev, p) == + IB_LINK_LAYER_INFINIBAND) { + mlx4_ib_invalidate_all_guid_record(ibdev, p); + } + mlx4_ib_info((struct ib_device *) ibdev_ptr, + "Port %d logical link is up\n", p); + ibev.event = IB_EVENT_PORT_ACTIVE; + break; + + case MLX4_DEV_EVENT_PORT_DOWN: + if (p > ibdev->num_ports) + return; + mlx4_ib_info((struct ib_device *) ibdev_ptr, + "Port %d logical link is down\n", p); + ibev.event = IB_EVENT_PORT_ERR; + break; + + case MLX4_DEV_EVENT_CATASTROPHIC_ERROR: + ibdev->ib_active = false; + ibev.event = IB_EVENT_DEVICE_FATAL; + break; + + case MLX4_DEV_EVENT_PORT_MGMT_CHANGE: + ew = kmalloc(sizeof *ew, GFP_ATOMIC); + if (!ew) { + pr_err("failed to allocate memory for events work\n"); + break; + } + + INIT_WORK(&ew->work, handle_port_mgmt_change_event); + memcpy(&ew->ib_eqe, eqe, sizeof *eqe); + ew->ib_dev = ibdev; + /* need to queue only for port owner, which uses GEN_EQE */ + if (mlx4_is_master(dev)) + queue_work(wq, &ew->work); + else + handle_port_mgmt_change_event(&ew->work); + return; + + case MLX4_DEV_EVENT_SLAVE_INIT: + /* here, p is the slave id */ + do_slave_init(ibdev, p, 1); + return; + + case MLX4_DEV_EVENT_SLAVE_SHUTDOWN: + /* here, p is the slave id */ + do_slave_init(ibdev, p, 0); + return; + + default: + return; + } + + ibev.device = ibdev_ptr; + ibev.element.port_num = (u8) p; + + ib_dispatch_event(&ibev); +} + +static struct mlx4_interface mlx4_ib_interface = { + .add = mlx4_ib_add, + .remove = mlx4_ib_remove, + .event = mlx4_ib_event, + .protocol = MLX4_PROT_IB_IPV6 +}; + +static int __init mlx4_ib_init(void) +{ + int err; + + wq = create_singlethread_workqueue("mlx4_ib"); + if (!wq) + return -ENOMEM; + + err = mlx4_ib_mcg_init(); + if (err) + goto clean_proc; + + err = mlx4_register_interface(&mlx4_ib_interface); + if (err) + goto clean_mcg; + + return 0; + +clean_mcg: + mlx4_ib_mcg_destroy(); + +clean_proc: + destroy_workqueue(wq); + return err; +} + +static void __exit mlx4_ib_cleanup(void) +{ + mlx4_unregister_interface(&mlx4_ib_interface); + mlx4_ib_mcg_destroy(); + destroy_workqueue(wq); +} + +module_init_order(mlx4_ib_init, SI_ORDER_MIDDLE); +module_exit(mlx4_ib_cleanup); + +static int +mlx4ib_evhand(module_t mod, int event, void *arg) +{ + return (0); +} + +static moduledata_t mlx4ib_mod = { + .name = "mlx4ib", + .evhand = mlx4ib_evhand, +}; + +DECLARE_MODULE(mlx4ib, mlx4ib_mod, SI_SUB_LAST, SI_ORDER_ANY); +MODULE_DEPEND(mlx4ib, mlx4, 1, 1, 1); +MODULE_DEPEND(mlx4ib, ibcore, 1, 1, 1); +MODULE_DEPEND(mlx4ib, linuxkpi, 1, 1, 1); Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_main.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_mr.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_mr.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_mr.c (revision 329159) @@ -0,0 +1,885 @@ +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include + +#include "mlx4_ib.h" + +static u32 convert_access(int acc) +{ + return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX4_PERM_ATOMIC : 0) | + (acc & IB_ACCESS_REMOTE_WRITE ? MLX4_PERM_REMOTE_WRITE : 0) | + (acc & IB_ACCESS_REMOTE_READ ? MLX4_PERM_REMOTE_READ : 0) | + (acc & IB_ACCESS_LOCAL_WRITE ? MLX4_PERM_LOCAL_WRITE : 0) | + (acc & IB_ACCESS_MW_BIND ? MLX4_PERM_BIND_MW : 0) | + MLX4_PERM_LOCAL_READ; +} +/* No suuport for Shared MR feature */ +#if 0 +static ssize_t shared_mr_proc_read(struct file *file, + char __user *buffer, + size_t len, + loff_t *offset) +{ + + return -ENOSYS; + +} + +static ssize_t shared_mr_proc_write(struct file *file, + const char __user *buffer, + size_t len, + loff_t *offset) +{ + + return -ENOSYS; +} + +static int shared_mr_mmap(struct file *filep, struct vm_area_struct *vma) +{ + + struct proc_dir_entry *pde = PDE(filep->f_path.dentry->d_inode); + struct mlx4_shared_mr_info *smr_info = + (struct mlx4_shared_mr_info *)pde->data; + + /* Prevent any mapping not on start of area */ + if (vma->vm_pgoff != 0) + return -EINVAL; + + return ib_umem_map_to_vma(smr_info->umem, + vma); + +} + +static const struct file_operations shared_mr_proc_ops = { + .owner = THIS_MODULE, + .read = shared_mr_proc_read, + .write = shared_mr_proc_write, + .mmap = shared_mr_mmap +}; + +static mode_t convert_shared_access(int acc) +{ + + return (acc & IB_ACCESS_SHARED_MR_USER_READ ? S_IRUSR : 0) | + (acc & IB_ACCESS_SHARED_MR_USER_WRITE ? S_IWUSR : 0) | + (acc & IB_ACCESS_SHARED_MR_GROUP_READ ? S_IRGRP : 0) | + (acc & IB_ACCESS_SHARED_MR_GROUP_WRITE ? S_IWGRP : 0) | + (acc & IB_ACCESS_SHARED_MR_OTHER_READ ? S_IROTH : 0) | + (acc & IB_ACCESS_SHARED_MR_OTHER_WRITE ? S_IWOTH : 0); + +} +#endif +struct ib_mr *mlx4_ib_get_dma_mr(struct ib_pd *pd, int acc) +{ + struct mlx4_ib_mr *mr; + int err; + + mr = kzalloc(sizeof *mr, GFP_KERNEL); + if (!mr) + return ERR_PTR(-ENOMEM); + + err = mlx4_mr_alloc(to_mdev(pd->device)->dev, to_mpd(pd)->pdn, 0, + ~0ull, convert_access(acc), 0, 0, &mr->mmr); + if (err) + goto err_free; + + err = mlx4_mr_enable(to_mdev(pd->device)->dev, &mr->mmr); + if (err) + goto err_mr; + + mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; + mr->umem = NULL; + + return &mr->ibmr; + +err_mr: + (void) mlx4_mr_free(to_mdev(pd->device)->dev, &mr->mmr); + +err_free: + kfree(mr); + + return ERR_PTR(err); +} + +static int mlx4_ib_umem_write_mtt_block(struct mlx4_ib_dev *dev, + struct mlx4_mtt *mtt, + u64 mtt_size, + u64 mtt_shift, + u64 len, + u64 cur_start_addr, + u64 *pages, + int *start_index, + int *npages) +{ + int k; + int err = 0; + u64 mtt_entries; + u64 cur_end_addr = cur_start_addr + len; + u64 cur_end_addr_aligned = 0; + + len += (cur_start_addr & (mtt_size-1ULL)); + cur_end_addr_aligned = round_up(cur_end_addr, mtt_size); + len += (cur_end_addr_aligned - cur_end_addr); + if (len & (mtt_size-1ULL)) { + WARN(1 , + "write_block: len %llx is not aligned to mtt_size %llx\n", + (unsigned long long)len, (unsigned long long)mtt_size); + return -EINVAL; + } + + + mtt_entries = (len >> mtt_shift); + + /* Align the MTT start address to + the mtt_size. + Required to handle cases when the MR + starts in the middle of an MTT record. + Was not required in old code since + the physical addresses provided by + the dma subsystem were page aligned, + which was also the MTT size. + */ + cur_start_addr = round_down(cur_start_addr, mtt_size); + /* A new block is started ...*/ + for (k = 0; k < mtt_entries; ++k) { + pages[*npages] = cur_start_addr + (mtt_size * k); + (*npages)++; + /* + * Be friendly to mlx4_write_mtt() and + * pass it chunks of appropriate size. + */ + if (*npages == PAGE_SIZE / sizeof(u64)) { + err = mlx4_write_mtt(dev->dev, + mtt, *start_index, + *npages, pages); + if (err) + return err; + + (*start_index) += *npages; + *npages = 0; + } + } + + return 0; +} + +int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt, + struct ib_umem *umem) +{ + u64 *pages; + u64 len = 0; + int err = 0; + u64 mtt_size; + u64 cur_start_addr = 0; + u64 mtt_shift; + int start_index = 0; + int npages = 0; + struct scatterlist *sg; + int i; + + pages = (u64 *) __get_free_page(GFP_KERNEL); + if (!pages) + return -ENOMEM; + + mtt_shift = mtt->page_shift; + mtt_size = 1ULL << mtt_shift; + + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { + if (cur_start_addr + len == + sg_dma_address(sg)) { + /* still the same block */ + len += sg_dma_len(sg); + continue; + } + /* A new block is started ...*/ + /* If len is malaligned, write an extra mtt entry to + cover the misaligned area (round up the division) + */ + err = mlx4_ib_umem_write_mtt_block(dev, + mtt, mtt_size, mtt_shift, + len, cur_start_addr, + pages, + &start_index, + &npages); + if (err) + goto out; + + cur_start_addr = + sg_dma_address(sg); + len = sg_dma_len(sg); + } + + /* Handle the last block */ + if (len > 0) { + /* If len is malaligned, write an extra mtt entry to cover + the misaligned area (round up the division) + */ + err = mlx4_ib_umem_write_mtt_block(dev, + mtt, mtt_size, mtt_shift, + len, cur_start_addr, + pages, + &start_index, + &npages); + if (err) + goto out; + } + + + if (npages) + err = mlx4_write_mtt(dev->dev, mtt, start_index, npages, pages); + +out: + free_page((unsigned long) pages); + return err; +} + +static inline u64 alignment_of(u64 ptr) +{ + return ilog2(ptr & (~(ptr-1))); +} + +static int mlx4_ib_umem_calc_block_mtt(u64 next_block_start, + u64 current_block_end, + u64 block_shift) +{ + /* Check whether the alignment of the new block + is aligned as well as the previous block. + Block address must start with zeros till size of entity_size. + */ + if ((next_block_start & ((1ULL << block_shift) - 1ULL)) != 0) + /* It is not as well aligned as the + previous block-reduce the mtt size + accordingly. + Here we take the last right bit + which is 1. + */ + block_shift = alignment_of(next_block_start); + + /* Check whether the alignment of the + end of previous block - is it aligned + as well as the start of the block + */ + if (((current_block_end) & ((1ULL << block_shift) - 1ULL)) != 0) + /* It is not as well aligned as + the start of the block - reduce the + mtt size accordingly. + */ + block_shift = alignment_of(current_block_end); + + return block_shift; +} + +/* Calculate optimal mtt size based on contiguous pages. +* Function will return also the number of pages that are not aligned to the + calculated mtt_size to be added to total number + of pages. For that we should check the first chunk length & last chunk + length and if not aligned to mtt_size we should increment + the non_aligned_pages number. + All chunks in the middle already handled as part of mtt shift calculation + for both their start & end addresses. +*/ +int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, + u64 start_va, + int *num_of_mtts) +{ + u64 block_shift = 31; + u64 current_block_len = 0; + u64 current_block_start = 0; + u64 misalignment_bits; + u64 first_block_start = 0; + u64 last_block_end = 0; + u64 total_len = 0; + u64 last_block_aligned_end = 0; + u64 min_shift = ilog2(umem->page_size); + struct scatterlist *sg; + int i; + u64 next_block_start; + u64 current_block_end; + + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { + /* Initialization - save the first chunk start as + the current_block_start - block means contiguous pages. + */ + if (current_block_len == 0 && current_block_start == 0) { + first_block_start = current_block_start = + sg_dma_address(sg); + /* Find the bits that are different between + the physical address and the virtual + address for the start of the MR. + */ + /* umem_get aligned the start_va to a page + boundary. Therefore, we need to align the + start va to the same boundary */ + /* misalignment_bits is needed to handle the + case of a single memory region. In this + case, the rest of the logic will not reduce + the block size. If we use a block size + which is bigger than the alignment of the + misalignment bits, we might use the virtual + page number instead of the physical page + number, resulting in access to the wrong + data. */ + misalignment_bits = + (start_va & (~(((u64)(umem->page_size))-1ULL))) + ^ current_block_start; + block_shift = min(alignment_of(misalignment_bits) + , block_shift); + } + + /* Go over the scatter entries and check + if they continue the previous scatter entry. + */ + next_block_start = + sg_dma_address(sg); + current_block_end = current_block_start + + current_block_len; + /* If we have a split (non-contig.) between two block*/ + if (current_block_end != next_block_start) { + block_shift = mlx4_ib_umem_calc_block_mtt( + next_block_start, + current_block_end, + block_shift); + + /* If we reached the minimum shift for 4k + page we stop the loop. + */ + if (block_shift <= min_shift) + goto end; + + /* If not saved yet we are in first block - + we save the length of first block to + calculate the non_aligned_pages number at + * the end. + */ + total_len += current_block_len; + + /* Start a new block */ + current_block_start = next_block_start; + current_block_len = + sg_dma_len(sg); + continue; + } + /* The scatter entry is another part of + the current block, increase the block size + * An entry in the scatter can be larger than + 4k (page) as of dma mapping + which merge some blocks together. + */ + current_block_len += + sg_dma_len(sg); + } + + /* Account for the last block in the total len */ + total_len += current_block_len; + /* Add to the first block the misalignment that it suffers from.*/ + total_len += (first_block_start & ((1ULL<> block_shift; +end: + if (block_shift < min_shift) { + /* If shift is less than the min we set a WARN and + return the min shift. + */ + WARN(1, + "mlx4_ib_umem_calc_optimal_mtt_size - unexpected shift %lld\n", + (unsigned long long)block_shift); + + block_shift = min_shift; + } + return block_shift; + +} + +/* No suuport for Shared MR */ +#if 0 +static int prepare_shared_mr(struct mlx4_ib_mr *mr, int access_flags, int mr_id) +{ + + struct proc_dir_entry *mr_proc_entry; + mode_t mode = S_IFREG; + char name_buff[16]; + + mode |= convert_shared_access(access_flags); + sprintf(name_buff, "%X", mr_id); + mr->smr_info = kmalloc(sizeof(struct mlx4_shared_mr_info), GFP_KERNEL); + mr->smr_info->mr_id = mr_id; + mr->smr_info->umem = mr->umem; + + mr_proc_entry = proc_create_data(name_buff, mode, + mlx4_mrs_dir_entry, + &shared_mr_proc_ops, + mr->smr_info); + + if (!mr_proc_entry) { + pr_err("prepare_shared_mr failed via proc\n"); + kfree(mr->smr_info); + return -ENODEV; + } + + current_uid_gid(&(mr_proc_entry->uid), &(mr_proc_entry->gid)); + mr_proc_entry->size = mr->umem->length; + return 0; + +} +static int is_shared_mr(int access_flags) +{ + /* We should check whether IB_ACCESS_SHARED_MR_USER_READ or + other shared bits were turned on. + */ + return !!(access_flags & (IB_ACCESS_SHARED_MR_USER_READ | + IB_ACCESS_SHARED_MR_USER_WRITE | + IB_ACCESS_SHARED_MR_GROUP_READ | + IB_ACCESS_SHARED_MR_GROUP_WRITE | + IB_ACCESS_SHARED_MR_OTHER_READ | + IB_ACCESS_SHARED_MR_OTHER_WRITE)); + +} + +static void free_smr_info(struct mlx4_ib_mr *mr) +{ + /* When master/parent shared mr is dereged there is + no ability to share this mr any more - its mr_id will be + returned to the kernel as part of ib_uverbs_dereg_mr + and may be allocated again as part of other reg_mr. + */ + char name_buff[16]; + + sprintf(name_buff, "%X", mr->smr_info->mr_id); + /* Remove proc entry is checking internally that no operation + was strated on that proc fs file and if in the middle + current process will wait till end of operation. + That's why no sync mechanism is needed when we release + below the shared umem. + */ + remove_proc_entry(name_buff, mlx4_mrs_dir_entry); + kfree(mr->smr_info); + mr->smr_info = NULL; +} +#endif + +static void mlx4_invalidate_umem(void *invalidation_cookie, + struct ib_umem *umem, + unsigned long addr, size_t size) +{ + struct mlx4_ib_mr *mr = (struct mlx4_ib_mr *)invalidation_cookie; + + /* This function is called under client peer lock so its resources are race protected */ + if (atomic_inc_return(&mr->invalidated) > 1) { + umem->invalidation_ctx->inflight_invalidation = 1; + goto end; + } + + umem->invalidation_ctx->peer_callback = 1; + mlx4_mr_free(to_mdev(mr->ibmr.device)->dev, &mr->mmr); + ib_umem_release(umem); + complete(&mr->invalidation_comp); + +end: + return; + +} + +struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, + u64 virt_addr, int access_flags, + struct ib_udata *udata, + int mr_id) +{ + struct mlx4_ib_dev *dev = to_mdev(pd->device); + struct mlx4_ib_mr *mr; + int shift; + int err; + int n; + struct ib_peer_memory_client *ib_peer_mem; + + mr = kzalloc(sizeof *mr, GFP_KERNEL); + if (!mr) + return ERR_PTR(-ENOMEM); + + mr->umem = ib_umem_get_ex(pd->uobject->context, start, length, + access_flags, 0, 1); + if (IS_ERR(mr->umem)) { + err = PTR_ERR(mr->umem); + goto err_free; + } + + ib_peer_mem = mr->umem->ib_peer_mem; + n = ib_umem_page_count(mr->umem); + shift = mlx4_ib_umem_calc_optimal_mtt_size(mr->umem, start, + &n); + err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, virt_addr, length, + convert_access(access_flags), n, shift, &mr->mmr); + if (err) + goto err_umem; + + err = mlx4_ib_umem_write_mtt(dev, &mr->mmr.mtt, mr->umem); + if (err) + goto err_mr; + + err = mlx4_mr_enable(dev->dev, &mr->mmr); + if (err) + goto err_mr; + + mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; +/* No suuport for Shared MR */ +#if 0 + /* Check whether MR should be shared */ + if (is_shared_mr(access_flags)) { + /* start address and length must be aligned to page size in order + to map a full page and preventing leakage of data */ + if (mr->umem->offset || (length & ~PAGE_MASK)) { + err = -EINVAL; + goto err_mr; + } + + err = prepare_shared_mr(mr, access_flags, mr_id); + if (err) + goto err_mr; + } +#endif + if (ib_peer_mem) { + if (access_flags & IB_ACCESS_MW_BIND) { + /* Prevent binding MW on peer clients. + * mlx4_invalidate_umem must be void, + * therefore, mlx4_mr_free should not fail + * when using peer clients. */ + err = -ENOSYS; + pr_err("MW is not supported with peer memory client"); + goto err_smr; + } + init_completion(&mr->invalidation_comp); + ib_umem_activate_invalidation_notifier(mr->umem, + mlx4_invalidate_umem, mr); + } + + atomic_set(&mr->invalidated, 0); + return &mr->ibmr; + +err_smr: +/* No suuport for Shared MR */ +#if 0 + if (mr->smr_info) + free_smr_info(mr); +#endif +err_mr: + (void) mlx4_mr_free(to_mdev(pd->device)->dev, &mr->mmr); + +err_umem: + ib_umem_release(mr->umem); + +err_free: + kfree(mr); + + return ERR_PTR(err); +} + +int mlx4_ib_dereg_mr(struct ib_mr *ibmr) +{ + struct mlx4_ib_mr *mr = to_mmr(ibmr); + struct ib_umem *umem = mr->umem; + int ret; + +/* No suuport for Shared MR */ +#if 0 + if (mr->smr_info) + free_smr_info(mr); +#endif + + if (atomic_inc_return(&mr->invalidated) > 1) { + wait_for_completion(&mr->invalidation_comp); + goto end; + } + + ret = mlx4_mr_free(to_mdev(ibmr->device)->dev, &mr->mmr); + if (ret) { + /* Error is not expected here, except when memory windows + * are bound to MR which is not supported with + * peer memory clients */ + atomic_set(&mr->invalidated, 0); + return ret; + } + + if (!umem) + goto end; + + ib_umem_release(mr->umem); +end: + + kfree(mr); + + return 0; +} + +struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type) +{ + struct mlx4_ib_dev *dev = to_mdev(pd->device); + struct mlx4_ib_mw *mw; + int err; + + mw = kmalloc(sizeof(*mw), GFP_KERNEL); + if (!mw) + return ERR_PTR(-ENOMEM); + + err = mlx4_mw_alloc(dev->dev, to_mpd(pd)->pdn, (enum mlx4_mw_type)type, &mw->mmw); + if (err) + goto err_free; + + err = mlx4_mw_enable(dev->dev, &mw->mmw); + if (err) + goto err_mw; + + mw->ibmw.rkey = mw->mmw.key; + + return &mw->ibmw; + +err_mw: + mlx4_mw_free(dev->dev, &mw->mmw); + +err_free: + kfree(mw); + + return ERR_PTR(err); +} + +int mlx4_ib_bind_mw(struct ib_qp *qp, struct ib_mw *mw, + struct ib_mw_bind *mw_bind) +{ + struct ib_send_wr wr; + struct ib_send_wr *bad_wr; + int ret; + + memset(&wr, 0, sizeof(wr)); + wr.opcode = IB_WR_BIND_MW; + wr.wr_id = mw_bind->wr_id; + wr.send_flags = mw_bind->send_flags; + wr.wr.bind_mw.mw = mw; + wr.wr.bind_mw.bind_info = mw_bind->bind_info; + wr.wr.bind_mw.rkey = ib_inc_rkey(mw->rkey); + + ret = mlx4_ib_post_send(qp, &wr, &bad_wr); + if (!ret) + mw->rkey = wr.wr.bind_mw.rkey; + + return ret; +} + +int mlx4_ib_dealloc_mw(struct ib_mw *ibmw) +{ + struct mlx4_ib_mw *mw = to_mmw(ibmw); + + mlx4_mw_free(to_mdev(ibmw->device)->dev, &mw->mmw); + kfree(mw); + + return 0; +} + +struct ib_mr *mlx4_ib_alloc_fast_reg_mr(struct ib_pd *pd, + int max_page_list_len) +{ + struct mlx4_ib_dev *dev = to_mdev(pd->device); + struct mlx4_ib_mr *mr; + int err; + + mr = kzalloc(sizeof *mr, GFP_KERNEL); + if (!mr) + return ERR_PTR(-ENOMEM); + + err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, 0, 0, 0, + max_page_list_len, 0, &mr->mmr); + if (err) + goto err_free; + + err = mlx4_mr_enable(dev->dev, &mr->mmr); + if (err) + goto err_mr; + + mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; + mr->umem = NULL; + + return &mr->ibmr; + +err_mr: + (void) mlx4_mr_free(dev->dev, &mr->mmr); + +err_free: + kfree(mr); + return ERR_PTR(err); +} + +struct ib_fast_reg_page_list *mlx4_ib_alloc_fast_reg_page_list(struct ib_device *ibdev, + int page_list_len) +{ + struct mlx4_ib_dev *dev = to_mdev(ibdev); + struct mlx4_ib_fast_reg_page_list *mfrpl; + int size = page_list_len * sizeof (u64); + + if (page_list_len > MLX4_MAX_FAST_REG_PAGES) + return ERR_PTR(-EINVAL); + + mfrpl = kmalloc(sizeof *mfrpl, GFP_KERNEL); + if (!mfrpl) + return ERR_PTR(-ENOMEM); + + mfrpl->ibfrpl.page_list = kmalloc(size, GFP_KERNEL); + if (!mfrpl->ibfrpl.page_list) + goto err_free; + + mfrpl->mapped_page_list = dma_alloc_coherent(&dev->dev->persist->pdev->dev, + size, &mfrpl->map, + GFP_KERNEL); + if (!mfrpl->mapped_page_list) + goto err_free; + + WARN_ON(mfrpl->map & 0x3f); + + return &mfrpl->ibfrpl; + +err_free: + kfree(mfrpl->ibfrpl.page_list); + kfree(mfrpl); + return ERR_PTR(-ENOMEM); +} + +void mlx4_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list) +{ + struct mlx4_ib_dev *dev = to_mdev(page_list->device); + struct mlx4_ib_fast_reg_page_list *mfrpl = to_mfrpl(page_list); + int size = page_list->max_page_list_len * sizeof (u64); + + dma_free_coherent(&dev->dev->persist->pdev->dev, size, mfrpl->mapped_page_list, + mfrpl->map); + kfree(mfrpl->ibfrpl.page_list); + kfree(mfrpl); +} + +struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int acc, + struct ib_fmr_attr *fmr_attr) +{ + struct mlx4_ib_dev *dev = to_mdev(pd->device); + struct mlx4_ib_fmr *fmr; + int err = -ENOMEM; + + fmr = kmalloc(sizeof *fmr, GFP_KERNEL); + if (!fmr) + return ERR_PTR(-ENOMEM); + + err = mlx4_fmr_alloc(dev->dev, to_mpd(pd)->pdn, convert_access(acc), + fmr_attr->max_pages, fmr_attr->max_maps, + fmr_attr->page_shift, &fmr->mfmr); + if (err) + goto err_free; + + err = mlx4_fmr_enable(to_mdev(pd->device)->dev, &fmr->mfmr); + if (err) + goto err_mr; + + fmr->ibfmr.rkey = fmr->ibfmr.lkey = fmr->mfmr.mr.key; + + return &fmr->ibfmr; + +err_mr: + (void) mlx4_mr_free(to_mdev(pd->device)->dev, &fmr->mfmr.mr); + +err_free: + kfree(fmr); + + return ERR_PTR(err); +} + +int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, + int npages, u64 iova) +{ + struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr); + struct mlx4_ib_dev *dev = to_mdev(ifmr->ibfmr.device); + + return mlx4_map_phys_fmr(dev->dev, &ifmr->mfmr, page_list, npages, iova, + &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey); +} + +int mlx4_ib_unmap_fmr(struct list_head *fmr_list) +{ + struct ib_fmr *ibfmr; + int err; + struct mlx4_dev *mdev = NULL; + + list_for_each_entry(ibfmr, fmr_list, list) { + if (mdev && to_mdev(ibfmr->device)->dev != mdev) + return -EINVAL; + mdev = to_mdev(ibfmr->device)->dev; + } + + if (!mdev) + return 0; + + list_for_each_entry(ibfmr, fmr_list, list) { + struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr); + + mlx4_fmr_unmap(mdev, &ifmr->mfmr, &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey); + } + + /* + * Make sure all MPT status updates are visible before issuing + * SYNC_TPT firmware command. + */ + wmb(); + + err = mlx4_SYNC_TPT(mdev); + if (err) + pr_warn("SYNC_TPT error %d when " + "unmapping FMRs\n", err); + + return 0; +} + +int mlx4_ib_fmr_dealloc(struct ib_fmr *ibfmr) +{ + struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr); + struct mlx4_ib_dev *dev = to_mdev(ibfmr->device); + int err; + + err = mlx4_fmr_free(dev->dev, &ifmr->mfmr); + + if (!err) + kfree(ifmr); + + return err; +} Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_mr.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_qp.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_qp.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_qp.c (revision 329159) @@ -0,0 +1,3634 @@ +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include "mlx4_ib.h" +#include "user.h" + +#define asm __asm + +enum { + MLX4_IB_ACK_REQ_FREQ = 8, +}; + +enum { + MLX4_IB_DEFAULT_SCHED_QUEUE = 0x83, + MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f, + MLX4_IB_LINK_TYPE_IB = 0, + MLX4_IB_LINK_TYPE_ETH = 1 +}; + +enum { + /* + * Largest possible UD header: send with GRH and immediate + * data plus 18 bytes for an Ethernet header with VLAN/802.1Q + * tag. (LRH would only use 8 bytes, so Ethernet is the + * biggest case) + */ + MLX4_IB_UD_HEADER_SIZE = 82, + MLX4_IB_LSO_HEADER_SPARE = 128, +}; + +enum { + MLX4_IB_IBOE_ETHERTYPE = 0x8915 +}; + +struct mlx4_ib_sqp { + struct mlx4_ib_qp qp; + int pkey_index; + u32 qkey; + u32 send_psn; + struct ib_ud_header ud_header; + u8 header_buf[MLX4_IB_UD_HEADER_SIZE]; +}; + +enum { + MLX4_IB_MIN_SQ_STRIDE = 6, + MLX4_IB_CACHE_LINE_SIZE = 64, +}; + +enum { + MLX4_RAW_QP_MTU = 7, + MLX4_RAW_QP_MSGMAX = 31, +}; + +static const __be32 mlx4_ib_opcode[] = { + [IB_WR_SEND] = cpu_to_be32(MLX4_OPCODE_SEND), + [IB_WR_LSO] = cpu_to_be32(MLX4_OPCODE_LSO), + [IB_WR_SEND_WITH_IMM] = cpu_to_be32(MLX4_OPCODE_SEND_IMM), + [IB_WR_RDMA_WRITE] = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE), + [IB_WR_RDMA_WRITE_WITH_IMM] = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM), + [IB_WR_RDMA_READ] = cpu_to_be32(MLX4_OPCODE_RDMA_READ), + [IB_WR_ATOMIC_CMP_AND_SWP] = cpu_to_be32(MLX4_OPCODE_ATOMIC_CS), + [IB_WR_ATOMIC_FETCH_AND_ADD] = cpu_to_be32(MLX4_OPCODE_ATOMIC_FA), + [IB_WR_SEND_WITH_INV] = cpu_to_be32(MLX4_OPCODE_SEND_INVAL), + [IB_WR_LOCAL_INV] = cpu_to_be32(MLX4_OPCODE_LOCAL_INVAL), + [IB_WR_FAST_REG_MR] = cpu_to_be32(MLX4_OPCODE_FMR), + [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_CS), + [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_FA), + [IB_WR_BIND_MW] = cpu_to_be32( + MLX4_OPCODE_BIND_MW), +}; + +#ifndef wc_wmb + #if defined(__i386__) + #define wc_wmb() asm volatile("lock; addl $0,0(%%esp) " ::: "memory") + #elif defined(__x86_64__) + #define wc_wmb() asm volatile("sfence" ::: "memory") + #elif defined(__ia64__) + #define wc_wmb() asm volatile("fwb" ::: "memory") + #else + #define wc_wmb() wmb() + #endif +#endif + +static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp) +{ + return container_of(mqp, struct mlx4_ib_sqp, qp); +} + +static int is_tunnel_qp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) +{ + if (!mlx4_is_master(dev->dev)) + return 0; + + return qp->mqp.qpn >= dev->dev->phys_caps.base_tunnel_sqpn && + qp->mqp.qpn < dev->dev->phys_caps.base_tunnel_sqpn + + 8 * MLX4_MFUNC_MAX; +} + +static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) +{ + int proxy_sqp = 0; + int real_sqp = 0; + int i; + /* PPF or Native -- real SQP */ + real_sqp = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) && + qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn && + qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 3); + if (real_sqp) + return 1; + /* VF or PF -- proxy SQP */ + if (mlx4_is_mfunc(dev->dev)) { + for (i = 0; i < dev->dev->caps.num_ports; i++) { + if (qp->mqp.qpn == dev->dev->caps.qp0_proxy[i] || + qp->mqp.qpn == dev->dev->caps.qp1_proxy[i]) { + proxy_sqp = 1; + break; + } + } + } + return proxy_sqp; +} + +/* used for INIT/CLOSE port logic */ +static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) +{ + int proxy_qp0 = 0; + int real_qp0 = 0; + int i; + /* PPF or Native -- real QP0 */ + real_qp0 = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) && + qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn && + qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 1); + if (real_qp0) + return 1; + /* VF or PF -- proxy QP0 */ + if (mlx4_is_mfunc(dev->dev)) { + for (i = 0; i < dev->dev->caps.num_ports; i++) { + if (qp->mqp.qpn == dev->dev->caps.qp0_proxy[i]) { + proxy_qp0 = 1; + break; + } + } + } + return proxy_qp0; +} + +static void *get_wqe(struct mlx4_ib_qp *qp, int offset) +{ + return mlx4_buf_offset(&qp->buf, offset); +} + +static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n) +{ + return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift)); +} + +static void *get_send_wqe(struct mlx4_ib_qp *qp, int n) +{ + return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift)); +} + +/* + * Stamp a SQ WQE so that it is invalid if prefetched by marking the + * first four bytes of every 64 byte chunk with + * 0x7FFFFFF | (invalid_ownership_value << 31). + * + * When the max work request size is less than or equal to the WQE + * basic block size, as an optimization, we can stamp all WQEs with + * 0xffffffff, and skip the very first chunk of each WQE. + */ +static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n, int size) +{ + __be32 *wqe; + int i; + int s; + int ind; + void *buf; + __be32 stamp; + struct mlx4_wqe_ctrl_seg *ctrl; + + if (qp->sq_max_wqes_per_wr > 1) { + s = roundup(size, 1U << qp->sq.wqe_shift); + for (i = 0; i < s; i += 64) { + ind = (i >> qp->sq.wqe_shift) + n; + stamp = ind & qp->sq.wqe_cnt ? cpu_to_be32(0x7fffffff) : + cpu_to_be32(0xffffffff); + buf = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1)); + wqe = buf + (i & ((1 << qp->sq.wqe_shift) - 1)); + *wqe = stamp; + } + } else { + ctrl = buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1)); + s = (ctrl->fence_size & 0x3f) << 4; + for (i = 64; i < s; i += 64) { + wqe = buf + i; + *wqe = cpu_to_be32(0xffffffff); + } + } +} + +static void post_nop_wqe(struct mlx4_ib_qp *qp, int n, int size) +{ + struct mlx4_wqe_ctrl_seg *ctrl; + struct mlx4_wqe_inline_seg *inl; + void *wqe; + int s; + + ctrl = wqe = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1)); + s = sizeof(struct mlx4_wqe_ctrl_seg); + + if (qp->ibqp.qp_type == IB_QPT_UD) { + struct mlx4_wqe_datagram_seg *dgram = wqe + sizeof *ctrl; + struct mlx4_av *av = (struct mlx4_av *)dgram->av; + memset(dgram, 0, sizeof *dgram); + av->port_pd = cpu_to_be32((qp->port << 24) | to_mpd(qp->ibqp.pd)->pdn); + s += sizeof(struct mlx4_wqe_datagram_seg); + } + + /* Pad the remainder of the WQE with an inline data segment. */ + if (size > s) { + inl = wqe + s; + inl->byte_count = cpu_to_be32(1 << 31 | (size - s - sizeof *inl)); + } + ctrl->srcrb_flags = 0; + ctrl->fence_size = size / 16; + /* + * Make sure descriptor is fully written before setting ownership bit + * (because HW can start executing as soon as we do). + */ + wmb(); + + ctrl->owner_opcode = cpu_to_be32(MLX4_OPCODE_NOP | MLX4_WQE_CTRL_NEC) | + (n & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0); + + stamp_send_wqe(qp, n + qp->sq_spare_wqes, size); +} + +/* Post NOP WQE to prevent wrap-around in the middle of WR */ +static inline unsigned pad_wraparound(struct mlx4_ib_qp *qp, int ind) +{ + unsigned s = qp->sq.wqe_cnt - (ind & (qp->sq.wqe_cnt - 1)); + if (unlikely(s < qp->sq_max_wqes_per_wr)) { + post_nop_wqe(qp, ind, s << qp->sq.wqe_shift); + ind += s; + } + return ind; +} + +static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type) +{ + struct ib_event event; + struct ib_qp *ibqp = &to_mibqp(qp)->ibqp; + + if (type == MLX4_EVENT_TYPE_PATH_MIG) + to_mibqp(qp)->port = to_mibqp(qp)->alt_port; + + if (ibqp->event_handler) { + event.device = ibqp->device; + event.element.qp = ibqp; + switch (type) { + case MLX4_EVENT_TYPE_PATH_MIG: + event.event = IB_EVENT_PATH_MIG; + break; + case MLX4_EVENT_TYPE_COMM_EST: + event.event = IB_EVENT_COMM_EST; + break; + case MLX4_EVENT_TYPE_SQ_DRAINED: + event.event = IB_EVENT_SQ_DRAINED; + break; + case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE: + event.event = IB_EVENT_QP_LAST_WQE_REACHED; + break; + case MLX4_EVENT_TYPE_WQ_CATAS_ERROR: + event.event = IB_EVENT_QP_FATAL; + break; + case MLX4_EVENT_TYPE_PATH_MIG_FAILED: + event.event = IB_EVENT_PATH_MIG_ERR; + break; + case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + event.event = IB_EVENT_QP_REQ_ERR; + break; + case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR: + event.event = IB_EVENT_QP_ACCESS_ERR; + break; + default: + pr_warn("Unexpected event type %d " + "on QP %06x\n", type, qp->qpn); + return; + } + + ibqp->event_handler(&event, ibqp->qp_context); + } +} + +static int send_wqe_overhead(enum mlx4_ib_qp_type type, u32 flags) +{ + /* + * UD WQEs must have a datagram segment. + * RC and UC WQEs might have a remote address segment. + * MLX WQEs need two extra inline data segments (for the UD + * header and space for the ICRC). + */ + switch (type) { + case MLX4_IB_QPT_UD: + return sizeof (struct mlx4_wqe_ctrl_seg) + + sizeof (struct mlx4_wqe_datagram_seg) + + ((flags & MLX4_IB_QP_LSO) ? MLX4_IB_LSO_HEADER_SPARE : 0); + case MLX4_IB_QPT_PROXY_SMI_OWNER: + case MLX4_IB_QPT_PROXY_SMI: + case MLX4_IB_QPT_PROXY_GSI: + return sizeof (struct mlx4_wqe_ctrl_seg) + + sizeof (struct mlx4_wqe_datagram_seg) + 64; + case MLX4_IB_QPT_TUN_SMI_OWNER: + case MLX4_IB_QPT_TUN_GSI: + return sizeof (struct mlx4_wqe_ctrl_seg) + + sizeof (struct mlx4_wqe_datagram_seg); + + case MLX4_IB_QPT_UC: + return sizeof (struct mlx4_wqe_ctrl_seg) + + sizeof (struct mlx4_wqe_raddr_seg); + case MLX4_IB_QPT_RC: + return sizeof (struct mlx4_wqe_ctrl_seg) + + sizeof (struct mlx4_wqe_masked_atomic_seg) + + sizeof (struct mlx4_wqe_raddr_seg); + case MLX4_IB_QPT_SMI: + case MLX4_IB_QPT_GSI: + return sizeof (struct mlx4_wqe_ctrl_seg) + + ALIGN(MLX4_IB_UD_HEADER_SIZE + + DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE, + MLX4_INLINE_ALIGN) * + sizeof (struct mlx4_wqe_inline_seg), + sizeof (struct mlx4_wqe_data_seg)) + + ALIGN(4 + + sizeof (struct mlx4_wqe_inline_seg), + sizeof (struct mlx4_wqe_data_seg)); + default: + return sizeof (struct mlx4_wqe_ctrl_seg); + } +} + +static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap, + int is_user, int has_rq, struct mlx4_ib_qp *qp) +{ + /* Sanity check RQ size before proceeding */ + if (cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE || + cap->max_recv_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg)) + return -EINVAL; + + if (!has_rq) { + if (cap->max_recv_wr) + return -EINVAL; + + qp->rq.wqe_cnt = qp->rq.max_gs = 0; + } else { + /* HW requires >= 1 RQ entry with >= 1 gather entry */ + if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge)) + return -EINVAL; + + qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, cap->max_recv_wr)); + qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge)); + qp->rq.wqe_shift = ilog2(qp->rq.max_gs * sizeof (struct mlx4_wqe_data_seg)); + } + + /* leave userspace return values as they were, so as not to break ABI */ + if (is_user) { + cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt; + cap->max_recv_sge = qp->rq.max_gs; + } else { + cap->max_recv_wr = qp->rq.max_post = + min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE, qp->rq.wqe_cnt); + cap->max_recv_sge = min(qp->rq.max_gs, + min(dev->dev->caps.max_sq_sg, + dev->dev->caps.max_rq_sg)); + } + + return 0; +} + +static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap, + enum mlx4_ib_qp_type type, struct mlx4_ib_qp *qp) +{ + int s; + + /* Sanity check SQ size before proceeding */ + if (cap->max_send_wr > (dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE) || + cap->max_send_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) || + cap->max_inline_data + send_wqe_overhead(type, qp->flags) + + sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz) + return -EINVAL; + + /* + * For MLX transport we need 2 extra S/G entries: + * one for the header and one for the checksum at the end + */ + if ((type == MLX4_IB_QPT_SMI || type == MLX4_IB_QPT_GSI || + type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) && + cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg) + return -EINVAL; + + s = max(cap->max_send_sge * sizeof (struct mlx4_wqe_data_seg), + cap->max_inline_data + sizeof (struct mlx4_wqe_inline_seg)) + + send_wqe_overhead(type, qp->flags); + + if (s > dev->dev->caps.max_sq_desc_sz) + return -EINVAL; + + /* + * Hermon supports shrinking WQEs, such that a single work + * request can include multiple units of 1 << wqe_shift. This + * way, work requests can differ in size, and do not have to + * be a power of 2 in size, saving memory and speeding up send + * WR posting. Unfortunately, if we do this then the + * wqe_index field in CQEs can't be used to look up the WR ID + * anymore, so we do this only if selective signaling is off. + * + * Further, on 32-bit platforms, we can't use vmap() to make + * the QP buffer virtually contiguous. Thus we have to use + * constant-sized WRs to make sure a WR is always fully within + * a single page-sized chunk. + * + * Finally, we use NOP work requests to pad the end of the + * work queue, to avoid wrap-around in the middle of WR. We + * set NEC bit to avoid getting completions with error for + * these NOP WRs, but since NEC is only supported starting + * with firmware 2.2.232, we use constant-sized WRs for older + * firmware. + * + * And, since MLX QPs only support SEND, we use constant-sized + * WRs in this case. + * + * We look for the smallest value of wqe_shift such that the + * resulting number of wqes does not exceed device + * capabilities. + * + * We set WQE size to at least 64 bytes, this way stamping + * invalidates each WQE. + */ + if (dev->dev->caps.fw_ver >= MLX4_FW_VER_WQE_CTRL_NEC && + qp->sq_signal_bits && BITS_PER_LONG == 64 && + type != MLX4_IB_QPT_SMI && type != MLX4_IB_QPT_GSI && + !(type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_PROXY_SMI | + MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) + qp->sq.wqe_shift = ilog2(64); + else + qp->sq.wqe_shift = ilog2(roundup_pow_of_two(s)); + + for (;;) { + qp->sq_max_wqes_per_wr = DIV_ROUND_UP(s, 1U << qp->sq.wqe_shift); + + /* + * We need to leave 2 KB + 1 WR of headroom in the SQ to + * allow HW to prefetch. + */ + qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + qp->sq_max_wqes_per_wr; + qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr * + qp->sq_max_wqes_per_wr + + qp->sq_spare_wqes); + + if (qp->sq.wqe_cnt <= dev->dev->caps.max_wqes) + break; + + if (qp->sq_max_wqes_per_wr <= 1) + return -EINVAL; + + ++qp->sq.wqe_shift; + } + + qp->sq.max_gs = (min(dev->dev->caps.max_sq_desc_sz, + (qp->sq_max_wqes_per_wr << qp->sq.wqe_shift)) - + send_wqe_overhead(type, qp->flags)) / + sizeof (struct mlx4_wqe_data_seg); + + qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + + (qp->sq.wqe_cnt << qp->sq.wqe_shift); + if (qp->rq.wqe_shift > qp->sq.wqe_shift) { + qp->rq.offset = 0; + qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; + } else { + qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift; + qp->sq.offset = 0; + } + + cap->max_send_wr = qp->sq.max_post = + (qp->sq.wqe_cnt - qp->sq_spare_wqes) / qp->sq_max_wqes_per_wr; + cap->max_send_sge = min(qp->sq.max_gs, + min(dev->dev->caps.max_sq_sg, + dev->dev->caps.max_rq_sg)); + qp->max_inline_data = cap->max_inline_data; + + return 0; +} + +static int set_user_sq_size(struct mlx4_ib_dev *dev, + struct mlx4_ib_qp *qp, + struct mlx4_ib_create_qp *ucmd) +{ + /* Sanity check SQ size before proceeding */ + if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes || + ucmd->log_sq_stride > + ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) || + ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE) + return -EINVAL; + + qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count; + qp->sq.wqe_shift = ucmd->log_sq_stride; + + qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + + (qp->sq.wqe_cnt << qp->sq.wqe_shift); + + return 0; +} + +static int alloc_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp) +{ + int i; + + qp->sqp_proxy_rcv = + kmalloc(sizeof (struct mlx4_ib_buf) * qp->rq.wqe_cnt, + GFP_KERNEL); + if (!qp->sqp_proxy_rcv) + return -ENOMEM; + for (i = 0; i < qp->rq.wqe_cnt; i++) { + qp->sqp_proxy_rcv[i].addr = + kmalloc(sizeof (struct mlx4_ib_proxy_sqp_hdr), + GFP_KERNEL); + if (!qp->sqp_proxy_rcv[i].addr) + goto err; + qp->sqp_proxy_rcv[i].map = + ib_dma_map_single(dev, qp->sqp_proxy_rcv[i].addr, + sizeof (struct mlx4_ib_proxy_sqp_hdr), + DMA_FROM_DEVICE); + if (unlikely(ib_dma_mapping_error(dev, + qp->sqp_proxy_rcv[i].map))) { + pr_warn("ib_dma_map_single failed\n"); + kfree(qp->sqp_proxy_rcv[i].addr); + goto err; + } + } + return 0; + +err: + while (i > 0) { + --i; + ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map, + sizeof (struct mlx4_ib_proxy_sqp_hdr), + DMA_FROM_DEVICE); + kfree(qp->sqp_proxy_rcv[i].addr); + } + kfree(qp->sqp_proxy_rcv); + qp->sqp_proxy_rcv = NULL; + return -ENOMEM; +} + +static void free_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp) +{ + int i; + + for (i = 0; i < qp->rq.wqe_cnt; i++) { + ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map, + sizeof (struct mlx4_ib_proxy_sqp_hdr), + DMA_FROM_DEVICE); + kfree(qp->sqp_proxy_rcv[i].addr); + } + kfree(qp->sqp_proxy_rcv); +} + +static int init_qpg_parent(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *pqp, + struct ib_qp_init_attr *attr, int *qpn) +{ + struct mlx4_ib_qpg_data *qpg_data; + int tss_num, rss_num; + int tss_align_num, rss_align_num; + int tss_base, rss_base = 0; + int err; + + /* Parent is part of the TSS range (in SW TSS ARP is sent via parent) */ + tss_num = 1 + attr->parent_attrib.tss_child_count; + tss_align_num = roundup_pow_of_two(tss_num); + rss_num = attr->parent_attrib.rss_child_count; + rss_align_num = roundup_pow_of_two(rss_num); + + if (rss_num > 1) { + /* RSS is requested */ + if (!(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS)) + return -ENOSYS; + if (rss_align_num > dev->dev->caps.max_rss_tbl_sz) + return -EINVAL; + /* We must work with power of two */ + attr->parent_attrib.rss_child_count = rss_align_num; + } + + qpg_data = kzalloc(sizeof *qpg_data, GFP_KERNEL); + if (!qpg_data) + return -ENOMEM; + + if(pqp->flags & MLX4_IB_QP_NETIF) + err = mlx4_ib_steer_qp_alloc(dev, tss_align_num, &tss_base); + else + err = mlx4_qp_reserve_range(dev->dev, tss_align_num, + tss_align_num, &tss_base, MLX4_RESERVE_ETH_BF_QP); + if (err) + goto err1; + + if (tss_num > 1) { + u32 alloc = BITS_TO_LONGS(tss_align_num) * sizeof(long); + qpg_data->tss_bitmap = kzalloc(alloc, GFP_KERNEL); + if (qpg_data->tss_bitmap == NULL) { + err = -ENOMEM; + goto err2; + } + bitmap_fill(qpg_data->tss_bitmap, tss_num); + /* Note parent takes first index */ + clear_bit(0, qpg_data->tss_bitmap); + } + + if (rss_num > 1) { + u32 alloc = BITS_TO_LONGS(rss_align_num) * sizeof(long); + err = mlx4_qp_reserve_range(dev->dev, rss_align_num, + 1, &rss_base, 0); + if (err) + goto err3; + qpg_data->rss_bitmap = kzalloc(alloc, GFP_KERNEL); + if (qpg_data->rss_bitmap == NULL) { + err = -ENOMEM; + goto err4; + } + bitmap_fill(qpg_data->rss_bitmap, rss_align_num); + } + + qpg_data->tss_child_count = attr->parent_attrib.tss_child_count; + qpg_data->rss_child_count = attr->parent_attrib.rss_child_count; + qpg_data->qpg_parent = pqp; + qpg_data->qpg_tss_mask_sz = ilog2(tss_align_num); + qpg_data->tss_qpn_base = tss_base; + qpg_data->rss_qpn_base = rss_base; + + pqp->qpg_data = qpg_data; + *qpn = tss_base; + + return 0; + +err4: + mlx4_qp_release_range(dev->dev, rss_base, rss_align_num); + +err3: + if (tss_num > 1) + kfree(qpg_data->tss_bitmap); + +err2: + if(pqp->flags & MLX4_IB_QP_NETIF) + mlx4_ib_steer_qp_free(dev, tss_base, tss_align_num); + else + mlx4_qp_release_range(dev->dev, tss_base, tss_align_num); + +err1: + kfree(qpg_data); + return err; +} + +static void free_qpg_parent(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *pqp) +{ + struct mlx4_ib_qpg_data *qpg_data = pqp->qpg_data; + int align_num; + + if (qpg_data->tss_child_count > 1) + kfree(qpg_data->tss_bitmap); + + align_num = roundup_pow_of_two(1 + qpg_data->tss_child_count); + if(pqp->flags & MLX4_IB_QP_NETIF) + mlx4_ib_steer_qp_free(dev, qpg_data->tss_qpn_base, align_num); + else + mlx4_qp_release_range(dev->dev, qpg_data->tss_qpn_base, align_num); + + if (qpg_data->rss_child_count > 1) { + kfree(qpg_data->rss_bitmap); + align_num = roundup_pow_of_two(qpg_data->rss_child_count); + mlx4_qp_release_range(dev->dev, qpg_data->rss_qpn_base, + align_num); + } + + kfree(qpg_data); +} + +static int alloc_qpg_qpn(struct ib_qp_init_attr *init_attr, + struct mlx4_ib_qp *pqp, int *qpn) +{ + struct mlx4_ib_qp *mqp = to_mqp(init_attr->qpg_parent); + struct mlx4_ib_qpg_data *qpg_data = mqp->qpg_data; + u32 idx, old; + + switch (init_attr->qpg_type) { + case IB_QPG_CHILD_TX: + if (qpg_data->tss_child_count == 0) + return -EINVAL; + do { + /* Parent took index 0 */ + idx = find_first_bit(qpg_data->tss_bitmap, + qpg_data->tss_child_count + 1); + if (idx >= qpg_data->tss_child_count + 1) + return -ENOMEM; + old = test_and_clear_bit(idx, qpg_data->tss_bitmap); + } while (old == 0); + idx += qpg_data->tss_qpn_base; + break; + case IB_QPG_CHILD_RX: + if (qpg_data->rss_child_count == 0) + return -EINVAL; + do { + idx = find_first_bit(qpg_data->rss_bitmap, + qpg_data->rss_child_count); + if (idx >= qpg_data->rss_child_count) + return -ENOMEM; + old = test_and_clear_bit(idx, qpg_data->rss_bitmap); + } while (old == 0); + idx += qpg_data->rss_qpn_base; + break; + default: + return -EINVAL; + } + + pqp->qpg_data = qpg_data; + *qpn = idx; + + return 0; +} + +static void free_qpg_qpn(struct mlx4_ib_qp *mqp, int qpn) +{ + struct mlx4_ib_qpg_data *qpg_data = mqp->qpg_data; + + switch (mqp->qpg_type) { + case IB_QPG_CHILD_TX: + /* Do range check */ + qpn -= qpg_data->tss_qpn_base; + set_bit(qpn, qpg_data->tss_bitmap); + break; + case IB_QPG_CHILD_RX: + qpn -= qpg_data->rss_qpn_base; + set_bit(qpn, qpg_data->rss_bitmap); + break; + default: + /* error */ + pr_warn("wrong qpg type (%d)\n", mqp->qpg_type); + break; + } +} + +static int alloc_qpn_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, + struct ib_qp_init_attr *attr, int *qpn) +{ + int err = 0; + + switch (attr->qpg_type) { + case IB_QPG_NONE: + /* Raw packet QPNs may not have bits 6,7 set in their qp_num; + * otherwise, the WQE BlueFlame setup flow wrongly causes + * VLAN insertion. */ + if (attr->qp_type == IB_QPT_RAW_PACKET) { + err = mlx4_qp_reserve_range(dev->dev, 1, 1, qpn, + MLX4_RESERVE_ETH_BF_QP); + } else { + if(qp->flags & MLX4_IB_QP_NETIF) + err = mlx4_ib_steer_qp_alloc(dev, 1, qpn); + else + err = mlx4_qp_reserve_range(dev->dev, 1, 1, qpn, 0); + } + break; + case IB_QPG_PARENT: + err = init_qpg_parent(dev, qp, attr, qpn); + break; + case IB_QPG_CHILD_TX: + case IB_QPG_CHILD_RX: + err = alloc_qpg_qpn(attr, qp, qpn); + break; + default: + qp->qpg_type = IB_QPG_NONE; + err = -EINVAL; + break; + } + if (err) + return err; + qp->qpg_type = attr->qpg_type; + return 0; +} + +static void free_qpn_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, + enum ib_qpg_type qpg_type, int qpn) +{ + switch (qpg_type) { + case IB_QPG_NONE: + if (qp->flags & MLX4_IB_QP_NETIF) + mlx4_ib_steer_qp_free(dev, qpn, 1); + else + mlx4_qp_release_range(dev->dev, qpn, 1); + break; + case IB_QPG_PARENT: + free_qpg_parent(dev, qp); + break; + case IB_QPG_CHILD_TX: + case IB_QPG_CHILD_RX: + free_qpg_qpn(qp, qpn); + break; + default: + break; + } +} + +/* Revert allocation on create_qp_common */ +static void unalloc_qpn_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, + struct ib_qp_init_attr *attr, int qpn) +{ + free_qpn_common(dev, qp, attr->qpg_type, qpn); +} + +static void release_qpn_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) +{ + free_qpn_common(dev, qp, qp->qpg_type, qp->mqp.qpn); +} + +static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, int sqpn, struct mlx4_ib_qp **caller_qp) +{ + int qpn; + int err; + struct mlx4_ib_sqp *sqp; + struct mlx4_ib_qp *qp; + enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr->qp_type; + + /* When tunneling special qps, we use a plain UD qp */ + if (sqpn) { + if (mlx4_is_mfunc(dev->dev) && + (!mlx4_is_master(dev->dev) || + !(init_attr->create_flags & MLX4_IB_SRIOV_SQP))) { + if (init_attr->qp_type == IB_QPT_GSI) + qp_type = MLX4_IB_QPT_PROXY_GSI; + else if (mlx4_is_master(dev->dev)) + qp_type = MLX4_IB_QPT_PROXY_SMI_OWNER; + else + qp_type = MLX4_IB_QPT_PROXY_SMI; + } + qpn = sqpn; + /* add extra sg entry for tunneling */ + init_attr->cap.max_recv_sge++; + } else if (init_attr->create_flags & MLX4_IB_SRIOV_TUNNEL_QP) { + struct mlx4_ib_qp_tunnel_init_attr *tnl_init = + container_of(init_attr, + struct mlx4_ib_qp_tunnel_init_attr, init_attr); + if ((tnl_init->proxy_qp_type != IB_QPT_SMI && + tnl_init->proxy_qp_type != IB_QPT_GSI) || + !mlx4_is_master(dev->dev)) + return -EINVAL; + if (tnl_init->proxy_qp_type == IB_QPT_GSI) + qp_type = MLX4_IB_QPT_TUN_GSI; + else if (tnl_init->slave == mlx4_master_func_num(dev->dev)) + qp_type = MLX4_IB_QPT_TUN_SMI_OWNER; + else + qp_type = MLX4_IB_QPT_TUN_SMI; + /* we are definitely in the PPF here, since we are creating + * tunnel QPs. base_tunnel_sqpn is therefore valid. */ + qpn = dev->dev->phys_caps.base_tunnel_sqpn + 8 * tnl_init->slave + + tnl_init->proxy_qp_type * 2 + tnl_init->port - 1; + sqpn = qpn; + } + + if (!*caller_qp) { + if (qp_type == MLX4_IB_QPT_SMI || qp_type == MLX4_IB_QPT_GSI || + (qp_type & (MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_SMI_OWNER | + MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) { + sqp = kzalloc(sizeof (struct mlx4_ib_sqp), GFP_KERNEL); + if (!sqp) + return -ENOMEM; + qp = &sqp->qp; + qp->pri.vid = qp->alt.vid = 0xFFFF; + } else { + qp = kzalloc(sizeof (struct mlx4_ib_qp), GFP_KERNEL); + if (!qp) + return -ENOMEM; + qp->pri.vid = qp->alt.vid = 0xFFFF; + } + } else + qp = *caller_qp; + + qp->mlx4_ib_qp_type = qp_type; + + if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) + qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK; + + if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) + qp->flags |= MLX4_IB_QP_LSO; + + if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) { + if (dev->dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED && + !mlx4_is_mfunc(dev->dev)) + qp->flags |= MLX4_IB_QP_NETIF; + else { + err = -EINVAL; + goto err; + } + } + + mutex_init(&qp->mutex); + spin_lock_init(&qp->sq.lock); + spin_lock_init(&qp->rq.lock); + INIT_LIST_HEAD(&qp->gid_list); + INIT_LIST_HEAD(&qp->steering_rules); + INIT_LIST_HEAD(&qp->rules_list); + + qp->state = IB_QPS_RESET; + if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) + qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); + + err = set_rq_size(dev, &init_attr->cap, !!pd->uobject, mlx4_ib_qp_has_rq(init_attr), qp); + if (err) + goto err; + + if (pd->uobject) { + struct mlx4_ib_create_qp ucmd; + int shift; + int n; + + if (!udata || ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) { + err = -EFAULT; + goto err; + } + + if (init_attr->create_flags & IB_QP_CREATE_CROSS_CHANNEL) + qp->flags |= MLX4_IB_QP_CAP_CROSS_CHANNEL; + + if (init_attr->create_flags & IB_QP_CREATE_MANAGED_SEND) + qp->flags |= MLX4_IB_QP_CAP_MANAGED_SEND; + + if (init_attr->create_flags & IB_QP_CREATE_MANAGED_RECV) + qp->flags |= MLX4_IB_QP_CAP_MANAGED_RECV; + + qp->sq_no_prefetch = ucmd.sq_no_prefetch; + + err = set_user_sq_size(dev, qp, &ucmd); + if (err) + goto err; + + qp->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, + qp->buf_size, 0, 0); + if (IS_ERR(qp->umem)) { + err = PTR_ERR(qp->umem); + goto err; + } + + n = ib_umem_page_count(qp->umem); + shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n); + err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt); + + if (err) + goto err_buf; + + err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem); + if (err) + goto err_mtt; + + if (mlx4_ib_qp_has_rq(init_attr)) { + err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context), + ucmd.db_addr, &qp->db); + if (err) + goto err_mtt; + } + } else { + qp->sq_no_prefetch = 0; + + err = set_kernel_sq_size(dev, &init_attr->cap, qp_type, qp); + if (err) + goto err; + + if (mlx4_ib_qp_has_rq(init_attr)) { + err = mlx4_db_alloc(dev->dev, &qp->db, 0, GFP_KERNEL); + if (err) + goto err; + + *qp->db.db = 0; + } + + if (qp->max_inline_data) { + err = mlx4_bf_alloc(dev->dev, &qp->bf, 0); + if (err) { + pr_debug("failed to allocate blue flame" + " register (%d)", err); + qp->bf.uar = &dev->priv_uar; + } + } else + qp->bf.uar = &dev->priv_uar; + + if (mlx4_buf_alloc(dev->dev, qp->buf_size, PAGE_SIZE * 2, &qp->buf, GFP_KERNEL)) { + err = -ENOMEM; + goto err_db; + } + + err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift, + &qp->mtt); + if (err) + goto err_buf; + + err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf, GFP_KERNEL); + if (err) + goto err_mtt; + + qp->sq.wrid = kmalloc(qp->sq.wqe_cnt * sizeof (u64), GFP_KERNEL); + qp->rq.wrid = kmalloc(qp->rq.wqe_cnt * sizeof (u64), GFP_KERNEL); + + if (!qp->sq.wrid || !qp->rq.wrid) { + err = -ENOMEM; + goto err_wrid; + } + } + + if (sqpn) { + if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER | + MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) { + if (alloc_proxy_bufs(pd->device, qp)) { + err = -ENOMEM; + goto err_wrid; + } + } + } else { + err = alloc_qpn_common(dev, qp, init_attr, &qpn); + if (err) + goto err_proxy; + } + + err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp, GFP_KERNEL); + if (err) + goto err_qpn; + + if (init_attr->qp_type == IB_QPT_XRC_TGT) + qp->mqp.qpn |= (1 << 23); + + /* + * Hardware wants QPN written in big-endian order (after + * shifting) for send doorbell. Precompute this value to save + * a little bit when posting sends. + */ + qp->doorbell_qpn = swab32(qp->mqp.qpn << 8); + + qp->mqp.event = mlx4_ib_qp_event; + if (!*caller_qp) + *caller_qp = qp; + return 0; + +err_qpn: + unalloc_qpn_common(dev, qp, init_attr, qpn); + +err_proxy: + if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI) + free_proxy_bufs(pd->device, qp); +err_wrid: + if (pd->uobject) { + if (mlx4_ib_qp_has_rq(init_attr)) + mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &qp->db); + } else { + kfree(qp->sq.wrid); + kfree(qp->rq.wrid); + } + +err_mtt: + mlx4_mtt_cleanup(dev->dev, &qp->mtt); + +err_buf: + if (pd->uobject) + ib_umem_release(qp->umem); + else + mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf); + +err_db: + if (!pd->uobject && mlx4_ib_qp_has_rq(init_attr)) + mlx4_db_free(dev->dev, &qp->db); + + if (qp->max_inline_data) + mlx4_bf_free(dev->dev, &qp->bf); + +err: + if (!*caller_qp) + kfree(qp); + return err; +} + +static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state) +{ + switch (state) { + case IB_QPS_RESET: return MLX4_QP_STATE_RST; + case IB_QPS_INIT: return MLX4_QP_STATE_INIT; + case IB_QPS_RTR: return MLX4_QP_STATE_RTR; + case IB_QPS_RTS: return MLX4_QP_STATE_RTS; + case IB_QPS_SQD: return MLX4_QP_STATE_SQD; + case IB_QPS_SQE: return MLX4_QP_STATE_SQER; + case IB_QPS_ERR: return MLX4_QP_STATE_ERR; + default: return -1; + } +} + +static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq) + __acquires(&send_cq->lock) __acquires(&recv_cq->lock) +{ + if (send_cq == recv_cq) { + spin_lock_irq(&send_cq->lock); + __acquire(&recv_cq->lock); + } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) { + spin_lock_irq(&send_cq->lock); + spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); + } else { + spin_lock_irq(&recv_cq->lock); + spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); + } +} + +static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq) + __releases(&send_cq->lock) __releases(&recv_cq->lock) +{ + if (send_cq == recv_cq) { + __release(&recv_cq->lock); + spin_unlock_irq(&send_cq->lock); + } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) { + spin_unlock(&recv_cq->lock); + spin_unlock_irq(&send_cq->lock); + } else { + spin_unlock(&send_cq->lock); + spin_unlock_irq(&recv_cq->lock); + } +} + +static void del_gid_entries(struct mlx4_ib_qp *qp) +{ + struct mlx4_ib_gid_entry *ge, *tmp; + + list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) { + list_del(&ge->list); + kfree(ge); + } +} + +static struct mlx4_ib_pd *get_pd(struct mlx4_ib_qp *qp) +{ + if (qp->ibqp.qp_type == IB_QPT_XRC_TGT) + return to_mpd(to_mxrcd(qp->ibqp.xrcd)->pd); + else + return to_mpd(qp->ibqp.pd); +} + +static void get_cqs(struct mlx4_ib_qp *qp, + struct mlx4_ib_cq **send_cq, struct mlx4_ib_cq **recv_cq) +{ + switch (qp->ibqp.qp_type) { + case IB_QPT_XRC_TGT: + *send_cq = to_mcq(to_mxrcd(qp->ibqp.xrcd)->cq); + *recv_cq = *send_cq; + break; + case IB_QPT_XRC_INI: + *send_cq = to_mcq(qp->ibqp.send_cq); + *recv_cq = *send_cq; + break; + default: + *send_cq = to_mcq(qp->ibqp.send_cq); + *recv_cq = to_mcq(qp->ibqp.recv_cq); + break; + } +} + +static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, + int is_user) +{ + struct mlx4_ib_cq *send_cq, *recv_cq; + + if (qp->state != IB_QPS_RESET) { + if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state), + MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp)) + pr_warn("modify QP %06x to RESET failed.\n", + qp->mqp.qpn); + if (qp->pri.smac) { + mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac); + qp->pri.smac = 0; + } + if (qp->alt.smac) { + mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac); + qp->alt.smac = 0; + } + if (qp->pri.vid < 0x1000) { + mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid); + qp->pri.vid = 0xFFFF; + qp->pri.candidate_vid = 0xFFFF; + qp->pri.update_vid = 0; + } + if (qp->alt.vid < 0x1000) { + mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid); + qp->alt.vid = 0xFFFF; + qp->alt.candidate_vid = 0xFFFF; + qp->alt.update_vid = 0; + } + } + + get_cqs(qp, &send_cq, &recv_cq); + + mlx4_ib_lock_cqs(send_cq, recv_cq); + + if (!is_user) { + __mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn, + qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL); + if (send_cq != recv_cq) + __mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL); + } + + mlx4_qp_remove(dev->dev, &qp->mqp); + + mlx4_ib_unlock_cqs(send_cq, recv_cq); + + mlx4_qp_free(dev->dev, &qp->mqp); + + if (!is_sqp(dev, qp) && !is_tunnel_qp(dev, qp)) + release_qpn_common(dev, qp); + + mlx4_mtt_cleanup(dev->dev, &qp->mtt); + + if (is_user) { + if (qp->rq.wqe_cnt) + mlx4_ib_db_unmap_user(to_mucontext(qp->ibqp.uobject->context), + &qp->db); + ib_umem_release(qp->umem); + } else { + kfree(qp->sq.wrid); + kfree(qp->rq.wrid); + if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER | + MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) + free_proxy_bufs(&dev->ib_dev, qp); + mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf); + if (qp->max_inline_data) + mlx4_bf_free(dev->dev, &qp->bf); + + if (qp->rq.wqe_cnt) + mlx4_db_free(dev->dev, &qp->db); + } + + del_gid_entries(qp); +} + +static u32 get_sqp_num(struct mlx4_ib_dev *dev, struct ib_qp_init_attr *attr) +{ + /* Native or PPF */ + if (!mlx4_is_mfunc(dev->dev) || + (mlx4_is_master(dev->dev) && + attr->create_flags & MLX4_IB_SRIOV_SQP)) { + return dev->dev->phys_caps.base_sqpn + + (attr->qp_type == IB_QPT_SMI ? 0 : 2) + + attr->port_num - 1; + } + /* PF or VF -- creating proxies */ + if (attr->qp_type == IB_QPT_SMI) + return dev->dev->caps.qp0_proxy[attr->port_num - 1]; + else + return dev->dev->caps.qp1_proxy[attr->port_num - 1]; +} + +static int check_qpg_attr(struct mlx4_ib_dev *dev, + struct ib_qp_init_attr *attr) +{ + if (attr->qpg_type == IB_QPG_NONE) + return 0; + + if (attr->qp_type != IB_QPT_UD && + attr->qp_type != IB_QPT_RAW_PACKET) + return -EINVAL; + + if (attr->qpg_type == IB_QPG_PARENT) { + if (attr->parent_attrib.tss_child_count == 1) + return -EINVAL; /* Doesn't make sense */ + if (attr->parent_attrib.rss_child_count == 1) + return -EINVAL; /* Doesn't make sense */ + if ((attr->parent_attrib.tss_child_count == 0) && + (attr->parent_attrib.rss_child_count == 0)) + /* Should be called with IP_QPG_NONE */ + return -EINVAL; + if (attr->parent_attrib.rss_child_count > 1) { + int rss_align_num; + if (!(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS)) + return -ENOSYS; + rss_align_num = roundup_pow_of_two( + attr->parent_attrib.rss_child_count); + if (rss_align_num > dev->dev->caps.max_rss_tbl_sz) + return -EINVAL; + } + } else { + struct mlx4_ib_qpg_data *qpg_data; + if (attr->qpg_parent == NULL) + return -EINVAL; + if (IS_ERR(attr->qpg_parent)) + return -EINVAL; + qpg_data = to_mqp(attr->qpg_parent)->qpg_data; + if (qpg_data == NULL) + return -EINVAL; + if (attr->qpg_type == IB_QPG_CHILD_TX && + !qpg_data->tss_child_count) + return -EINVAL; + if (attr->qpg_type == IB_QPG_CHILD_RX && + !qpg_data->rss_child_count) + return -EINVAL; + } + return 0; +} + +#define RESERVED_FLAGS_MASK ((((unsigned int)IB_QP_CREATE_RESERVED_END - 1) | IB_QP_CREATE_RESERVED_END) \ + & ~(IB_QP_CREATE_RESERVED_START - 1)) + +static enum mlx4_ib_qp_flags to_mlx4_ib_qp_flags(enum ib_qp_create_flags ib_qp_flags) +{ + enum mlx4_ib_qp_flags mlx4_ib_qp_flags = 0; + + if (ib_qp_flags & IB_QP_CREATE_IPOIB_UD_LSO) + mlx4_ib_qp_flags |= MLX4_IB_QP_LSO; + + if (ib_qp_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) + mlx4_ib_qp_flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK; + + if (ib_qp_flags & IB_QP_CREATE_NETIF_QP) + mlx4_ib_qp_flags |= MLX4_IB_QP_NETIF; + + if (ib_qp_flags & IB_QP_CREATE_CROSS_CHANNEL) + mlx4_ib_qp_flags |= MLX4_IB_QP_CAP_CROSS_CHANNEL; + + if (ib_qp_flags & IB_QP_CREATE_MANAGED_SEND) + mlx4_ib_qp_flags |= MLX4_IB_QP_CAP_MANAGED_SEND; + + if (ib_qp_flags & IB_QP_CREATE_MANAGED_RECV) + mlx4_ib_qp_flags |= MLX4_IB_QP_CAP_MANAGED_RECV; + + /* reserved flags */ + mlx4_ib_qp_flags |= (ib_qp_flags & RESERVED_FLAGS_MASK); + + return mlx4_ib_qp_flags; +} + +struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + struct mlx4_ib_qp *qp = NULL; + int err; + u16 xrcdn = 0; + enum mlx4_ib_qp_flags mlx4_qp_flags = to_mlx4_ib_qp_flags(init_attr->create_flags); + struct ib_device *device; + + /* see ib_core::ib_create_qp same handling */ + device = pd ? pd->device : init_attr->xrcd->device; + /* + * We only support LSO, vendor flag1, and multicast loopback blocking, + * and only for kernel UD QPs. + */ + if (mlx4_qp_flags & ~(MLX4_IB_QP_LSO | + MLX4_IB_QP_CAP_CROSS_CHANNEL | + MLX4_IB_QP_CAP_MANAGED_SEND | + MLX4_IB_QP_CAP_MANAGED_RECV | + MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK | + MLX4_IB_SRIOV_TUNNEL_QP | MLX4_IB_SRIOV_SQP | + MLX4_IB_QP_NETIF)) + return ERR_PTR(-EINVAL); + + if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) { + if (init_attr->qp_type != IB_QPT_UD) + return ERR_PTR(-EINVAL); + } + + if (mlx4_qp_flags & + (MLX4_IB_QP_CAP_CROSS_CHANNEL | + MLX4_IB_QP_CAP_MANAGED_SEND | + MLX4_IB_QP_CAP_MANAGED_RECV)) { + pr_debug("%s Does not support cross-channel operations\n", + to_mdev(device)->ib_dev.name); + return ERR_PTR(-EINVAL); + } + + if ((init_attr->create_flags & + ~(IB_QP_CREATE_CROSS_CHANNEL | + IB_QP_CREATE_MANAGED_SEND | + IB_QP_CREATE_MANAGED_RECV)) && + (((mlx4_qp_flags & ~MLX4_IB_SRIOV_SQP) && + init_attr->qp_type != IB_QPT_UD) || + ((mlx4_qp_flags & MLX4_IB_SRIOV_SQP) && + init_attr->qp_type > IB_QPT_GSI))) + return ERR_PTR(-EINVAL); + + err = check_qpg_attr(to_mdev(device), init_attr); + if (err) + return ERR_PTR(err); + + switch (init_attr->qp_type) { + case IB_QPT_XRC_TGT: + pd = to_mxrcd(init_attr->xrcd)->pd; + xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn; + init_attr->send_cq = to_mxrcd(init_attr->xrcd)->cq; + /* fall through */ + case IB_QPT_XRC_INI: + if (!(to_mdev(device)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)) + return ERR_PTR(-ENOSYS); + init_attr->recv_cq = init_attr->send_cq; + /* fall through */ + case IB_QPT_RC: + case IB_QPT_UC: + case IB_QPT_RAW_PACKET: + qp = kzalloc(sizeof *qp, GFP_KERNEL); + if (!qp) + return ERR_PTR(-ENOMEM); + qp->pri.vid = qp->alt.vid = 0xFFFF; + /* fall through */ + case IB_QPT_UD: + { + err = create_qp_common(to_mdev(device), pd, init_attr, udata, 0, &qp); + if (err) { + kfree(qp); + return ERR_PTR(err); + } + + qp->ibqp.qp_num = qp->mqp.qpn; + qp->xrcdn = xrcdn; + + break; + } + case IB_QPT_SMI: + case IB_QPT_GSI: + { + /* Userspace is not allowed to create special QPs: */ + if (udata) + return ERR_PTR(-EINVAL); + + err = create_qp_common(to_mdev(device), pd, init_attr, udata, + get_sqp_num(to_mdev(device), init_attr), + &qp); + if (err) + return ERR_PTR(err); + + qp->port = init_attr->port_num; + qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1; + + break; + } + default: + /* Don't support raw QPs */ + return ERR_PTR(-EINVAL); + } + + return &qp->ibqp; +} + +int mlx4_ib_destroy_qp(struct ib_qp *qp) +{ + struct mlx4_ib_dev *dev = to_mdev(qp->device); + struct mlx4_ib_qp *mqp = to_mqp(qp); + struct mlx4_ib_pd *pd; + + if (is_qp0(dev, mqp)) + mlx4_CLOSE_PORT(dev->dev, mqp->port); + + pd = get_pd(mqp); + destroy_qp_common(dev, mqp, !!pd->ibpd.uobject); + + if (is_sqp(dev, mqp)) + kfree(to_msqp(mqp)); + else + kfree(mqp); + + return 0; +} + +static int to_mlx4_st(struct mlx4_ib_dev *dev, enum mlx4_ib_qp_type type) +{ + switch (type) { + case MLX4_IB_QPT_RC: return MLX4_QP_ST_RC; + case MLX4_IB_QPT_UC: return MLX4_QP_ST_UC; + case MLX4_IB_QPT_UD: return MLX4_QP_ST_UD; + case MLX4_IB_QPT_XRC_INI: + case MLX4_IB_QPT_XRC_TGT: return MLX4_QP_ST_XRC; + case MLX4_IB_QPT_SMI: + case MLX4_IB_QPT_GSI: + case MLX4_IB_QPT_RAW_PACKET: return MLX4_QP_ST_MLX; + + case MLX4_IB_QPT_PROXY_SMI_OWNER: + case MLX4_IB_QPT_TUN_SMI_OWNER: return (mlx4_is_mfunc(dev->dev) ? + MLX4_QP_ST_MLX : -1); + case MLX4_IB_QPT_PROXY_SMI: + case MLX4_IB_QPT_TUN_SMI: + case MLX4_IB_QPT_PROXY_GSI: + case MLX4_IB_QPT_TUN_GSI: return (mlx4_is_mfunc(dev->dev) ? + MLX4_QP_ST_UD : -1); + default: return -1; + } +} + +static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr, + int attr_mask) +{ + u8 dest_rd_atomic; + u32 access_flags; + u32 hw_access_flags = 0; + + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) + dest_rd_atomic = attr->max_dest_rd_atomic; + else + dest_rd_atomic = qp->resp_depth; + + if (attr_mask & IB_QP_ACCESS_FLAGS) + access_flags = attr->qp_access_flags; + else + access_flags = qp->atomic_rd_en; + + if (!dest_rd_atomic) + access_flags &= IB_ACCESS_REMOTE_WRITE; + + if (access_flags & IB_ACCESS_REMOTE_READ) + hw_access_flags |= MLX4_QP_BIT_RRE; + if (access_flags & IB_ACCESS_REMOTE_ATOMIC) + hw_access_flags |= MLX4_QP_BIT_RAE; + if (access_flags & IB_ACCESS_REMOTE_WRITE) + hw_access_flags |= MLX4_QP_BIT_RWE; + + return cpu_to_be32(hw_access_flags); +} + +static void store_sqp_attrs(struct mlx4_ib_sqp *sqp, const struct ib_qp_attr *attr, + int attr_mask) +{ + if (attr_mask & IB_QP_PKEY_INDEX) + sqp->pkey_index = attr->pkey_index; + if (attr_mask & IB_QP_QKEY) + sqp->qkey = attr->qkey; + if (attr_mask & IB_QP_SQ_PSN) + sqp->send_psn = attr->sq_psn; +} + +static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port) +{ + path->sched_queue = (path->sched_queue & 0xbf) | ((port - 1) << 6); +} + +static int ib_rate_to_mlx4(struct mlx4_ib_dev *dev, u8 rate) +{ + if (rate == IB_RATE_PORT_CURRENT) { + return 0; + } else if (rate < IB_RATE_2_5_GBPS || rate > IB_RATE_300_GBPS) { + return -EINVAL; + } else { + while (rate != IB_RATE_2_5_GBPS && + !(1 << (rate + MLX4_STAT_RATE_OFFSET) & + dev->dev->caps.stat_rate_support)) + --rate; + } + + return rate + MLX4_STAT_RATE_OFFSET; +} + +static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah, + u8 *smac, u16 vlan_id, struct mlx4_ib_qp *qp, + struct mlx4_qp_path *path, u8 port, int is_primary) +{ + int is_eth = rdma_port_get_link_layer(&dev->ib_dev, port) == + IB_LINK_LAYER_ETHERNET; + u16 vlan_tag; + int vidx; + int smac_index; + int err; + u64 u64_mac; + struct mlx4_roce_smac_vlan_info *smac_info; + + path->grh_mylmc = ah->src_path_bits & 0x7f; + path->rlid = cpu_to_be16(ah->dlid); + + err = ib_rate_to_mlx4(dev, ah->static_rate); + if (err < 0) + return err; + path->static_rate = err; + + if (ah->ah_flags & IB_AH_GRH) { + if (ah->grh.sgid_index >= dev->dev->caps.gid_table_len[port]) { + pr_err("sgid_index (%u) too large. max is %d\n", + ah->grh.sgid_index, dev->dev->caps.gid_table_len[port] - 1); + return -1; + } + + path->grh_mylmc |= 1 << 7; + path->mgid_index = ah->grh.sgid_index; + path->hop_limit = ah->grh.hop_limit; + path->tclass_flowlabel = + cpu_to_be32((ah->grh.traffic_class << 20) | + (ah->grh.flow_label)); + memcpy(path->rgid, ah->grh.dgid.raw, 16); + } + + if (is_eth) { + if (!(ah->ah_flags & IB_AH_GRH)) + return -1; + + path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | + ((port - 1) << 6) | ((ah->sl & 7) << 3); + + if (is_primary) + smac_info = &qp->pri; + else + smac_info = &qp->alt; + + vlan_tag = vlan_id; + if (vlan_tag < 0x1000) { + if (smac_info->vid < 0x1000) { + /* both valid vlan ids */ + if (smac_info->vid != vlan_tag) { + /* different VIDs. unreg old and reg new */ + err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx); + if (err) + return err; + smac_info->candidate_vid = vlan_tag; + smac_info->candidate_vlan_index = vidx; + smac_info->candidate_vlan_port = port; + smac_info->update_vid = 1; + path->vlan_index = vidx; + path->fl = 1 << 6; + } else { + path->vlan_index = smac_info->vlan_index; + path->fl = 1 << 6; + } + } else { + /* no current vlan tag in qp */ + err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx); + if (err) + return err; + smac_info->candidate_vid = vlan_tag; + smac_info->candidate_vlan_index = vidx; + smac_info->candidate_vlan_port = port; + smac_info->update_vid = 1; + path->vlan_index = vidx; + path->fl = 1 << 6; + } + } else { + /* have current vlan tag. unregister it at modify-qp success */ + if (smac_info->vid < 0x1000) { + smac_info->candidate_vid = 0xFFFF; + smac_info->update_vid = 1; + } + } + + + /* get smac_index for RoCE use. + * If no smac was yet assigned, register one. + * If one was already assigned, but the new mac differs, + * unregister the old one and register the new one. + */ + u64_mac = mlx4_mac_to_u64(smac); + + if (!smac_info->smac || smac_info->smac != u64_mac) { + /* register candidate now, unreg if needed, after success */ + smac_index = mlx4_register_mac(dev->dev, port, u64_mac); + if (smac_index >= 0) { + smac_info->candidate_smac_index = smac_index; + smac_info->candidate_smac = u64_mac; + smac_info->candidate_smac_port = port; + } else + return -EINVAL; + } else + smac_index = smac_info->smac_index; + + memcpy(path->dmac, ah->dmac, 6); + path->ackto = MLX4_IB_LINK_TYPE_ETH; + /* put MAC table smac index for IBoE */ + path->grh_mylmc = (u8) (smac_index) | 0x80 ; + + } else + path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | + ((port - 1) << 6) | ((ah->sl & 0xf) << 2); + + return 0; +} + +static void update_mcg_macs(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) +{ + struct mlx4_ib_gid_entry *ge, *tmp; + + list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) { + if (!ge->added && mlx4_ib_add_mc(dev, qp, &ge->gid)) { + ge->added = 1; + ge->port = qp->port; + } + } +} + +static int handle_eth_ud_smac_index(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, const u8 *smac, + struct mlx4_qp_context *context) +{ + struct net_device *ndev; + u64 u64_mac; + int smac_index; + + + ndev = dev->iboe.netdevs[qp->port - 1]; + if (ndev) { + smac = IF_LLADDR(ndev); + u64_mac = mlx4_mac_to_u64(smac); + } else { + u64_mac = dev->dev->caps.def_mac[qp->port]; + } + + context->pri_path.sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | ((qp->port - 1) << 6); + if (!qp->pri.smac) { + smac_index = mlx4_register_mac(dev->dev, qp->port, u64_mac); + if (smac_index >= 0) { + qp->pri.candidate_smac_index = smac_index; + qp->pri.candidate_smac = u64_mac; + qp->pri.candidate_smac_port = qp->port; + context->pri_path.grh_mylmc = 0x80 | (u8) smac_index; + } else + return -ENOENT; + } + return 0; +} +static int __mlx4_ib_modify_qp(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, int attr_mask, + enum ib_qp_state cur_state, enum ib_qp_state new_state) +{ + struct mlx4_ib_dev *dev = to_mdev(ibqp->device); + struct mlx4_ib_qp *qp = to_mqp(ibqp); + struct mlx4_ib_pd *pd; + struct mlx4_ib_cq *send_cq, *recv_cq; + struct mlx4_qp_context *context; + enum mlx4_qp_optpar optpar = 0; + int sqd_event; + int steer_qp = 0; + int err = -EINVAL; + int is_eth = -1; + + context = kzalloc(sizeof *context, GFP_KERNEL); + if (!context) + return -ENOMEM; + + context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) | + (to_mlx4_st(dev, qp->mlx4_ib_qp_type) << 16)); + + if (!(attr_mask & IB_QP_PATH_MIG_STATE)) + context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11); + else { + optpar |= MLX4_QP_OPTPAR_PM_STATE; + switch (attr->path_mig_state) { + case IB_MIG_MIGRATED: + context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11); + break; + case IB_MIG_REARM: + context->flags |= cpu_to_be32(MLX4_QP_PM_REARM << 11); + break; + case IB_MIG_ARMED: + context->flags |= cpu_to_be32(MLX4_QP_PM_ARMED << 11); + break; + } + } + + if (qp->max_inlr_data) + context->param3 |= cpu_to_be32(1 << 25); + + if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI) + context->mtu_msgmax = (IB_MTU_4096 << 5) | 11; + else if (ibqp->qp_type == IB_QPT_RAW_PACKET) + context->mtu_msgmax = (MLX4_RAW_QP_MTU << 5) | MLX4_RAW_QP_MSGMAX; + else if (ibqp->qp_type == IB_QPT_UD) { + if (qp->flags & MLX4_IB_QP_LSO) + context->mtu_msgmax = (IB_MTU_4096 << 5) | + ilog2(dev->dev->caps.max_gso_sz); + else + context->mtu_msgmax = (IB_MTU_4096 << 5) | 12; + } else if (attr_mask & IB_QP_PATH_MTU) { + if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) { + pr_err("path MTU (%u) is invalid\n", + attr->path_mtu); + goto out; + } + context->mtu_msgmax = (attr->path_mtu << 5) | + ilog2(dev->dev->caps.max_msg_sz); + } + + if (qp->rq.wqe_cnt) + context->rq_size_stride = ilog2(qp->rq.wqe_cnt) << 3; + context->rq_size_stride |= qp->rq.wqe_shift - 4; + + if (qp->sq.wqe_cnt) + context->sq_size_stride = ilog2(qp->sq.wqe_cnt) << 3; + context->sq_size_stride |= qp->sq.wqe_shift - 4; + + if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { + context->sq_size_stride |= !!qp->sq_no_prefetch << 7; + context->xrcd = cpu_to_be32((u32) qp->xrcdn); + context->param3 |= cpu_to_be32(1 << 30); + } + + if (qp->ibqp.uobject) + context->usr_page = cpu_to_be32(to_mucontext(ibqp->uobject->context)->uar.index); + else + context->usr_page = cpu_to_be32(qp->bf.uar->index); + + if (attr_mask & IB_QP_DEST_QPN) + context->remote_qpn = cpu_to_be32(attr->dest_qp_num); + + if (attr_mask & IB_QP_PORT) { + if (cur_state == IB_QPS_SQD && new_state == IB_QPS_SQD && + !(attr_mask & IB_QP_AV)) { + mlx4_set_sched(&context->pri_path, attr->port_num); + optpar |= MLX4_QP_OPTPAR_SCHED_QUEUE; + } + } + + if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { + if (dev->counters[qp->port - 1].counter_index != -1) { + context->pri_path.counter_index = + dev->counters[qp->port - 1].counter_index; + optpar |= MLX4_QP_OPTPAR_COUNTER_INDEX; + } else { + context->pri_path.counter_index = 0xff; + } + + if (qp->flags & MLX4_IB_QP_NETIF && + (qp->qpg_type == IB_QPG_NONE || qp->qpg_type == IB_QPG_PARENT)) { + mlx4_ib_steer_qp_reg(dev, qp, 1); + steer_qp = 1; + } + } + + if (attr_mask & IB_QP_PKEY_INDEX) { + if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) + context->pri_path.disable_pkey_check = 0x40; + context->pri_path.pkey_index = attr->pkey_index; + optpar |= MLX4_QP_OPTPAR_PKEY_INDEX; + } + + if ((attr_mask & IB_QP_AV) && (ibqp->qp_type != IB_QPT_RAW_PACKET)) { + if (mlx4_set_path(dev, &attr->ah_attr, (u8 *)attr->smac, + attr_mask & IB_QP_VID ? + attr->vlan_id : 0xffff , + qp, &context->pri_path, + attr_mask & IB_QP_PORT ? + attr->port_num : qp->port, 1)) + goto out; + + optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH | + MLX4_QP_OPTPAR_SCHED_QUEUE); + } + + if (attr_mask & IB_QP_TIMEOUT) { + context->pri_path.ackto |= attr->timeout << 3; + optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT; + } + + if (attr_mask & IB_QP_ALT_PATH) { + if (attr->alt_port_num == 0 || + attr->alt_port_num > dev->dev->caps.num_ports) + goto out; + + if (attr->alt_pkey_index >= + dev->dev->caps.pkey_table_len[attr->alt_port_num]) + goto out; + + if (mlx4_set_path(dev, &attr->alt_ah_attr, (u8 *)attr->smac, + attr_mask & IB_QP_ALT_VID ? + attr->alt_vlan_id : 0xffff, + qp, &context->alt_path, + attr->alt_port_num, 0)) + goto out; + + context->alt_path.pkey_index = attr->alt_pkey_index; + context->alt_path.ackto = attr->alt_timeout << 3; + context->alt_path.counter_index = dev->counters[attr->alt_port_num - 1].counter_index; + optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH; + } + + pd = get_pd(qp); + get_cqs(qp, &send_cq, &recv_cq); + context->pd = cpu_to_be32(pd->pdn); + context->cqn_send = cpu_to_be32(send_cq->mcq.cqn); + context->cqn_recv = cpu_to_be32(recv_cq->mcq.cqn); + context->params1 = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28); + + /* Set "fast registration enabled" for all kernel QPs */ + if (!qp->ibqp.uobject) + context->params1 |= cpu_to_be32(1 << 11); + + if (attr_mask & IB_QP_RNR_RETRY) { + context->params1 |= cpu_to_be32(attr->rnr_retry << 13); + optpar |= MLX4_QP_OPTPAR_RNR_RETRY; + } + + if (attr_mask & IB_QP_RETRY_CNT) { + context->params1 |= cpu_to_be32(attr->retry_cnt << 16); + optpar |= MLX4_QP_OPTPAR_RETRY_COUNT; + } + + if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { + if (attr->max_rd_atomic) + context->params1 |= + cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21); + optpar |= MLX4_QP_OPTPAR_SRA_MAX; + } + + if (attr_mask & IB_QP_SQ_PSN) + context->next_send_psn = cpu_to_be32(attr->sq_psn); + + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { + if (attr->max_dest_rd_atomic) + context->params2 |= + cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21); + optpar |= MLX4_QP_OPTPAR_RRA_MAX; + } + + if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) { + context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask); + optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE; + } + + if (ibqp->srq) + context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC); + + if (attr_mask & IB_QP_MIN_RNR_TIMER) { + context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24); + optpar |= MLX4_QP_OPTPAR_RNR_TIMEOUT; + } + if (attr_mask & IB_QP_RQ_PSN) + context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn); + + /* proxy and tunnel qp qkeys will be changed in modify-qp wrappers */ + if (attr_mask & IB_QP_QKEY) { + if (qp->mlx4_ib_qp_type & + (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) + context->qkey = cpu_to_be32(IB_QP_SET_QKEY); + else { + if (mlx4_is_mfunc(dev->dev) && + !(qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) && + (attr->qkey & MLX4_RESERVED_QKEY_MASK) == + MLX4_RESERVED_QKEY_BASE) { + pr_err("Cannot use reserved QKEY" + " 0x%x (range 0xffff0000..0xffffffff" + " is reserved)\n", attr->qkey); + err = -EINVAL; + goto out; + } + context->qkey = cpu_to_be32(attr->qkey); + } + optpar |= MLX4_QP_OPTPAR_Q_KEY; + } + + if (ibqp->srq) + context->srqn = cpu_to_be32(1 << 24 | to_msrq(ibqp->srq)->msrq.srqn); + + if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) + context->db_rec_addr = cpu_to_be64(qp->db.dma); + + if (cur_state == IB_QPS_INIT && + new_state == IB_QPS_RTR && + (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI || + ibqp->qp_type == IB_QPT_UD || + ibqp->qp_type == IB_QPT_RAW_PACKET)) { + context->pri_path.sched_queue = (qp->port - 1) << 6; + if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI || + qp->mlx4_ib_qp_type & + (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) { + context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE; + if (qp->mlx4_ib_qp_type != MLX4_IB_QPT_SMI) + context->pri_path.fl = 0x80; + } else { + if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) + context->pri_path.fl = 0x80; + context->pri_path.sched_queue |= MLX4_IB_DEFAULT_SCHED_QUEUE; + } + if (ibqp->qp_type == IB_QPT_RAW_PACKET && + (attr_mask & IB_QP_AV)) { + context->pri_path.sched_queue |= + ((attr->ah_attr.sl & 0xf) << 3); + context->pri_path.feup = 1 << 6; + } + is_eth = rdma_port_get_link_layer(&dev->ib_dev, qp->port) == + IB_LINK_LAYER_ETHERNET; + if (is_eth) { + if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI || + qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) + context->pri_path.feup = 1 << 7; /* don't fsm */ + /* handle smac_index */ + if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_UD || + qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI || + qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI) { + err = handle_eth_ud_smac_index(dev, qp, (const u8 *)attr->smac, context); + if (err) + return -EINVAL; + } + } + } + + if (ibqp->qp_type == IB_QPT_UD) + if (is_eth && (new_state == IB_QPS_RTR)) { + context->pri_path.ackto = MLX4_IB_LINK_TYPE_ETH; + optpar |= MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH; + } + + if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD && + attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify) + sqd_event = 1; + else + sqd_event = 0; + + if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) + context->rlkey_roce_mode |= (1 << 4); + + if ((attr_mask & IB_QP_GROUP_RSS) && + (qp->qpg_data->rss_child_count > 1)) { + struct mlx4_ib_qpg_data *qpg_data = qp->qpg_data; + void *rss_context_base = &context->pri_path; + struct mlx4_rss_context *rss_context = + (struct mlx4_rss_context *) (rss_context_base + + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH); + + context->flags |= cpu_to_be32(1 << MLX4_RSS_QPC_FLAG_OFFSET); + + /* This should be tbl_sz_base_qpn */ + rss_context->base_qpn = cpu_to_be32(qpg_data->rss_qpn_base | + (ilog2(qpg_data->rss_child_count) << 24)); + rss_context->default_qpn = cpu_to_be32(qpg_data->rss_qpn_base); + /* This should be flags_hash_fn */ + rss_context->flags = MLX4_RSS_TCP_IPV6 | + MLX4_RSS_TCP_IPV4; + if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UDP_RSS) { + rss_context->base_qpn_udp = rss_context->default_qpn; + rss_context->flags |= MLX4_RSS_IPV6 | + MLX4_RSS_IPV4 | + MLX4_RSS_UDP_IPV6 | + MLX4_RSS_UDP_IPV4; + } + if (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_TOP) { + static const u32 rsskey[10] = { 0xD181C62C, 0xF7F4DB5B, + 0x1983A2FC, 0x943E1ADB, 0xD9389E6B, 0xD1039C2C, + 0xA74499AD, 0x593D56D9, 0xF3253C06, 0x2ADC1FFC}; + rss_context->hash_fn = MLX4_RSS_HASH_TOP; + memcpy(rss_context->rss_key, rsskey, + sizeof(rss_context->rss_key)); + } else { + rss_context->hash_fn = MLX4_RSS_HASH_XOR; + memset(rss_context->rss_key, 0, + sizeof(rss_context->rss_key)); + } + } + /* + * Before passing a kernel QP to the HW, make sure that the + * ownership bits of the send queue are set and the SQ + * headroom is stamped so that the hardware doesn't start + * processing stale work requests. + */ + if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { + struct mlx4_wqe_ctrl_seg *ctrl; + int i; + + for (i = 0; i < qp->sq.wqe_cnt; ++i) { + ctrl = get_send_wqe(qp, i); + ctrl->owner_opcode = cpu_to_be32(1 << 31); + if (qp->sq_max_wqes_per_wr == 1) + ctrl->fence_size = 1 << (qp->sq.wqe_shift - 4); + + stamp_send_wqe(qp, i, 1 << qp->sq.wqe_shift); + } + } + + if ((qp->port && rdma_port_get_link_layer(&dev->ib_dev, qp->port) == + IB_LINK_LAYER_ETHERNET) && (qp->ibqp.qp_type == IB_QPT_RAW_PACKET)) + context->pri_path.ackto = (context->pri_path.ackto & 0xf8) | + MLX4_IB_LINK_TYPE_ETH; + + err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state), + to_mlx4_state(new_state), context, optpar, + sqd_event, &qp->mqp); + if (err) + goto out; + + qp->state = new_state; + + if (attr_mask & IB_QP_ACCESS_FLAGS) + qp->atomic_rd_en = attr->qp_access_flags; + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) + qp->resp_depth = attr->max_dest_rd_atomic; + if (attr_mask & IB_QP_PORT) { + qp->port = attr->port_num; + update_mcg_macs(dev, qp); + } + if (attr_mask & IB_QP_ALT_PATH) + qp->alt_port = attr->alt_port_num; + + if (is_sqp(dev, qp)) + store_sqp_attrs(to_msqp(qp), attr, attr_mask); + + /* + * If we moved QP0 to RTR, bring the IB link up; if we moved + * QP0 to RESET or ERROR, bring the link back down. + */ + if (is_qp0(dev, qp)) { + if (cur_state != IB_QPS_RTR && new_state == IB_QPS_RTR) + if (mlx4_INIT_PORT(dev->dev, qp->port)) + pr_warn("INIT_PORT failed for port %d\n", + qp->port); + + if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR && + (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR)) + mlx4_CLOSE_PORT(dev->dev, qp->port); + } + + /* + * If we moved a kernel QP to RESET, clean up all old CQ + * entries and reinitialize the QP. + */ + if (new_state == IB_QPS_RESET) { + if (!ibqp->uobject) { + mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn, + ibqp->srq ? to_msrq(ibqp->srq) : NULL); + if (send_cq != recv_cq) + mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL); + + qp->rq.head = 0; + qp->rq.tail = 0; + qp->sq.head = 0; + qp->sq.tail = 0; + qp->sq_next_wqe = 0; + if (qp->rq.wqe_cnt) + *qp->db.db = 0; + + if (qp->flags & MLX4_IB_QP_NETIF && + (qp->qpg_type == IB_QPG_NONE || + qp->qpg_type == IB_QPG_PARENT)) + mlx4_ib_steer_qp_reg(dev, qp, 0); + } + if (qp->pri.smac) { + mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac); + qp->pri.smac = 0; + } + if (qp->alt.smac) { + mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac); + qp->alt.smac = 0; + } + if (qp->pri.vid < 0x1000) { + mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid); + qp->pri.vid = 0xFFFF; + qp->pri.candidate_vid = 0xFFFF; + qp->pri.update_vid = 0; + } + + if (qp->alt.vid < 0x1000) { + mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid); + qp->alt.vid = 0xFFFF; + qp->alt.candidate_vid = 0xFFFF; + qp->alt.update_vid = 0; + } + } + +out: + if (err && steer_qp) + mlx4_ib_steer_qp_reg(dev, qp, 0); + kfree(context); + if (qp->pri.candidate_smac) { + if (err) + mlx4_unregister_mac(dev->dev, qp->pri.candidate_smac_port, qp->pri.candidate_smac); + else { + if (qp->pri.smac) { + mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac); + } + qp->pri.smac = qp->pri.candidate_smac; + qp->pri.smac_index = qp->pri.candidate_smac_index; + qp->pri.smac_port = qp->pri.candidate_smac_port; + + } + qp->pri.candidate_smac = 0; + qp->pri.candidate_smac_index = 0; + qp->pri.candidate_smac_port = 0; + } + if (qp->alt.candidate_smac) { + if (err) + mlx4_unregister_mac(dev->dev, qp->alt.candidate_smac_port, qp->pri.candidate_smac); + else { + if (qp->pri.smac) { + mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac); + } + qp->alt.smac = qp->alt.candidate_smac; + qp->alt.smac_index = qp->alt.candidate_smac_index; + qp->alt.smac_port = qp->alt.candidate_smac_port; + + } + qp->pri.candidate_smac = 0; + qp->pri.candidate_smac_index = 0; + qp->pri.candidate_smac_port = 0; + } + + if (qp->pri.update_vid) { + if (err) { + if (qp->pri.candidate_vid < 0x1000) + mlx4_unregister_vlan(dev->dev, qp->pri.candidate_vlan_port, + qp->pri.candidate_vid); + } else { + if (qp->pri.vid < 0x1000) + mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, + qp->pri.vid); + qp->pri.vid = qp->pri.candidate_vid; + qp->pri.vlan_port = qp->pri.candidate_vlan_port; + qp->pri.vlan_index = qp->pri.candidate_vlan_index; + } + qp->pri.candidate_vid = 0xFFFF; + qp->pri.update_vid = 0; + } + + if (qp->alt.update_vid) { + if (err) { + if (qp->alt.candidate_vid < 0x1000) + mlx4_unregister_vlan(dev->dev, qp->alt.candidate_vlan_port, + qp->alt.candidate_vid); + } else { + if (qp->alt.vid < 0x1000) + mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, + qp->alt.vid); + qp->alt.vid = qp->alt.candidate_vid; + qp->alt.vlan_port = qp->alt.candidate_vlan_port; + qp->alt.vlan_index = qp->alt.candidate_vlan_index; + } + qp->alt.candidate_vid = 0xFFFF; + qp->alt.update_vid = 0; + } + + return err; +} + +int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + struct mlx4_ib_dev *dev = to_mdev(ibqp->device); + struct mlx4_ib_qp *qp = to_mqp(ibqp); + enum ib_qp_state cur_state, new_state; + int err = -EINVAL; + int ll; + + mutex_lock(&qp->mutex); + + cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state; + new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; + + if (cur_state == new_state && cur_state == IB_QPS_RESET) { + ll = IB_LINK_LAYER_UNSPECIFIED; + } else { + int port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port; + ll = rdma_port_get_link_layer(&dev->ib_dev, port); + } + + if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask, ll)) { + pr_debug("qpn 0x%x: invalid attribute mask specified " + "for transition %d to %d. qp_type %d," + " attr_mask 0x%x\n", + ibqp->qp_num, cur_state, new_state, + ibqp->qp_type, attr_mask); + goto out; + } + + if ((attr_mask & IB_QP_PORT) && + (attr->port_num == 0 || attr->port_num > dev->num_ports)) { + pr_debug("qpn 0x%x: invalid port number (%d) specified " + "for transition %d to %d. qp_type %d\n", + ibqp->qp_num, attr->port_num, cur_state, + new_state, ibqp->qp_type); + goto out; + } + + if (attr_mask & IB_QP_PKEY_INDEX) { + int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port; + if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p]) { + pr_debug("qpn 0x%x: invalid pkey index (%d) specified " + "for transition %d to %d. qp_type %d\n", + ibqp->qp_num, attr->pkey_index, cur_state, + new_state, ibqp->qp_type); + goto out; + } + } + + if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && + attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) { + pr_debug("qpn 0x%x: max_rd_atomic (%d) too large. " + "Transition %d to %d. qp_type %d\n", + ibqp->qp_num, attr->max_rd_atomic, cur_state, + new_state, ibqp->qp_type); + goto out; + } + + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && + attr->max_dest_rd_atomic > dev->dev->caps.max_qp_dest_rdma) { + pr_debug("qpn 0x%x: max_dest_rd_atomic (%d) too large. " + "Transition %d to %d. qp_type %d\n", + ibqp->qp_num, attr->max_dest_rd_atomic, cur_state, + new_state, ibqp->qp_type); + goto out; + } + + if (cur_state == new_state && cur_state == IB_QPS_RESET) { + err = 0; + goto out; + } + + err = __mlx4_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state); + +out: + mutex_unlock(&qp->mutex); + return err; +} + +static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp, + struct ib_send_wr *wr, + void *wqe, unsigned *mlx_seg_len) +{ + struct mlx4_ib_dev *mdev = to_mdev(sqp->qp.ibqp.device); + struct ib_device *ib_dev = &mdev->ib_dev; + struct mlx4_wqe_mlx_seg *mlx = wqe; + struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx; + struct mlx4_ib_ah *ah = to_mah(wr->wr.ud.ah); + u16 pkey; + u32 qkey; + int send_size; + int header_size; + int spc; + int i; + + if (wr->opcode != IB_WR_SEND) + return -EINVAL; + + send_size = 0; + + for (i = 0; i < wr->num_sge; ++i) + send_size += wr->sg_list[i].length; + + /* for proxy-qp0 sends, need to add in size of tunnel header */ + /* for tunnel-qp0 sends, tunnel header is already in s/g list */ + if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) + send_size += sizeof (struct mlx4_ib_tunnel_header); + + ib_ud_header_init(send_size, 1, 0, 0, 0, 0, &sqp->ud_header); + + if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) { + sqp->ud_header.lrh.service_level = + be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28; + sqp->ud_header.lrh.destination_lid = + cpu_to_be16(ah->av.ib.g_slid & 0x7f); + sqp->ud_header.lrh.source_lid = + cpu_to_be16(ah->av.ib.g_slid & 0x7f); + } + + mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); + + /* force loopback */ + mlx->flags |= cpu_to_be32(MLX4_WQE_MLX_VL15 | 0x1 | MLX4_WQE_MLX_SLR); + mlx->rlid = sqp->ud_header.lrh.destination_lid; + + sqp->ud_header.lrh.virtual_lane = 0; + sqp->ud_header.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED); + ib_get_cached_pkey(ib_dev, sqp->qp.port, 0, &pkey); + sqp->ud_header.bth.pkey = cpu_to_be16(pkey); + if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER) + sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn); + else + sqp->ud_header.bth.destination_qpn = + cpu_to_be32(mdev->dev->caps.qp0_tunnel[sqp->qp.port - 1]); + + sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1)); + if (mlx4_get_parav_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey)) + return -EINVAL; + sqp->ud_header.deth.qkey = cpu_to_be32(qkey); + sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.mqp.qpn); + + sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY; + sqp->ud_header.immediate_present = 0; + + header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf); + + /* + * Inline data segments may not cross a 64 byte boundary. If + * our UD header is bigger than the space available up to the + * next 64 byte boundary in the WQE, use two inline data + * segments to hold the UD header. + */ + spc = MLX4_INLINE_ALIGN - + ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1)); + if (header_size <= spc) { + inl->byte_count = cpu_to_be32(1 << 31 | header_size); + memcpy(inl + 1, sqp->header_buf, header_size); + i = 1; + } else { + inl->byte_count = cpu_to_be32(1 << 31 | spc); + memcpy(inl + 1, sqp->header_buf, spc); + + inl = (void *) (inl + 1) + spc; + memcpy(inl + 1, sqp->header_buf + spc, header_size - spc); + /* + * Need a barrier here to make sure all the data is + * visible before the byte_count field is set. + * Otherwise the HCA prefetcher could grab the 64-byte + * chunk with this inline segment and get a valid (!= + * 0xffffffff) byte count but stale data, and end up + * generating a packet with bad headers. + * + * The first inline segment's byte_count field doesn't + * need a barrier, because it comes after a + * control/MLX segment and therefore is at an offset + * of 16 mod 64. + */ + wmb(); + inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc)); + i = 2; + } + + *mlx_seg_len = + ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16); + return 0; +} + +static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr, + void *wqe, unsigned *mlx_seg_len) +{ + struct ib_device *ib_dev = sqp->qp.ibqp.device; + struct mlx4_wqe_mlx_seg *mlx = wqe; + struct mlx4_wqe_ctrl_seg *ctrl = wqe; + struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx; + struct mlx4_ib_ah *ah = to_mah(wr->wr.ud.ah); + union ib_gid sgid; + u16 pkey; + int send_size; + int header_size; + int spc; + int i; + int is_eth; + int is_vlan = 0; + int is_grh; + u16 uninitialized_var(vlan); + int err = 0; + + send_size = 0; + for (i = 0; i < wr->num_sge; ++i) + send_size += wr->sg_list[i].length; + + is_eth = rdma_port_get_link_layer(sqp->qp.ibqp.device, sqp->qp.port) == IB_LINK_LAYER_ETHERNET; + is_grh = mlx4_ib_ah_grh_present(ah); + if (is_eth) { + if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) { + /* When multi-function is enabled, the ib_core gid + * indexes don't necessarily match the hw ones, so + * we must use our own cache */ + err = mlx4_get_roce_gid_from_slave(to_mdev(ib_dev)->dev, + be32_to_cpu(ah->av.ib.port_pd) >> 24, + ah->av.ib.gid_index, &sgid.raw[0]); + if (err) + return err; + } else { + err = ib_get_cached_gid(ib_dev, + be32_to_cpu(ah->av.ib.port_pd) >> 24, + ah->av.ib.gid_index, &sgid); + if (err) + return err; + } + + if (is_eth && ah->av.eth.vlan != 0xffff) { + vlan = cpu_to_be16(ah->av.eth.vlan) & 0x0fff; + is_vlan = 1; + } + } + ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh, 0, &sqp->ud_header); + + if (!is_eth) { + sqp->ud_header.lrh.service_level = + be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28; + sqp->ud_header.lrh.destination_lid = ah->av.ib.dlid; + sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.ib.g_slid & 0x7f); + } + + if (is_grh) { + sqp->ud_header.grh.traffic_class = + (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff; + sqp->ud_header.grh.flow_label = + ah->av.ib.sl_tclass_flowlabel & cpu_to_be32(0xfffff); + sqp->ud_header.grh.hop_limit = ah->av.ib.hop_limit; + if (is_eth) + memcpy(sqp->ud_header.grh.source_gid.raw, sgid.raw, 16); + else { + if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) { + /* When multi-function is enabled, the ib_core gid + * indexes don't necessarily match the hw ones, so + * we must use our own cache */ + sqp->ud_header.grh.source_gid.global.subnet_prefix = + to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1]. + subnet_prefix; + sqp->ud_header.grh.source_gid.global.interface_id = + to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1]. + guid_cache[ah->av.ib.gid_index]; + } else + ib_get_cached_gid(ib_dev, + be32_to_cpu(ah->av.ib.port_pd) >> 24, + ah->av.ib.gid_index, + &sqp->ud_header.grh.source_gid); + } + memcpy(sqp->ud_header.grh.destination_gid.raw, + ah->av.ib.dgid, 16); + } + + mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); + + if (!is_eth) { + mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) | + (sqp->ud_header.lrh.destination_lid == + IB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) | + (sqp->ud_header.lrh.service_level << 8)); + if (ah->av.ib.port_pd & cpu_to_be32(0x80000000)) + mlx->flags |= cpu_to_be32(0x1); /* force loopback */ + mlx->rlid = sqp->ud_header.lrh.destination_lid; + } + + switch (wr->opcode) { + case IB_WR_SEND: + sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY; + sqp->ud_header.immediate_present = 0; + break; + case IB_WR_SEND_WITH_IMM: + sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; + sqp->ud_header.immediate_present = 1; + sqp->ud_header.immediate_data = wr->ex.imm_data; + break; + default: + return -EINVAL; + } + + if (is_eth) { + u8 *smac; + struct in6_addr in6; + + u16 pcp = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 29) << 13; + + mlx->sched_prio = cpu_to_be16(pcp); + + memcpy(sqp->ud_header.eth.dmac_h, ah->av.eth.mac, 6); + /* FIXME: cache smac value? */ + memcpy(&ctrl->srcrb_flags16[0], ah->av.eth.mac, 2); + memcpy(&ctrl->imm, ah->av.eth.mac + 2, 4); + memcpy(&in6, sgid.raw, sizeof(in6)); + + if (!mlx4_is_mfunc(to_mdev(ib_dev)->dev)) + smac = IF_LLADDR(to_mdev(sqp->qp.ibqp.device)->iboe.netdevs[sqp->qp.port - 1]); + else + smac = ah->av.eth.s_mac; /* use the src mac of the tunnel */ + memcpy(sqp->ud_header.eth.smac_h, smac, 6); + + if (!memcmp(sqp->ud_header.eth.smac_h, sqp->ud_header.eth.dmac_h, 6)) + mlx->flags |= cpu_to_be32(MLX4_WQE_CTRL_FORCE_LOOPBACK); + if (!is_vlan) { + sqp->ud_header.eth.type = cpu_to_be16(MLX4_IB_IBOE_ETHERTYPE); + } else { + sqp->ud_header.vlan.type = cpu_to_be16(MLX4_IB_IBOE_ETHERTYPE); + sqp->ud_header.vlan.tag = cpu_to_be16(vlan | pcp); + } + } else { + sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0; + if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE) + sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE; + } + sqp->ud_header.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED); + if (!sqp->qp.ibqp.qp_num) + ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey); + else + ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->wr.ud.pkey_index, &pkey); + sqp->ud_header.bth.pkey = cpu_to_be16(pkey); + sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn); + sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1)); + sqp->ud_header.deth.qkey = cpu_to_be32(wr->wr.ud.remote_qkey & 0x80000000 ? + sqp->qkey : wr->wr.ud.remote_qkey); + sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num); + + header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf); + + if (0) { + pr_err("built UD header of size %d:\n", header_size); + for (i = 0; i < header_size / 4; ++i) { + if (i % 8 == 0) + pr_err(" [%02x] ", i * 4); + pr_cont(" %08x", + be32_to_cpu(((__be32 *) sqp->header_buf)[i])); + if ((i + 1) % 8 == 0) + pr_cont("\n"); + } + pr_err("\n"); + } + + /* + * Inline data segments may not cross a 64 byte boundary. If + * our UD header is bigger than the space available up to the + * next 64 byte boundary in the WQE, use two inline data + * segments to hold the UD header. + */ + spc = MLX4_INLINE_ALIGN - + ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1)); + if (header_size <= spc) { + inl->byte_count = cpu_to_be32(1 << 31 | header_size); + memcpy(inl + 1, sqp->header_buf, header_size); + i = 1; + } else { + inl->byte_count = cpu_to_be32(1 << 31 | spc); + memcpy(inl + 1, sqp->header_buf, spc); + + inl = (void *) (inl + 1) + spc; + memcpy(inl + 1, sqp->header_buf + spc, header_size - spc); + /* + * Need a barrier here to make sure all the data is + * visible before the byte_count field is set. + * Otherwise the HCA prefetcher could grab the 64-byte + * chunk with this inline segment and get a valid (!= + * 0xffffffff) byte count but stale data, and end up + * generating a packet with bad headers. + * + * The first inline segment's byte_count field doesn't + * need a barrier, because it comes after a + * control/MLX segment and therefore is at an offset + * of 16 mod 64. + */ + wmb(); + inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc)); + i = 2; + } + + *mlx_seg_len = + ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16); + return 0; +} + +static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq) +{ + unsigned cur; + struct mlx4_ib_cq *cq; + + cur = wq->head - wq->tail; + if (likely(cur + nreq < wq->max_post)) + return 0; + + cq = to_mcq(ib_cq); + spin_lock(&cq->lock); + cur = wq->head - wq->tail; + spin_unlock(&cq->lock); + + return cur + nreq >= wq->max_post; +} + +static __be32 convert_access(int acc) +{ + return (acc & IB_ACCESS_REMOTE_ATOMIC ? + cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_ATOMIC) : 0) | + (acc & IB_ACCESS_REMOTE_WRITE ? + cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_WRITE) : 0) | + (acc & IB_ACCESS_REMOTE_READ ? + cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_READ) : 0) | + (acc & IB_ACCESS_LOCAL_WRITE ? cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_WRITE) : 0) | + cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_READ); +} + +static void set_fmr_seg(struct mlx4_wqe_fmr_seg *fseg, struct ib_send_wr *wr) +{ + struct mlx4_ib_fast_reg_page_list *mfrpl = to_mfrpl(wr->wr.fast_reg.page_list); + int i; + + for (i = 0; i < wr->wr.fast_reg.page_list_len; ++i) + mfrpl->mapped_page_list[i] = + cpu_to_be64(wr->wr.fast_reg.page_list->page_list[i] | + MLX4_MTT_FLAG_PRESENT); + + fseg->flags = convert_access(wr->wr.fast_reg.access_flags); + fseg->mem_key = cpu_to_be32(wr->wr.fast_reg.rkey); + fseg->buf_list = cpu_to_be64(mfrpl->map); + fseg->start_addr = cpu_to_be64(wr->wr.fast_reg.iova_start); + fseg->reg_len = cpu_to_be64(wr->wr.fast_reg.length); + fseg->offset = 0; /* XXX -- is this just for ZBVA? */ + fseg->page_size = cpu_to_be32(wr->wr.fast_reg.page_shift); + fseg->reserved[0] = 0; + fseg->reserved[1] = 0; +} + +static void set_bind_seg(struct mlx4_wqe_bind_seg *bseg, struct ib_send_wr *wr) +{ + bseg->flags1 = + convert_access(wr->wr.bind_mw.bind_info.mw_access_flags) & + cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_READ | + MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_WRITE | + MLX4_WQE_FMR_AND_BIND_PERM_ATOMIC); + bseg->flags2 = 0; + if (wr->wr.bind_mw.mw->type == IB_MW_TYPE_2) + bseg->flags2 |= cpu_to_be32(MLX4_WQE_BIND_TYPE_2); + if (wr->wr.bind_mw.bind_info.mw_access_flags & IB_ZERO_BASED) + bseg->flags2 |= cpu_to_be32(MLX4_WQE_BIND_ZERO_BASED); + bseg->new_rkey = cpu_to_be32(wr->wr.bind_mw.rkey); + bseg->lkey = cpu_to_be32(wr->wr.bind_mw.bind_info.mr->lkey); + bseg->addr = cpu_to_be64(wr->wr.bind_mw.bind_info.addr); + bseg->length = cpu_to_be64(wr->wr.bind_mw.bind_info.length); +} + +static void set_local_inv_seg(struct mlx4_wqe_local_inval_seg *iseg, u32 rkey) +{ + iseg->mem_key = cpu_to_be32(rkey); + + iseg->reserved1 = 0; + iseg->reserved2 = 0; + iseg->reserved3[0] = 0; + iseg->reserved3[1] = 0; +} + +static __always_inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg, + u64 remote_addr, u32 rkey) +{ + rseg->raddr = cpu_to_be64(remote_addr); + rseg->rkey = cpu_to_be32(rkey); + rseg->reserved = 0; +} + +static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg, struct ib_send_wr *wr) +{ + if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) { + aseg->swap_add = cpu_to_be64(wr->wr.atomic.swap); + aseg->compare = cpu_to_be64(wr->wr.atomic.compare_add); + } else if (wr->opcode == IB_WR_MASKED_ATOMIC_FETCH_AND_ADD) { + aseg->swap_add = cpu_to_be64(wr->wr.atomic.compare_add); + aseg->compare = cpu_to_be64(wr->wr.atomic.compare_add_mask); + } else { + aseg->swap_add = cpu_to_be64(wr->wr.atomic.compare_add); + aseg->compare = 0; + } + +} + +static void set_masked_atomic_seg(struct mlx4_wqe_masked_atomic_seg *aseg, + struct ib_send_wr *wr) +{ + aseg->swap_add = cpu_to_be64(wr->wr.atomic.swap); + aseg->swap_add_mask = cpu_to_be64(wr->wr.atomic.swap_mask); + aseg->compare = cpu_to_be64(wr->wr.atomic.compare_add); + aseg->compare_mask = cpu_to_be64(wr->wr.atomic.compare_add_mask); +} + +static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg, + struct ib_send_wr *wr) +{ + memcpy(dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof (struct mlx4_av)); + dseg->dqpn = cpu_to_be32(wr->wr.ud.remote_qpn); + dseg->qkey = cpu_to_be32(wr->wr.ud.remote_qkey); + dseg->vlan = to_mah(wr->wr.ud.ah)->av.eth.vlan; + memcpy(dseg->mac, to_mah(wr->wr.ud.ah)->av.eth.mac, 6); +} + +static void set_tunnel_datagram_seg(struct mlx4_ib_dev *dev, + struct mlx4_wqe_datagram_seg *dseg, + struct ib_send_wr *wr, enum ib_qp_type qpt) +{ + union mlx4_ext_av *av = &to_mah(wr->wr.ud.ah)->av; + struct mlx4_av sqp_av = {0}; + int port = *((u8 *) &av->ib.port_pd) & 0x3; + + /* force loopback */ + sqp_av.port_pd = av->ib.port_pd | cpu_to_be32(0x80000000); + sqp_av.g_slid = av->ib.g_slid & 0x7f; /* no GRH */ + sqp_av.sl_tclass_flowlabel = av->ib.sl_tclass_flowlabel & + cpu_to_be32(0xf0000000); + + memcpy(dseg->av, &sqp_av, sizeof (struct mlx4_av)); + /* This function used only for sending on QP1 proxies */ + dseg->dqpn = cpu_to_be32(dev->dev->caps.qp1_tunnel[port - 1]); + /* Use QKEY from the QP context, which is set by master */ + dseg->qkey = cpu_to_be32(IB_QP_SET_QKEY); +} + +static void build_tunnel_header(struct ib_send_wr *wr, void *wqe, unsigned *mlx_seg_len) +{ + struct mlx4_wqe_inline_seg *inl = wqe; + struct mlx4_ib_tunnel_header hdr; + struct mlx4_ib_ah *ah = to_mah(wr->wr.ud.ah); + int spc; + int i; + + memcpy(&hdr.av, &ah->av, sizeof hdr.av); + hdr.remote_qpn = cpu_to_be32(wr->wr.ud.remote_qpn); + hdr.pkey_index = cpu_to_be16(wr->wr.ud.pkey_index); + hdr.qkey = cpu_to_be32(wr->wr.ud.remote_qkey); + memcpy(hdr.mac, ah->av.eth.mac, 6); + hdr.vlan = cpu_to_be16(ah->av.eth.vlan); + + spc = MLX4_INLINE_ALIGN - + ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1)); + if (sizeof (hdr) <= spc) { + memcpy(inl + 1, &hdr, sizeof (hdr)); + wmb(); + inl->byte_count = cpu_to_be32(1 << 31 | sizeof (hdr)); + i = 1; + } else { + memcpy(inl + 1, &hdr, spc); + wmb(); + inl->byte_count = cpu_to_be32(1 << 31 | spc); + + inl = (void *) (inl + 1) + spc; + memcpy(inl + 1, (void *) &hdr + spc, sizeof (hdr) - spc); + wmb(); + inl->byte_count = cpu_to_be32(1 << 31 | (sizeof (hdr) - spc)); + i = 2; + } + + *mlx_seg_len = + ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + sizeof (hdr), 16); +} + +static void set_mlx_icrc_seg(void *dseg) +{ + u32 *t = dseg; + struct mlx4_wqe_inline_seg *iseg = dseg; + + t[1] = 0; + + /* + * Need a barrier here before writing the byte_count field to + * make sure that all the data is visible before the + * byte_count field is set. Otherwise, if the segment begins + * a new cacheline, the HCA prefetcher could grab the 64-byte + * chunk and get a valid (!= * 0xffffffff) byte count but + * stale data, and end up sending the wrong data. + */ + wmb(); + + iseg->byte_count = cpu_to_be32((1 << 31) | 4); +} + +static void set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg) +{ + dseg->lkey = cpu_to_be32(sg->lkey); + dseg->addr = cpu_to_be64(sg->addr); + + /* + * Need a barrier here before writing the byte_count field to + * make sure that all the data is visible before the + * byte_count field is set. Otherwise, if the segment begins + * a new cacheline, the HCA prefetcher could grab the 64-byte + * chunk and get a valid (!= * 0xffffffff) byte count but + * stale data, and end up sending the wrong data. + */ + wmb(); + + dseg->byte_count = cpu_to_be32(sg->length); +} + +static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg) +{ + dseg->byte_count = cpu_to_be32(sg->length); + dseg->lkey = cpu_to_be32(sg->lkey); + dseg->addr = cpu_to_be64(sg->addr); +} + +static int build_lso_seg(struct mlx4_wqe_lso_seg *wqe, struct ib_send_wr *wr, + struct mlx4_ib_qp *qp, unsigned *lso_seg_len, + __be32 *lso_hdr_sz, __be32 *blh) +{ + unsigned halign = ALIGN(sizeof *wqe + wr->wr.ud.hlen, 16); + + if (unlikely(halign > MLX4_IB_CACHE_LINE_SIZE)) + *blh = cpu_to_be32(1 << 6); + + if (unlikely(!(qp->flags & MLX4_IB_QP_LSO) && + wr->num_sge > qp->sq.max_gs - (halign >> 4))) + return -EINVAL; + + memcpy(wqe->header, wr->wr.ud.header, wr->wr.ud.hlen); + + *lso_hdr_sz = cpu_to_be32((wr->wr.ud.mss - wr->wr.ud.hlen) << 16 | + wr->wr.ud.hlen); + *lso_seg_len = halign; + return 0; +} + +static __be32 send_ieth(struct ib_send_wr *wr) +{ + switch (wr->opcode) { + case IB_WR_SEND_WITH_IMM: + case IB_WR_RDMA_WRITE_WITH_IMM: + return wr->ex.imm_data; + + case IB_WR_SEND_WITH_INV: + return cpu_to_be32(wr->ex.invalidate_rkey); + + default: + return 0; + } +} + +static void add_zero_len_inline(void *wqe) +{ + struct mlx4_wqe_inline_seg *inl = wqe; + memset(wqe, 0, 16); + inl->byte_count = cpu_to_be32(1 << 31); +} + +static int lay_inline_data(struct mlx4_ib_qp *qp, struct ib_send_wr *wr, + void *wqe, int *sz) +{ + struct mlx4_wqe_inline_seg *seg; + void *addr; + int len, seg_len; + int num_seg; + int off, to_copy; + int i; + int inl = 0; + + seg = wqe; + wqe += sizeof *seg; + off = ((unsigned long)wqe) & (unsigned long)(MLX4_INLINE_ALIGN - 1); + num_seg = 0; + seg_len = 0; + + for (i = 0; i < wr->num_sge; ++i) { + addr = (void *) (unsigned long)(wr->sg_list[i].addr); + len = wr->sg_list[i].length; + inl += len; + + if (inl > qp->max_inline_data) { + inl = 0; + return -1; + } + + while (len >= MLX4_INLINE_ALIGN - off) { + to_copy = MLX4_INLINE_ALIGN - off; + memcpy(wqe, addr, to_copy); + len -= to_copy; + wqe += to_copy; + addr += to_copy; + seg_len += to_copy; + wmb(); /* see comment below */ + seg->byte_count = htonl(MLX4_INLINE_SEG | seg_len); + seg_len = 0; + seg = wqe; + wqe += sizeof *seg; + off = sizeof *seg; + ++num_seg; + } + + memcpy(wqe, addr, len); + wqe += len; + seg_len += len; + off += len; + } + + if (seg_len) { + ++num_seg; + /* + * Need a barrier here to make sure + * all the data is visible before the + * byte_count field is set. Otherwise + * the HCA prefetcher could grab the + * 64-byte chunk with this inline + * segment and get a valid (!= + * 0xffffffff) byte count but stale + * data, and end up sending the wrong + * data. + */ + wmb(); + seg->byte_count = htonl(MLX4_INLINE_SEG | seg_len); + } + + *sz = (inl + num_seg * sizeof *seg + 15) / 16; + + return 0; +} + +/* + * Avoid using memcpy() to copy to BlueFlame page, since memcpy() + * implementations may use move-string-buffer assembler instructions, + * which do not guarantee order of copying. + */ +static void mlx4_bf_copy(unsigned long *dst, unsigned long *src, + unsigned bytecnt) +{ + __iowrite64_copy(dst, src, bytecnt / 8); +} + +int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, + struct ib_send_wr **bad_wr) +{ + struct mlx4_ib_qp *qp = to_mqp(ibqp); + void *wqe; + struct mlx4_wqe_ctrl_seg *uninitialized_var(ctrl); + struct mlx4_wqe_data_seg *dseg; + unsigned long flags; + int nreq; + int err = 0; + unsigned ind; + int uninitialized_var(stamp); + int uninitialized_var(size); + unsigned uninitialized_var(seglen); + __be32 dummy; + __be32 *lso_wqe; + __be32 uninitialized_var(lso_hdr_sz); + __be32 blh; + int i; + int inl = 0; + spin_lock_irqsave(&qp->sq.lock, flags); + + ind = qp->sq_next_wqe; + + for (nreq = 0; wr; ++nreq, wr = wr->next) { + lso_wqe = &dummy; + blh = 0; + + if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { + err = -ENOMEM; + *bad_wr = wr; + goto out; + } + + if (unlikely(wr->num_sge > qp->sq.max_gs)) { + err = -EINVAL; + *bad_wr = wr; + goto out; + } + + ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1)); + *((u32 *) (&ctrl->vlan_tag)) = 0; + qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = wr->wr_id; + + ctrl->srcrb_flags = + (wr->send_flags & IB_SEND_SIGNALED ? + cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) | + (wr->send_flags & IB_SEND_SOLICITED ? + cpu_to_be32(MLX4_WQE_CTRL_SOLICITED) : 0) | + ((wr->send_flags & IB_SEND_IP_CSUM) ? + cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM | + MLX4_WQE_CTRL_TCP_UDP_CSUM) : 0) | + qp->sq_signal_bits; + + ctrl->imm = send_ieth(wr); + + wqe += sizeof *ctrl; + size = sizeof *ctrl / 16; + + switch (qp->mlx4_ib_qp_type) { + case MLX4_IB_QPT_RC: + case MLX4_IB_QPT_UC: + switch (wr->opcode) { + case IB_WR_ATOMIC_CMP_AND_SWP: + case IB_WR_ATOMIC_FETCH_AND_ADD: + case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD: + set_raddr_seg(wqe, wr->wr.atomic.remote_addr, + wr->wr.atomic.rkey); + wqe += sizeof (struct mlx4_wqe_raddr_seg); + + set_atomic_seg(wqe, wr); + wqe += sizeof (struct mlx4_wqe_atomic_seg); + + size += (sizeof (struct mlx4_wqe_raddr_seg) + + sizeof (struct mlx4_wqe_atomic_seg)) / 16; + + break; + + case IB_WR_MASKED_ATOMIC_CMP_AND_SWP: + set_raddr_seg(wqe, wr->wr.atomic.remote_addr, + wr->wr.atomic.rkey); + wqe += sizeof (struct mlx4_wqe_raddr_seg); + + set_masked_atomic_seg(wqe, wr); + wqe += sizeof (struct mlx4_wqe_masked_atomic_seg); + + size += (sizeof (struct mlx4_wqe_raddr_seg) + + sizeof (struct mlx4_wqe_masked_atomic_seg)) / 16; + + break; + + case IB_WR_RDMA_READ: + case IB_WR_RDMA_WRITE: + case IB_WR_RDMA_WRITE_WITH_IMM: + set_raddr_seg(wqe, wr->wr.rdma.remote_addr, + wr->wr.rdma.rkey); + wqe += sizeof (struct mlx4_wqe_raddr_seg); + size += sizeof (struct mlx4_wqe_raddr_seg) / 16; + break; + + case IB_WR_LOCAL_INV: + ctrl->srcrb_flags |= + cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER); + set_local_inv_seg(wqe, wr->ex.invalidate_rkey); + wqe += sizeof (struct mlx4_wqe_local_inval_seg); + size += sizeof (struct mlx4_wqe_local_inval_seg) / 16; + break; + + case IB_WR_FAST_REG_MR: + ctrl->srcrb_flags |= + cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER); + set_fmr_seg(wqe, wr); + wqe += sizeof (struct mlx4_wqe_fmr_seg); + size += sizeof (struct mlx4_wqe_fmr_seg) / 16; + break; + + case IB_WR_BIND_MW: + ctrl->srcrb_flags |= + cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER); + set_bind_seg(wqe, wr); + wqe += sizeof(struct mlx4_wqe_bind_seg); + size += sizeof(struct mlx4_wqe_bind_seg) / 16; + default: + /* No extra segments required for sends */ + break; + } + break; + + case MLX4_IB_QPT_TUN_SMI_OWNER: + err = build_sriov_qp0_header(to_msqp(qp), wr, ctrl, &seglen); + if (unlikely(err)) { + *bad_wr = wr; + goto out; + } + wqe += seglen; + size += seglen / 16; + break; + case MLX4_IB_QPT_TUN_SMI: + case MLX4_IB_QPT_TUN_GSI: + /* this is a UD qp used in MAD responses to slaves. */ + set_datagram_seg(wqe, wr); + /* set the forced-loopback bit in the data seg av */ + *(__be32 *) wqe |= cpu_to_be32(0x80000000); + wqe += sizeof (struct mlx4_wqe_datagram_seg); + size += sizeof (struct mlx4_wqe_datagram_seg) / 16; + break; + case MLX4_IB_QPT_UD: + set_datagram_seg(wqe, wr); + wqe += sizeof (struct mlx4_wqe_datagram_seg); + size += sizeof (struct mlx4_wqe_datagram_seg) / 16; + + if (wr->opcode == IB_WR_LSO) { + err = build_lso_seg(wqe, wr, qp, &seglen, &lso_hdr_sz, &blh); + if (unlikely(err)) { + *bad_wr = wr; + goto out; + } + lso_wqe = (__be32 *) wqe; + wqe += seglen; + size += seglen / 16; + } + break; + + case MLX4_IB_QPT_PROXY_SMI_OWNER: + if (unlikely(!mlx4_is_master(to_mdev(ibqp->device)->dev))) { + err = -ENOSYS; + *bad_wr = wr; + goto out; + } + err = build_sriov_qp0_header(to_msqp(qp), wr, ctrl, &seglen); + if (unlikely(err)) { + *bad_wr = wr; + goto out; + } + wqe += seglen; + size += seglen / 16; + /* to start tunnel header on a cache-line boundary */ + add_zero_len_inline(wqe); + wqe += 16; + size++; + build_tunnel_header(wr, wqe, &seglen); + wqe += seglen; + size += seglen / 16; + break; + case MLX4_IB_QPT_PROXY_SMI: + /* don't allow QP0 sends on guests */ + err = -ENOSYS; + *bad_wr = wr; + goto out; + case MLX4_IB_QPT_PROXY_GSI: + /* If we are tunneling special qps, this is a UD qp. + * In this case we first add a UD segment targeting + * the tunnel qp, and then add a header with address + * information */ + set_tunnel_datagram_seg(to_mdev(ibqp->device), wqe, wr, ibqp->qp_type); + wqe += sizeof (struct mlx4_wqe_datagram_seg); + size += sizeof (struct mlx4_wqe_datagram_seg) / 16; + build_tunnel_header(wr, wqe, &seglen); + wqe += seglen; + size += seglen / 16; + break; + + case MLX4_IB_QPT_SMI: + case MLX4_IB_QPT_GSI: + err = build_mlx_header(to_msqp(qp), wr, ctrl, &seglen); + if (unlikely(err)) { + *bad_wr = wr; + goto out; + } + wqe += seglen; + size += seglen / 16; + break; + + default: + break; + } + + /* + * Write data segments in reverse order, so as to + * overwrite cacheline stamp last within each + * cacheline. This avoids issues with WQE + * prefetching. + */ + dseg = wqe; + dseg += wr->num_sge - 1; + + /* Add one more inline data segment for ICRC for MLX sends */ + if (unlikely(qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI || + qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI || + qp->mlx4_ib_qp_type & + (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER))) { + set_mlx_icrc_seg(dseg + 1); + size += sizeof (struct mlx4_wqe_data_seg) / 16; + } + + if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) { + int sz; + err = lay_inline_data(qp, wr, wqe, &sz); + if (!err) { + inl = 1; + size += sz; + } + } else { + size += wr->num_sge * + (sizeof(struct mlx4_wqe_data_seg) / 16); + for (i = wr->num_sge - 1; i >= 0; --i, --dseg) + set_data_seg(dseg, wr->sg_list + i); + } + + /* + * Possibly overwrite stamping in cacheline with LSO + * segment only after making sure all data segments + * are written. + */ + wmb(); + *lso_wqe = lso_hdr_sz; + ctrl->fence_size = (wr->send_flags & IB_SEND_FENCE ? + MLX4_WQE_CTRL_FENCE : 0) | size; + + /* + * Make sure descriptor is fully written before + * setting ownership bit (because HW can start + * executing as soon as we do). + */ + wmb(); + + if (wr->opcode < 0 || wr->opcode >= ARRAY_SIZE(mlx4_ib_opcode)) { + *bad_wr = wr; + err = -EINVAL; + goto out; + } + + ctrl->owner_opcode = mlx4_ib_opcode[wr->opcode] | + (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0) | blh; + + stamp = ind + qp->sq_spare_wqes; + ind += DIV_ROUND_UP(size * 16, 1U << qp->sq.wqe_shift); + + /* + * We can improve latency by not stamping the last + * send queue WQE until after ringing the doorbell, so + * only stamp here if there are still more WQEs to post. + * + * Same optimization applies to padding with NOP wqe + * in case of WQE shrinking (used to prevent wrap-around + * in the middle of WR). + */ + if (wr->next) { + stamp_send_wqe(qp, stamp, size * 16); + ind = pad_wraparound(qp, ind); + } + } + +out: + if (nreq == 1 && inl && size > 1 && size < qp->bf.buf_size / 16) { + ctrl->owner_opcode |= htonl((qp->sq_next_wqe & 0xffff) << 8); + /* We set above doorbell_qpn bits to 0 as part of vlan + * tag initialization, so |= should be correct. + */ + *(u32 *) (&ctrl->vlan_tag) |= qp->doorbell_qpn; + /* + * Make sure that descriptor is written to memory + * before writing to BlueFlame page. + */ + wmb(); + + ++qp->sq.head; + + mlx4_bf_copy(qp->bf.reg + qp->bf.offset, (unsigned long *) ctrl, + ALIGN(size * 16, 64)); + wc_wmb(); + + qp->bf.offset ^= qp->bf.buf_size; + + } else if (nreq) { + qp->sq.head += nreq; + + /* + * Make sure that descriptors are written before + * doorbell record. + */ + wmb(); + + writel(qp->doorbell_qpn, qp->bf.uar->map + MLX4_SEND_DOORBELL); + + /* + * Make sure doorbells don't leak out of SQ spinlock + * and reach the HCA out of order. + */ + mmiowb(); + + } + + if (likely(nreq)) { + stamp_send_wqe(qp, stamp, size * 16); + ind = pad_wraparound(qp, ind); + qp->sq_next_wqe = ind; + } + + spin_unlock_irqrestore(&qp->sq.lock, flags); + + return err; +} + +int mlx4_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr) +{ + struct mlx4_ib_qp *qp = to_mqp(ibqp); + struct mlx4_wqe_data_seg *scat; + unsigned long flags; + int err = 0; + int nreq; + int ind; + int max_gs; + int i; + + max_gs = qp->rq.max_gs; + spin_lock_irqsave(&qp->rq.lock, flags); + + ind = qp->rq.head & (qp->rq.wqe_cnt - 1); + + for (nreq = 0; wr; ++nreq, wr = wr->next) { + if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) { + err = -ENOMEM; + *bad_wr = wr; + goto out; + } + + if (unlikely(wr->num_sge > qp->rq.max_gs)) { + err = -EINVAL; + *bad_wr = wr; + goto out; + } + + scat = get_recv_wqe(qp, ind); + + if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER | + MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) { + ib_dma_sync_single_for_device(ibqp->device, + qp->sqp_proxy_rcv[ind].map, + sizeof (struct mlx4_ib_proxy_sqp_hdr), + DMA_FROM_DEVICE); + scat->byte_count = + cpu_to_be32(sizeof (struct mlx4_ib_proxy_sqp_hdr)); + /* use dma lkey from upper layer entry */ + scat->lkey = cpu_to_be32(wr->sg_list->lkey); + scat->addr = cpu_to_be64(qp->sqp_proxy_rcv[ind].map); + scat++; + max_gs--; + } + + for (i = 0; i < wr->num_sge; ++i) + __set_data_seg(scat + i, wr->sg_list + i); + + if (i < max_gs) { + scat[i].byte_count = 0; + scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY); + scat[i].addr = 0; + } + + qp->rq.wrid[ind] = wr->wr_id; + + ind = (ind + 1) & (qp->rq.wqe_cnt - 1); + } + +out: + if (likely(nreq)) { + qp->rq.head += nreq; + + /* + * Make sure that descriptors are written before + * doorbell record. + */ + wmb(); + + *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff); + } + + spin_unlock_irqrestore(&qp->rq.lock, flags); + + return err; +} + +static inline enum ib_qp_state to_ib_qp_state(enum mlx4_qp_state mlx4_state) +{ + switch (mlx4_state) { + case MLX4_QP_STATE_RST: return IB_QPS_RESET; + case MLX4_QP_STATE_INIT: return IB_QPS_INIT; + case MLX4_QP_STATE_RTR: return IB_QPS_RTR; + case MLX4_QP_STATE_RTS: return IB_QPS_RTS; + case MLX4_QP_STATE_SQ_DRAINING: + case MLX4_QP_STATE_SQD: return IB_QPS_SQD; + case MLX4_QP_STATE_SQER: return IB_QPS_SQE; + case MLX4_QP_STATE_ERR: return IB_QPS_ERR; + default: return -1; + } +} + +static inline enum ib_mig_state to_ib_mig_state(int mlx4_mig_state) +{ + switch (mlx4_mig_state) { + case MLX4_QP_PM_ARMED: return IB_MIG_ARMED; + case MLX4_QP_PM_REARM: return IB_MIG_REARM; + case MLX4_QP_PM_MIGRATED: return IB_MIG_MIGRATED; + default: return -1; + } +} + +static int to_ib_qp_access_flags(int mlx4_flags) +{ + int ib_flags = 0; + + if (mlx4_flags & MLX4_QP_BIT_RRE) + ib_flags |= IB_ACCESS_REMOTE_READ; + if (mlx4_flags & MLX4_QP_BIT_RWE) + ib_flags |= IB_ACCESS_REMOTE_WRITE; + if (mlx4_flags & MLX4_QP_BIT_RAE) + ib_flags |= IB_ACCESS_REMOTE_ATOMIC; + + return ib_flags; +} + +static void to_ib_ah_attr(struct mlx4_ib_dev *ibdev, struct ib_ah_attr *ib_ah_attr, + struct mlx4_qp_path *path) +{ + struct mlx4_dev *dev = ibdev->dev; + int is_eth; + + memset(ib_ah_attr, 0, sizeof *ib_ah_attr); + ib_ah_attr->port_num = path->sched_queue & 0x40 ? 2 : 1; + + if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->caps.num_ports) + return; + + is_eth = rdma_port_get_link_layer(&ibdev->ib_dev, ib_ah_attr->port_num) == + IB_LINK_LAYER_ETHERNET; + if (is_eth) + ib_ah_attr->sl = ((path->sched_queue >> 3) & 0x7) | + ((path->sched_queue & 4) << 1); + else + ib_ah_attr->sl = (path->sched_queue >> 2) & 0xf; + + ib_ah_attr->dlid = be16_to_cpu(path->rlid); + ib_ah_attr->src_path_bits = path->grh_mylmc & 0x7f; + ib_ah_attr->static_rate = path->static_rate ? path->static_rate - 5 : 0; + ib_ah_attr->ah_flags = (path->grh_mylmc & (1 << 7)) ? IB_AH_GRH : 0; + if (ib_ah_attr->ah_flags) { + ib_ah_attr->grh.sgid_index = path->mgid_index; + ib_ah_attr->grh.hop_limit = path->hop_limit; + ib_ah_attr->grh.traffic_class = + (be32_to_cpu(path->tclass_flowlabel) >> 20) & 0xff; + ib_ah_attr->grh.flow_label = + be32_to_cpu(path->tclass_flowlabel) & 0xfffff; + memcpy(ib_ah_attr->grh.dgid.raw, + path->rgid, sizeof ib_ah_attr->grh.dgid.raw); + } +} + +int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, + struct ib_qp_init_attr *qp_init_attr) +{ + struct mlx4_ib_dev *dev = to_mdev(ibqp->device); + struct mlx4_ib_qp *qp = to_mqp(ibqp); + struct mlx4_qp_context context; + int mlx4_state; + int err = 0; + + mutex_lock(&qp->mutex); + + if (qp->state == IB_QPS_RESET) { + qp_attr->qp_state = IB_QPS_RESET; + goto done; + } + + err = mlx4_qp_query(dev->dev, &qp->mqp, &context); + if (err) { + err = -EINVAL; + goto out; + } + + mlx4_state = be32_to_cpu(context.flags) >> 28; + + qp->state = to_ib_qp_state(mlx4_state); + qp_attr->qp_state = qp->state; + qp_attr->path_mtu = context.mtu_msgmax >> 5; + qp_attr->path_mig_state = + to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3); + qp_attr->qkey = be32_to_cpu(context.qkey); + qp_attr->rq_psn = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff; + qp_attr->sq_psn = be32_to_cpu(context.next_send_psn) & 0xffffff; + qp_attr->dest_qp_num = be32_to_cpu(context.remote_qpn) & 0xffffff; + qp_attr->qp_access_flags = + to_ib_qp_access_flags(be32_to_cpu(context.params2)); + + if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) { + to_ib_ah_attr(dev, &qp_attr->ah_attr, &context.pri_path); + to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context.alt_path); + qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f; + qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num; + } + + qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f; + if (qp_attr->qp_state == IB_QPS_INIT) + qp_attr->port_num = qp->port; + else + qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1; + + /* qp_attr->en_sqd_async_notify is only applicable in modify qp */ + qp_attr->sq_draining = mlx4_state == MLX4_QP_STATE_SQ_DRAINING; + + qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context.params1) >> 21) & 0x7); + + qp_attr->max_dest_rd_atomic = + 1 << ((be32_to_cpu(context.params2) >> 21) & 0x7); + qp_attr->min_rnr_timer = + (be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f; + qp_attr->timeout = context.pri_path.ackto >> 3; + qp_attr->retry_cnt = (be32_to_cpu(context.params1) >> 16) & 0x7; + qp_attr->rnr_retry = (be32_to_cpu(context.params1) >> 13) & 0x7; + qp_attr->alt_timeout = context.alt_path.ackto >> 3; + +done: + qp_attr->cur_qp_state = qp_attr->qp_state; + qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt; + qp_attr->cap.max_recv_sge = qp->rq.max_gs; + + if (!ibqp->uobject) { + qp_attr->cap.max_send_wr = qp->sq.wqe_cnt; + qp_attr->cap.max_send_sge = qp->sq.max_gs; + } else { + qp_attr->cap.max_send_wr = 0; + qp_attr->cap.max_send_sge = 0; + } + + /* + * We don't support inline sends for kernel QPs (yet), and we + * don't know what userspace's value should be. + */ + qp_attr->cap.max_inline_data = 0; + + qp_init_attr->cap = qp_attr->cap; + + qp_init_attr->create_flags = 0; + if (qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) + qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK; + + if (qp->flags & MLX4_IB_QP_LSO) + qp_init_attr->create_flags |= IB_QP_CREATE_IPOIB_UD_LSO; + + if (qp->flags & MLX4_IB_QP_NETIF) + qp_init_attr->create_flags |= IB_QP_CREATE_NETIF_QP; + + qp_init_attr->sq_sig_type = + qp->sq_signal_bits == cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) ? + IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; + + if (qp->flags & MLX4_IB_QP_CAP_CROSS_CHANNEL) + qp_init_attr->create_flags |= IB_QP_CREATE_CROSS_CHANNEL; + + if (qp->flags & MLX4_IB_QP_CAP_MANAGED_SEND) + qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_SEND; + + if (qp->flags & MLX4_IB_QP_CAP_MANAGED_RECV) + qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_RECV; + + qp_init_attr->qpg_type = ibqp->qpg_type; + if (ibqp->qpg_type == IB_QPG_PARENT) + qp_init_attr->cap.qpg_tss_mask_sz = qp->qpg_data->qpg_tss_mask_sz; + else + qp_init_attr->cap.qpg_tss_mask_sz = 0; + +out: + mutex_unlock(&qp->mutex); + return err; +} Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_qp.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_srq.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_srq.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_srq.c (revision 329159) @@ -0,0 +1,369 @@ +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include + +#include "mlx4_ib.h" +#include "user.h" + +static void *get_wqe(struct mlx4_ib_srq *srq, int n) +{ + return mlx4_buf_offset(&srq->buf, n << srq->msrq.wqe_shift); +} + +static void mlx4_ib_srq_event(struct mlx4_srq *srq, enum mlx4_event type) +{ + struct ib_event event; + struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq; + + if (ibsrq->event_handler) { + event.device = ibsrq->device; + event.element.srq = ibsrq; + switch (type) { + case MLX4_EVENT_TYPE_SRQ_LIMIT: + event.event = IB_EVENT_SRQ_LIMIT_REACHED; + break; + case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR: + event.event = IB_EVENT_SRQ_ERR; + break; + default: + pr_warn("Unexpected event type %d " + "on SRQ %06x\n", type, srq->srqn); + return; + } + + ibsrq->event_handler(&event, ibsrq->srq_context); + } +} + +struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata) +{ + struct mlx4_ib_dev *dev = to_mdev(pd->device); + struct mlx4_ib_srq *srq; + struct mlx4_wqe_srq_next_seg *next; + struct mlx4_wqe_data_seg *scatter; + u32 cqn; + u16 xrcdn; + int desc_size; + int buf_size; + int err; + int i; + + /* Sanity check SRQ size before proceeding */ + if (init_attr->attr.max_wr >= dev->dev->caps.max_srq_wqes || + init_attr->attr.max_sge > dev->dev->caps.max_srq_sge) + return ERR_PTR(-EINVAL); + + srq = kmalloc(sizeof *srq, GFP_KERNEL); + if (!srq) + return ERR_PTR(-ENOMEM); + + mutex_init(&srq->mutex); + spin_lock_init(&srq->lock); + srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); + srq->msrq.max_gs = init_attr->attr.max_sge; + + desc_size = max(32UL, + roundup_pow_of_two(sizeof (struct mlx4_wqe_srq_next_seg) + + srq->msrq.max_gs * + sizeof (struct mlx4_wqe_data_seg))); + srq->msrq.wqe_shift = ilog2(desc_size); + + buf_size = srq->msrq.max * desc_size; + + if (pd->uobject) { + struct mlx4_ib_create_srq ucmd; + + if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { + err = -EFAULT; + goto err_srq; + } + + srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, + buf_size, 0, 0); + if (IS_ERR(srq->umem)) { + err = PTR_ERR(srq->umem); + goto err_srq; + } + + err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem), + ilog2(srq->umem->page_size), &srq->mtt); + if (err) + goto err_buf; + + err = mlx4_ib_umem_write_mtt(dev, &srq->mtt, srq->umem); + if (err) + goto err_mtt; + + err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context), + ucmd.db_addr, &srq->db); + if (err) + goto err_mtt; + } else { + err = mlx4_db_alloc(dev->dev, &srq->db, 0, GFP_KERNEL); + if (err) + goto err_srq; + + *srq->db.db = 0; + + if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &srq->buf, + GFP_KERNEL)) { + err = -ENOMEM; + goto err_db; + } + + srq->head = 0; + srq->tail = srq->msrq.max - 1; + srq->wqe_ctr = 0; + + for (i = 0; i < srq->msrq.max; ++i) { + next = get_wqe(srq, i); + next->next_wqe_index = + cpu_to_be16((i + 1) & (srq->msrq.max - 1)); + + for (scatter = (void *) (next + 1); + (void *) scatter < (void *) next + desc_size; + ++scatter) + scatter->lkey = cpu_to_be32(MLX4_INVALID_LKEY); + } + + err = mlx4_mtt_init(dev->dev, srq->buf.npages, srq->buf.page_shift, + &srq->mtt); + if (err) + goto err_buf; + + err = mlx4_buf_write_mtt(dev->dev, &srq->mtt, &srq->buf, GFP_KERNEL); + if (err) + goto err_mtt; + + srq->wrid = kmalloc(srq->msrq.max * sizeof (u64), GFP_KERNEL); + if (!srq->wrid) { + err = -ENOMEM; + goto err_mtt; + } + } + + cqn = (init_attr->srq_type == IB_SRQT_XRC) ? + to_mcq(init_attr->ext.xrc.cq)->mcq.cqn : 0; + xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? + to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn : + (u16) dev->dev->caps.reserved_xrcds; + err = mlx4_srq_alloc(dev->dev, to_mpd(pd)->pdn, cqn, xrcdn, &srq->mtt, + srq->db.dma, &srq->msrq); + if (err) + goto err_wrid; + + srq->msrq.event = mlx4_ib_srq_event; + srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; + + if (pd->uobject) + if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) { + err = -EFAULT; + goto err_wrid; + } + + init_attr->attr.max_wr = srq->msrq.max - 1; + + return &srq->ibsrq; + +err_wrid: + if (pd->uobject) + mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db); + else + kfree(srq->wrid); + +err_mtt: + mlx4_mtt_cleanup(dev->dev, &srq->mtt); + +err_buf: + if (pd->uobject) + ib_umem_release(srq->umem); + else + mlx4_buf_free(dev->dev, buf_size, &srq->buf); + +err_db: + if (!pd->uobject) + mlx4_db_free(dev->dev, &srq->db); + +err_srq: + kfree(srq); + + return ERR_PTR(err); +} + +int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, + enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) +{ + struct mlx4_ib_dev *dev = to_mdev(ibsrq->device); + struct mlx4_ib_srq *srq = to_msrq(ibsrq); + int ret; + + /* We don't support resizing SRQs (yet?) */ + if (attr_mask & IB_SRQ_MAX_WR) + return -EINVAL; + + if (attr_mask & IB_SRQ_LIMIT) { + if (attr->srq_limit >= srq->msrq.max) + return -EINVAL; + + mutex_lock(&srq->mutex); + ret = mlx4_srq_arm(dev->dev, &srq->msrq, attr->srq_limit); + mutex_unlock(&srq->mutex); + + if (ret) + return ret; + } + + return 0; +} + +int mlx4_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) +{ + struct mlx4_ib_dev *dev = to_mdev(ibsrq->device); + struct mlx4_ib_srq *srq = to_msrq(ibsrq); + int ret; + int limit_watermark; + + ret = mlx4_srq_query(dev->dev, &srq->msrq, &limit_watermark); + if (ret) + return ret; + + srq_attr->srq_limit = limit_watermark; + srq_attr->max_wr = srq->msrq.max - 1; + srq_attr->max_sge = srq->msrq.max_gs; + + return 0; +} + +int mlx4_ib_destroy_srq(struct ib_srq *srq) +{ + struct mlx4_ib_dev *dev = to_mdev(srq->device); + struct mlx4_ib_srq *msrq = to_msrq(srq); + + mlx4_srq_free(dev->dev, &msrq->msrq); + mlx4_mtt_cleanup(dev->dev, &msrq->mtt); + + if (srq->uobject) { + mlx4_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db); + ib_umem_release(msrq->umem); + } else { + kfree(msrq->wrid); + mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift, + &msrq->buf); + mlx4_db_free(dev->dev, &msrq->db); + } + + kfree(msrq); + + return 0; +} + +void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index) +{ + struct mlx4_wqe_srq_next_seg *next; + + /* always called with interrupts disabled. */ + spin_lock(&srq->lock); + + next = get_wqe(srq, srq->tail); + next->next_wqe_index = cpu_to_be16(wqe_index); + srq->tail = wqe_index; + + spin_unlock(&srq->lock); +} + +int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr) +{ + struct mlx4_ib_srq *srq = to_msrq(ibsrq); + struct mlx4_wqe_srq_next_seg *next; + struct mlx4_wqe_data_seg *scat; + unsigned long flags; + int err = 0; + int nreq; + int i; + + spin_lock_irqsave(&srq->lock, flags); + + for (nreq = 0; wr; ++nreq, wr = wr->next) { + if (unlikely(wr->num_sge > srq->msrq.max_gs)) { + err = -EINVAL; + *bad_wr = wr; + break; + } + + if (unlikely(srq->head == srq->tail)) { + err = -ENOMEM; + *bad_wr = wr; + break; + } + + srq->wrid[srq->head] = wr->wr_id; + + next = get_wqe(srq, srq->head); + srq->head = be16_to_cpu(next->next_wqe_index); + scat = (struct mlx4_wqe_data_seg *) (next + 1); + + for (i = 0; i < wr->num_sge; ++i) { + scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length); + scat[i].lkey = cpu_to_be32(wr->sg_list[i].lkey); + scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); + } + + if (i < srq->msrq.max_gs) { + scat[i].byte_count = 0; + scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY); + scat[i].addr = 0; + } + } + + if (likely(nreq)) { + srq->wqe_ctr += nreq; + + /* + * Make sure that descriptors are written before + * doorbell record. + */ + wmb(); + + *srq->db.db = cpu_to_be32(srq->wqe_ctr); + } + + spin_unlock_irqrestore(&srq->lock, flags); + + return err; +} Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_srq.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_sysfs.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_sysfs.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_sysfs.c (revision 329159) @@ -0,0 +1,801 @@ +/* + * Copyright (c) 2012 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/*#include "core_priv.h"*/ +#include "mlx4_ib.h" +#include +#include +#include + +#include +/*show_admin_alias_guid returns the administratively assigned value of that GUID. + * Values returned in buf parameter string: + * 0 - requests opensm to assign a value. + * ffffffffffffffff - delete this entry. + * other - value assigned by administrator. + */ +static ssize_t show_admin_alias_guid(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int record_num;/*0-15*/ + int guid_index_in_rec; /*0 - 7*/ + struct mlx4_ib_iov_sysfs_attr *mlx4_ib_iov_dentry = + container_of(attr, struct mlx4_ib_iov_sysfs_attr, dentry); + struct mlx4_ib_iov_port *port = mlx4_ib_iov_dentry->ctx; + struct mlx4_ib_dev *mdev = port->dev; + + record_num = mlx4_ib_iov_dentry->entry_num / 8 ; + guid_index_in_rec = mlx4_ib_iov_dentry->entry_num % 8 ; + + return sprintf(buf, "%llx\n", + (long long)be64_to_cpu(*(__be64 *)&mdev->sriov.alias_guid. + ports_guid[port->num - 1]. + all_rec_per_port[record_num]. + all_recs[8 * guid_index_in_rec])); +} + +/* store_admin_alias_guid stores the (new) administratively assigned value of that GUID. + * Values in buf parameter string: + * 0 - requests opensm to assign a value. + * 0xffffffffffffffff - delete this entry. + * other - guid value assigned by the administrator. + */ +static ssize_t store_admin_alias_guid(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + int record_num;/*0-15*/ + int guid_index_in_rec; /*0 - 7*/ + struct mlx4_ib_iov_sysfs_attr *mlx4_ib_iov_dentry = + container_of(attr, struct mlx4_ib_iov_sysfs_attr, dentry); + struct mlx4_ib_iov_port *port = mlx4_ib_iov_dentry->ctx; + struct mlx4_ib_dev *mdev = port->dev; + unsigned long long sysadmin_ag_val; + + record_num = mlx4_ib_iov_dentry->entry_num / 8; + guid_index_in_rec = mlx4_ib_iov_dentry->entry_num % 8; + if (0 == record_num && 0 == guid_index_in_rec) { + pr_err("GUID 0 block 0 is RO\n"); + return count; + } + sscanf(buf, "%llx", &sysadmin_ag_val); + *(__be64 *)&mdev->sriov.alias_guid.ports_guid[port->num - 1]. + all_rec_per_port[record_num]. + all_recs[GUID_REC_SIZE * guid_index_in_rec] = + cpu_to_be64(sysadmin_ag_val); + + /* Change the state to be pending for update */ + mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].status + = MLX4_GUID_INFO_STATUS_IDLE ; + + mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].method + = MLX4_GUID_INFO_RECORD_SET; + + switch (sysadmin_ag_val) { + case MLX4_GUID_FOR_DELETE_VAL: + mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].method + = MLX4_GUID_INFO_RECORD_DELETE; + mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].ownership + = MLX4_GUID_SYSADMIN_ASSIGN; + break; + /* The sysadmin requests the SM to re-assign */ + case MLX4_NOT_SET_GUID: + mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].ownership + = MLX4_GUID_DRIVER_ASSIGN; + break; + /* The sysadmin requests a specific value.*/ + default: + mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].ownership + = MLX4_GUID_SYSADMIN_ASSIGN; + break; + } + + /* set the record index */ + mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].guid_indexes + = mlx4_ib_get_aguid_comp_mask_from_ix(guid_index_in_rec); + + mlx4_ib_init_alias_guid_work(mdev, port->num - 1); + + return count; +} + +static ssize_t show_port_gid(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct mlx4_ib_iov_sysfs_attr *mlx4_ib_iov_dentry = + container_of(attr, struct mlx4_ib_iov_sysfs_attr, dentry); + struct mlx4_ib_iov_port *port = mlx4_ib_iov_dentry->ctx; + struct mlx4_ib_dev *mdev = port->dev; + union ib_gid gid; + ssize_t ret; + + ret = __mlx4_ib_query_gid(&mdev->ib_dev, port->num, + mlx4_ib_iov_dentry->entry_num, &gid, 1); + if (ret) + return ret; + ret = sprintf(buf, "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n", + be16_to_cpu(((__be16 *) gid.raw)[0]), + be16_to_cpu(((__be16 *) gid.raw)[1]), + be16_to_cpu(((__be16 *) gid.raw)[2]), + be16_to_cpu(((__be16 *) gid.raw)[3]), + be16_to_cpu(((__be16 *) gid.raw)[4]), + be16_to_cpu(((__be16 *) gid.raw)[5]), + be16_to_cpu(((__be16 *) gid.raw)[6]), + be16_to_cpu(((__be16 *) gid.raw)[7])); + return ret; +} + +static ssize_t show_phys_port_pkey(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct mlx4_ib_iov_sysfs_attr *mlx4_ib_iov_dentry = + container_of(attr, struct mlx4_ib_iov_sysfs_attr, dentry); + struct mlx4_ib_iov_port *port = mlx4_ib_iov_dentry->ctx; + struct mlx4_ib_dev *mdev = port->dev; + u16 pkey; + ssize_t ret; + + ret = __mlx4_ib_query_pkey(&mdev->ib_dev, port->num, + mlx4_ib_iov_dentry->entry_num, &pkey, 1); + if (ret) + return ret; + + return sprintf(buf, "0x%04x\n", pkey); +} + +#define DENTRY_REMOVE(_dentry) \ +do { \ + sysfs_remove_file((_dentry)->kobj, &(_dentry)->dentry.attr); \ +} while (0); + +static int create_sysfs_entry(void *_ctx, struct mlx4_ib_iov_sysfs_attr *_dentry, + char *_name, struct kobject *_kobj, + ssize_t (*show)(struct device *dev, + struct device_attribute *attr, + char *buf), + ssize_t (*store)(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) + ) +{ + int ret = 0; + struct mlx4_ib_iov_sysfs_attr *vdentry = _dentry; + + vdentry->ctx = _ctx; + vdentry->dentry.show = show; + vdentry->dentry.store = store; + sysfs_attr_init(&vdentry->dentry.attr); + vdentry->dentry.attr.name = vdentry->name; + vdentry->dentry.attr.mode = 0; + vdentry->kobj = _kobj; + snprintf(vdentry->name, 15, "%s", _name); + + if (vdentry->dentry.store) + vdentry->dentry.attr.mode |= S_IWUSR; + + if (vdentry->dentry.show) + vdentry->dentry.attr.mode |= S_IRUGO; + + ret = sysfs_create_file(vdentry->kobj, &vdentry->dentry.attr); + if (ret) { + pr_err("failed to create %s\n", vdentry->dentry.attr.name); + vdentry->ctx = NULL; + return ret; + } + + return ret; +} + +int add_sysfs_port_mcg_attr(struct mlx4_ib_dev *device, int port_num, + struct attribute *attr) +{ + struct mlx4_ib_iov_port *port = &device->iov_ports[port_num - 1]; + int ret; + + ret = sysfs_create_file(port->mcgs_parent, attr); + if (ret) + pr_err("failed to create %s\n", attr->name); + + return ret; +} + +void del_sysfs_port_mcg_attr(struct mlx4_ib_dev *device, int port_num, + struct attribute *attr) +{ + struct mlx4_ib_iov_port *port = &device->iov_ports[port_num - 1]; + + sysfs_remove_file(port->mcgs_parent, attr); +} + +static int add_port_entries(struct mlx4_ib_dev *device, int port_num) +{ + int i; + char buff[10]; + struct mlx4_ib_iov_port *port = NULL; + int ret = 0 ; + struct ib_port_attr attr; + + /* get the physical gid and pkey table sizes.*/ + ret = __mlx4_ib_query_port(&device->ib_dev, port_num, &attr, 1); + if (ret) + goto err; + + port = &device->iov_ports[port_num - 1]; + port->dev = device; + port->num = port_num; + /* Directory structure: + * iov - + * port num - + * admin_guids + * gids (operational) + * mcg_table + */ + port->dentr_ar = kzalloc(sizeof (struct mlx4_ib_iov_sysfs_attr_ar), + GFP_KERNEL); + if (!port->dentr_ar) { + ret = -ENOMEM; + goto err; + } + sprintf(buff, "%d", port_num); + port->cur_port = kobject_create_and_add(buff, + kobject_get(device->ports_parent)); + if (!port->cur_port) { + ret = -ENOMEM; + goto kobj_create_err; + } + /* admin GUIDs */ + port->admin_alias_parent = kobject_create_and_add("admin_guids", + kobject_get(port->cur_port)); + if (!port->admin_alias_parent) { + ret = -ENOMEM; + goto err_admin_guids; + } + for (i = 0 ; i < attr.gid_tbl_len; i++) { + sprintf(buff, "%d", i); + port->dentr_ar->dentries[i].entry_num = i; + ret = create_sysfs_entry(port, &port->dentr_ar->dentries[i], + buff, port->admin_alias_parent, + show_admin_alias_guid, store_admin_alias_guid); + if (ret) + goto err_admin_alias_parent; + } + + /* gids subdirectory (operational gids) */ + port->gids_parent = kobject_create_and_add("gids", + kobject_get(port->cur_port)); + if (!port->gids_parent) { + ret = -ENOMEM; + goto err_gids; + } + + for (i = 0 ; i < attr.gid_tbl_len; i++) { + sprintf(buff, "%d", i); + port->dentr_ar->dentries[attr.gid_tbl_len + i].entry_num = i; + ret = create_sysfs_entry(port, + &port->dentr_ar->dentries[attr.gid_tbl_len + i], + buff, + port->gids_parent, show_port_gid, NULL); + if (ret) + goto err_gids_parent; + } + + /* physical port pkey table */ + port->pkeys_parent = + kobject_create_and_add("pkeys", kobject_get(port->cur_port)); + if (!port->pkeys_parent) { + ret = -ENOMEM; + goto err_pkeys; + } + + for (i = 0 ; i < attr.pkey_tbl_len; i++) { + sprintf(buff, "%d", i); + port->dentr_ar->dentries[2 * attr.gid_tbl_len + i].entry_num = i; + ret = create_sysfs_entry(port, + &port->dentr_ar->dentries[2 * attr.gid_tbl_len + i], + buff, port->pkeys_parent, + show_phys_port_pkey, NULL); + if (ret) + goto err_pkeys_parent; + } + + /* MCGs table */ + port->mcgs_parent = + kobject_create_and_add("mcgs", kobject_get(port->cur_port)); + if (!port->mcgs_parent) { + ret = -ENOMEM; + goto err_mcgs; + } + return 0; + +err_mcgs: + kobject_put(port->cur_port); + +err_pkeys_parent: + kobject_put(port->pkeys_parent); + +err_pkeys: + kobject_put(port->cur_port); + +err_gids_parent: + kobject_put(port->gids_parent); + +err_gids: + kobject_put(port->cur_port); + +err_admin_alias_parent: + kobject_put(port->admin_alias_parent); + +err_admin_guids: + kobject_put(port->cur_port); + kobject_put(port->cur_port); /* once more for create_and_add buff */ + +kobj_create_err: + kobject_put(device->ports_parent); + kfree(port->dentr_ar); + +err: + pr_err("add_port_entries FAILED: for port:%d, error: %d\n", + port_num, ret); + return ret; +} + +static void get_name(struct mlx4_ib_dev *dev, char *name, int i, int max) +{ + char base_name[9]; + + /* pci_name format is: bus:dev:func -> xxxx:yy:zz.n */ + strlcpy(name, pci_name(dev->dev->persist->pdev), max); + strncpy(base_name, name, 8); /*till xxxx:yy:*/ + base_name[8] = '\0'; + /* with no ARI only 3 last bits are used so when the fn is higher than 8 + * need to add it to the dev num, so count in the last number will be + * modulo 8 */ + sprintf(name, "%s%.2d.%d", base_name, (i/8), (i%8)); +} + +struct mlx4_port { + struct kobject kobj; + struct mlx4_ib_dev *dev; + struct attribute_group pkey_group; + struct attribute_group gid_group; + u8 port_num; + int slave; +}; + + +static void mlx4_port_release(struct kobject *kobj) +{ + struct mlx4_port *p = container_of(kobj, struct mlx4_port, kobj); + struct attribute *a; + int i; + + for (i = 0; (a = p->pkey_group.attrs[i]); ++i) + kfree(a); + kfree(p->pkey_group.attrs); + for (i = 0; (a = p->gid_group.attrs[i]); ++i) + kfree(a); + kfree(p->gid_group.attrs); + kfree(p); +} + +struct port_attribute { + struct attribute attr; + ssize_t (*show)(struct mlx4_port *, struct port_attribute *, char *buf); + ssize_t (*store)(struct mlx4_port *, struct port_attribute *, + const char *buf, size_t count); +}; + +static ssize_t port_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct port_attribute *port_attr = + container_of(attr, struct port_attribute, attr); + struct mlx4_port *p = container_of(kobj, struct mlx4_port, kobj); + + if (!port_attr->show) + return -EIO; + return port_attr->show(p, port_attr, buf); +} + +static ssize_t port_attr_store(struct kobject *kobj, + struct attribute *attr, + const char *buf, size_t size) +{ + struct port_attribute *port_attr = + container_of(attr, struct port_attribute, attr); + struct mlx4_port *p = container_of(kobj, struct mlx4_port, kobj); + + if (!port_attr->store) + return -EIO; + return port_attr->store(p, port_attr, buf, size); +} + +static const struct sysfs_ops port_sysfs_ops = { + .show = port_attr_show, + .store = port_attr_store, +}; + +static struct kobj_type port_type = { + .release = mlx4_port_release, + .sysfs_ops = &port_sysfs_ops, +}; + +struct port_table_attribute { + struct port_attribute attr; + char name[8]; + int index; +}; + +static ssize_t show_port_pkey(struct mlx4_port *p, struct port_attribute *attr, + char *buf) +{ + struct port_table_attribute *tab_attr = + container_of(attr, struct port_table_attribute, attr); + ssize_t ret = -ENODEV; + + if (p->dev->pkeys.virt2phys_pkey[p->slave][p->port_num - 1][tab_attr->index] >= + (p->dev->dev->caps.pkey_table_len[p->port_num])) + ret = sprintf(buf, "none\n"); + else + ret = sprintf(buf, "%d\n", + p->dev->pkeys.virt2phys_pkey[p->slave] + [p->port_num - 1][tab_attr->index]); + return ret; +} + +static ssize_t store_port_pkey(struct mlx4_port *p, struct port_attribute *attr, + const char *buf, size_t count) +{ + struct port_table_attribute *tab_attr = + container_of(attr, struct port_table_attribute, attr); + int idx; + int err; + + /* do not allow remapping Dom0 virtual pkey table */ + if (p->slave == mlx4_master_func_num(p->dev->dev)) + return -EINVAL; + + if (!strncasecmp(buf, "no", 2)) + idx = p->dev->dev->phys_caps.pkey_phys_table_len[p->port_num] - 1; + else if (sscanf(buf, "%i", &idx) != 1 || + idx >= p->dev->dev->caps.pkey_table_len[p->port_num] || + idx < 0) + return -EINVAL; + + p->dev->pkeys.virt2phys_pkey[p->slave][p->port_num - 1] + [tab_attr->index] = idx; + mlx4_sync_pkey_table(p->dev->dev, p->slave, p->port_num, + tab_attr->index, idx); + err = mlx4_gen_pkey_eqe(p->dev->dev, p->slave, p->port_num); + if (err) { + pr_err("mlx4_gen_pkey_eqe failed for slave %d," + " port %d, index %d\n", p->slave, p->port_num, idx); + return err; + } + return count; +} + +static ssize_t show_port_gid_idx(struct mlx4_port *p, + struct port_attribute *attr, char *buf) +{ + return sprintf(buf, "%d\n", p->slave); +} + +static struct attribute ** +alloc_group_attrs(ssize_t (*show)(struct mlx4_port *, + struct port_attribute *, char *buf), + ssize_t (*store)(struct mlx4_port *, struct port_attribute *, + const char *buf, size_t count), + int len) +{ + struct attribute **tab_attr; + struct port_table_attribute *element; + int i; + + tab_attr = kcalloc(1 + len, sizeof (struct attribute *), GFP_KERNEL); + if (!tab_attr) + return NULL; + + for (i = 0; i < len; i++) { + element = kzalloc(sizeof (struct port_table_attribute), + GFP_KERNEL); + if (!element) + goto err; + if (snprintf(element->name, sizeof (element->name), + "%d", i) >= sizeof (element->name)) { + kfree(element); + goto err; + } + sysfs_attr_init(&element->attr.attr); + element->attr.attr.name = element->name; + if (store) { + element->attr.attr.mode = S_IWUSR | S_IRUGO; + element->attr.store = store; + } else + element->attr.attr.mode = S_IRUGO; + + element->attr.show = show; + element->index = i; + tab_attr[i] = &element->attr.attr; + } + return tab_attr; + +err: + while (--i >= 0) + kfree(tab_attr[i]); + kfree(tab_attr); + return NULL; +} + +static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave) +{ + struct mlx4_port *p; + int i; + int ret; + int is_eth = rdma_port_get_link_layer(&dev->ib_dev, port_num) == + IB_LINK_LAYER_ETHERNET; + + p = kzalloc(sizeof *p, GFP_KERNEL); + if (!p) + return -ENOMEM; + + p->dev = dev; + p->port_num = port_num; + p->slave = slave; + + ret = kobject_init_and_add(&p->kobj, &port_type, + kobject_get(dev->dev_ports_parent[slave]), + "%d", port_num); + if (ret) + goto err_alloc; + + p->pkey_group.name = "pkey_idx"; + if (is_eth) + p->pkey_group.attrs = + alloc_group_attrs(show_port_pkey, NULL, + dev->dev->caps.pkey_table_len[port_num]); + else + p->pkey_group.attrs = + alloc_group_attrs(show_port_pkey, store_port_pkey, + dev->dev->caps.pkey_table_len[port_num]); + if (!p->pkey_group.attrs) + goto err_alloc; + + ret = sysfs_create_group(&p->kobj, &p->pkey_group); + if (ret) + goto err_free_pkey; + + p->gid_group.name = "gid_idx"; + p->gid_group.attrs = alloc_group_attrs(show_port_gid_idx, NULL, 1); + if (!p->gid_group.attrs) + goto err_free_pkey; + + ret = sysfs_create_group(&p->kobj, &p->gid_group); + if (ret) + goto err_free_gid; + + list_add_tail(&p->kobj.entry, &dev->pkeys.pkey_port_list[slave]); + return 0; + +err_free_gid: + kfree(p->gid_group.attrs[0]); + kfree(p->gid_group.attrs); + +err_free_pkey: + for (i = 0; i < dev->dev->caps.pkey_table_len[port_num]; ++i) + kfree(p->pkey_group.attrs[i]); + kfree(p->pkey_group.attrs); + +err_alloc: + kobject_put(dev->dev_ports_parent[slave]); + kfree(p); + return ret; +} + +static int register_one_pkey_tree(struct mlx4_ib_dev *dev, int slave) +{ + char name[32]; + int err; + int port; + struct kobject *p, *t; + struct mlx4_port *mport; + + get_name(dev, name, slave, sizeof name); + + dev->pkeys.device_parent[slave] = + kobject_create_and_add(name, kobject_get(dev->iov_parent)); + + if (!dev->pkeys.device_parent[slave]) { + err = -ENOMEM; + goto fail_dev; + } + + INIT_LIST_HEAD(&dev->pkeys.pkey_port_list[slave]); + + dev->dev_ports_parent[slave] = + kobject_create_and_add("ports", + kobject_get(dev->pkeys.device_parent[slave])); + + if (!dev->dev_ports_parent[slave]) { + err = -ENOMEM; + goto err_ports; + } + + for (port = 1; port <= dev->dev->caps.num_ports; ++port) { + err = add_port(dev, port, slave); + if (err) + goto err_add; + } + return 0; + +err_add: + list_for_each_entry_safe(p, t, + &dev->pkeys.pkey_port_list[slave], + entry) { + list_del(&p->entry); + mport = container_of(p, struct mlx4_port, kobj); + sysfs_remove_group(p, &mport->pkey_group); + sysfs_remove_group(p, &mport->gid_group); + kobject_put(p); + } + kobject_put(dev->dev_ports_parent[slave]); + +err_ports: + kobject_put(dev->pkeys.device_parent[slave]); + /* extra put for the device_parent create_and_add */ + kobject_put(dev->pkeys.device_parent[slave]); + +fail_dev: + kobject_put(dev->iov_parent); + return err; +} + +static int register_pkey_tree(struct mlx4_ib_dev *device) +{ + int i; + + if (!mlx4_is_master(device->dev)) + return 0; + + for (i = 0; i <= device->dev->persist->num_vfs; ++i) + register_one_pkey_tree(device, i); + + return 0; +} + +static void unregister_pkey_tree(struct mlx4_ib_dev *device) +{ + int slave; + struct kobject *p, *t; + struct mlx4_port *port; + + if (!mlx4_is_master(device->dev)) + return; + + for (slave = device->dev->persist->num_vfs; slave >= 0; --slave) { + list_for_each_entry_safe(p, t, + &device->pkeys.pkey_port_list[slave], + entry) { + list_del(&p->entry); + port = container_of(p, struct mlx4_port, kobj); + sysfs_remove_group(p, &port->pkey_group); + sysfs_remove_group(p, &port->gid_group); + kobject_put(p); + kobject_put(device->dev_ports_parent[slave]); + } + kobject_put(device->dev_ports_parent[slave]); + kobject_put(device->pkeys.device_parent[slave]); + kobject_put(device->pkeys.device_parent[slave]); + kobject_put(device->iov_parent); + } +} + +int mlx4_ib_device_register_sysfs(struct mlx4_ib_dev *dev) +{ + int i; + int ret = 0; + + if (!mlx4_is_master(dev->dev)) + return 0; + + dev->iov_parent = + kobject_create_and_add("iov", + kobject_get(dev->ib_dev.ports_parent->parent)); + if (!dev->iov_parent) { + ret = -ENOMEM; + goto err; + } + dev->ports_parent = + kobject_create_and_add("ports", + kobject_get(dev->iov_parent)); + if (!dev->iov_parent) { + ret = -ENOMEM; + goto err_ports; + } + + for (i = 1; i <= dev->ib_dev.phys_port_cnt; ++i) { + ret = add_port_entries(dev, i); + if (ret) + goto err_add_entries; + } + + ret = register_pkey_tree(dev); + if (ret) + goto err_add_entries; + return 0; + +err_add_entries: + kobject_put(dev->ports_parent); + +err_ports: + kobject_put(dev->iov_parent); +err: + kobject_put(dev->ib_dev.ports_parent->parent); + pr_err("mlx4_ib_device_register_sysfs error (%d)\n", ret); + return ret; +} + +static void unregister_alias_guid_tree(struct mlx4_ib_dev *device) +{ + struct mlx4_ib_iov_port *p; + int i; + + if (!mlx4_is_master(device->dev)) + return; + + for (i = 0; i < device->dev->caps.num_ports; i++) { + p = &device->iov_ports[i]; + kobject_put(p->admin_alias_parent); + kobject_put(p->gids_parent); + kobject_put(p->pkeys_parent); + kobject_put(p->mcgs_parent); + kobject_put(p->cur_port); + kobject_put(p->cur_port); + kobject_put(p->cur_port); + kobject_put(p->cur_port); + kobject_put(p->cur_port); + kobject_put(p->dev->ports_parent); + kfree(p->dentr_ar); + } +} + +void mlx4_ib_device_unregister_sysfs(struct mlx4_ib_dev *device) +{ + unregister_alias_guid_tree(device); + unregister_pkey_tree(device); + kobject_put(device->ports_parent); + kobject_put(device->iov_parent); + kobject_put(device->iov_parent); + kobject_put(device->ib_dev.ports_parent->parent); +} Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_sysfs.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_exp.h =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/mlx4_exp.h (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/mlx4_exp.h (revision 329159) @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2006, 2007 Cisco Systems. All rights reserved. + * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MLX4_EXP_H +#define MLX4_EXP_H + +#include +#include "mlx4_ib.h" + +struct ib_qp *mlx4_ib_exp_create_qp(struct ib_pd *pd, + struct ib_exp_qp_init_attr *init_attr, + struct ib_udata *udata); +int mlx4_ib_exp_query_device(struct ib_device *ibdev, + struct ib_exp_device_attr *props); + +#endif /* MLX4_EXP_H */ Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_exp.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_ah.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_ah.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_ah.c (revision 329159) @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include "mlx4_ib.h" + +int mlx4_ib_resolve_grh(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah_attr, + u8 *mac, int *is_mcast, u8 port) +{ + struct in6_addr in6; + + *is_mcast = 0; + + memcpy(&in6, ah_attr->grh.dgid.raw, sizeof in6); + if (rdma_link_local_addr(&in6)) + rdma_get_ll_mac(&in6, mac); + else if (rdma_is_multicast_addr(&in6)) { + rdma_get_mcast_mac(&in6, mac); + *is_mcast = 1; + } else + return -EINVAL; + + return 0; +} + +static struct ib_ah *create_ib_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr, + struct mlx4_ib_ah *ah) +{ + struct mlx4_dev *dev = to_mdev(pd->device)->dev; + + ah->av.ib.port_pd = cpu_to_be32(to_mpd(pd)->pdn | (ah_attr->port_num << 24)); + ah->av.ib.g_slid = ah_attr->src_path_bits; + if (ah_attr->ah_flags & IB_AH_GRH) { + ah->av.ib.g_slid |= 0x80; + ah->av.ib.gid_index = ah_attr->grh.sgid_index; + ah->av.ib.hop_limit = ah_attr->grh.hop_limit; + ah->av.ib.sl_tclass_flowlabel |= + cpu_to_be32((ah_attr->grh.traffic_class << 20) | + ah_attr->grh.flow_label); + memcpy(ah->av.ib.dgid, ah_attr->grh.dgid.raw, 16); + } + + ah->av.ib.dlid = cpu_to_be16(ah_attr->dlid); + if (ah_attr->static_rate) { + ah->av.ib.stat_rate = ah_attr->static_rate + MLX4_STAT_RATE_OFFSET; + while (ah->av.ib.stat_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET && + !(1 << ah->av.ib.stat_rate & dev->caps.stat_rate_support)) + --ah->av.ib.stat_rate; + } + ah->av.ib.sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 28); + + return &ah->ibah; +} + +static struct ib_ah *create_iboe_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr, + struct mlx4_ib_ah *ah) +{ + struct mlx4_ib_dev *ibdev = to_mdev(pd->device); + struct mlx4_dev *dev = ibdev->dev; + int is_mcast = 0; + struct in6_addr in6; + u16 vlan_tag; + + memcpy(&in6, ah_attr->grh.dgid.raw, sizeof(in6)); + if (rdma_is_multicast_addr(&in6)) { + is_mcast = 1; + resolve_mcast_mac(&in6, ah->av.eth.mac); + } else { + memcpy(ah->av.eth.mac, ah_attr->dmac, 6); + } + vlan_tag = ah_attr->vlan_id; + if (vlan_tag < 0x1000) + vlan_tag |= (ah_attr->sl & 7) << 13; + ah->av.eth.port_pd = cpu_to_be32(to_mpd(pd)->pdn | (ah_attr->port_num << 24)); + ah->av.eth.gid_index = ah_attr->grh.sgid_index; + ah->av.eth.vlan = cpu_to_be16(vlan_tag); + if (ah_attr->static_rate) { + ah->av.eth.stat_rate = ah_attr->static_rate + MLX4_STAT_RATE_OFFSET; + while (ah->av.eth.stat_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET && + !(1 << ah->av.eth.stat_rate & dev->caps.stat_rate_support)) + --ah->av.eth.stat_rate; + } + + /* + * HW requires multicast LID so we just choose one. + */ + if (is_mcast) + ah->av.ib.dlid = cpu_to_be16(0xc000); + + memcpy(ah->av.eth.dgid, ah_attr->grh.dgid.raw, 16); + ah->av.eth.sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 29); + + return &ah->ibah; +} + +struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr) +{ + struct mlx4_ib_ah *ah; + struct ib_ah *ret; + + ah = kzalloc(sizeof *ah, GFP_ATOMIC); + if (!ah) + return ERR_PTR(-ENOMEM); + + if (rdma_port_get_link_layer(pd->device, ah_attr->port_num) == IB_LINK_LAYER_ETHERNET) { + if (!(ah_attr->ah_flags & IB_AH_GRH)) { + ret = ERR_PTR(-EINVAL); + } else { + /* + * TBD: need to handle the case when we get + * called in an atomic context and there we + * might sleep. We don't expect this + * currently since we're working with link + * local addresses which we can translate + * without going to sleep. + */ + ret = create_iboe_ah(pd, ah_attr, ah); + } + + if (IS_ERR(ret)) + kfree(ah); + + return ret; + } else + return create_ib_ah(pd, ah_attr, ah); /* never fails */ +} + +int mlx4_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr) +{ + struct mlx4_ib_ah *ah = to_mah(ibah); + enum rdma_link_layer ll; + + memset(ah_attr, 0, sizeof *ah_attr); + ah_attr->sl = be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28; + ah_attr->port_num = be32_to_cpu(ah->av.ib.port_pd) >> 24; + ll = rdma_port_get_link_layer(ibah->device, ah_attr->port_num); + ah_attr->dlid = ll == IB_LINK_LAYER_INFINIBAND ? be16_to_cpu(ah->av.ib.dlid) : 0; + if (ah->av.ib.stat_rate) + ah_attr->static_rate = ah->av.ib.stat_rate - MLX4_STAT_RATE_OFFSET; + ah_attr->src_path_bits = ah->av.ib.g_slid & 0x7F; + + if (mlx4_ib_ah_grh_present(ah)) { + ah_attr->ah_flags = IB_AH_GRH; + + ah_attr->grh.traffic_class = + be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20; + ah_attr->grh.flow_label = + be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) & 0xfffff; + ah_attr->grh.hop_limit = ah->av.ib.hop_limit; + ah_attr->grh.sgid_index = ah->av.ib.gid_index; + memcpy(ah_attr->grh.dgid.raw, ah->av.ib.dgid, 16); + } + + return 0; +} + +int mlx4_ib_destroy_ah(struct ib_ah *ah) +{ + kfree(to_mah(ah)); + return 0; +} Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_ah.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_alias_GUID.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_alias_GUID.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_alias_GUID.c (revision 329159) @@ -0,0 +1,702 @@ +/* + * Copyright (c) 2012 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + /***********************************************************/ +/*This file support the handling of the Alias GUID feature. */ +/***********************************************************/ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mlx4_ib.h" + +/* +The driver keeps the current state of all guids, as they are in the HW. +Whenever we receive an smp mad GUIDInfo record, the data will be cached. +*/ + +struct mlx4_alias_guid_work_context { + u8 port; + struct mlx4_ib_dev *dev ; + struct ib_sa_query *sa_query; + struct completion done; + int query_id; + struct list_head list; + int block_num; + u8 method; +}; + +struct mlx4_next_alias_guid_work { + u8 port; + u8 block_num; + struct mlx4_sriov_alias_guid_info_rec_det rec_det; +}; + + +void mlx4_ib_update_cache_on_guid_change(struct mlx4_ib_dev *dev, int block_num, + u8 port_num, u8 *p_data) +{ + int i; + u64 guid_indexes; + int slave_id; + int port_index = port_num - 1; + + if (!mlx4_is_master(dev->dev)) + return; + + guid_indexes = be64_to_cpu((__force __be64) dev->sriov.alias_guid. + ports_guid[port_num - 1]. + all_rec_per_port[block_num].guid_indexes); + pr_debug("port: %d, guid_indexes: 0x%llx\n", port_num, + (unsigned long long)guid_indexes); + + for (i = 0; i < NUM_ALIAS_GUID_IN_REC; i++) { + /* The location of the specific index starts from bit number 4 + * until bit num 11 */ + if (test_bit(i + 4, (unsigned long *)&guid_indexes)) { + slave_id = (block_num * NUM_ALIAS_GUID_IN_REC) + i ; + if (slave_id >= dev->dev->num_slaves) { + pr_debug("The last slave: %d\n", slave_id); + return; + } + + /* cache the guid: */ + memcpy(&dev->sriov.demux[port_index].guid_cache[slave_id], + &p_data[i * GUID_REC_SIZE], + GUID_REC_SIZE); + } else + pr_debug("Guid number: %d in block: %d" + " was not updated\n", i, block_num); + } +} + +static __be64 get_cached_alias_guid(struct mlx4_ib_dev *dev, int port, int index) +{ + if (index >= NUM_ALIAS_GUID_PER_PORT) { + pr_err("%s: ERROR: asked for index:%d\n", __func__, index); + return (__force __be64) -1; + } + return *(__be64 *)&dev->sriov.demux[port - 1].guid_cache[index]; +} + + +ib_sa_comp_mask mlx4_ib_get_aguid_comp_mask_from_ix(int index) +{ + return IB_SA_COMP_MASK(4 + index); +} + +/* + * Whenever new GUID is set/unset (guid table change) create event and + * notify the relevant slave (master also should be notified). + * If the GUID value is not as we have in the cache the slave will not be + * updated; in this case it waits for the smp_snoop or the port management + * event to call the function and to update the slave. + * block_number - the index of the block (16 blocks available) + * port_number - 1 or 2 + */ +void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev, + int block_num, u8 port_num, + u8 *p_data) +{ + int i; + u64 guid_indexes; + int slave_id; + enum slave_port_state new_state; + enum slave_port_state prev_state; + __be64 tmp_cur_ag, form_cache_ag; + enum slave_port_gen_event gen_event; + + if (!mlx4_is_master(dev->dev)) + return; + + guid_indexes = be64_to_cpu((__force __be64) dev->sriov.alias_guid. + ports_guid[port_num - 1]. + all_rec_per_port[block_num].guid_indexes); + pr_debug("port: %d, guid_indexes: 0x%llx\n", port_num, + (unsigned long long)guid_indexes); + + /*calculate the slaves and notify them*/ + for (i = 0; i < NUM_ALIAS_GUID_IN_REC; i++) { + /* the location of the specific index runs from bits 4..11 */ + if (!(test_bit(i + 4, (unsigned long *)&guid_indexes))) + continue; + + slave_id = (block_num * NUM_ALIAS_GUID_IN_REC) + i ; + if (slave_id >= dev->dev->num_slaves) + return; + tmp_cur_ag = *(__be64 *)&p_data[i * GUID_REC_SIZE]; + form_cache_ag = get_cached_alias_guid(dev, port_num, + (NUM_ALIAS_GUID_IN_REC * block_num) + i); + /* + * Check if guid is not the same as in the cache, + * If it is different, wait for the snoop_smp or the port mgmt + * change event to update the slave on its port state change + */ + if (tmp_cur_ag != form_cache_ag) + continue; + mlx4_gen_guid_change_eqe(dev->dev, slave_id, port_num); + + /*2 cases: Valid GUID, and Invalid Guid*/ + + if (tmp_cur_ag != MLX4_NOT_SET_GUID) { /*valid GUID*/ + prev_state = mlx4_get_slave_port_state(dev->dev, slave_id, port_num); + new_state = set_and_calc_slave_port_state(dev->dev, slave_id, port_num, + MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID, + &gen_event); + pr_debug("slave: %d, port: %d prev_port_state: %d," + " new_port_state: %d, gen_event: %d\n", + slave_id, port_num, prev_state, new_state, gen_event); + if (gen_event == SLAVE_PORT_GEN_EVENT_UP) { + pr_debug("sending PORT_UP event to slave: %d, port: %d\n", + slave_id, port_num); + mlx4_gen_port_state_change_eqe(dev->dev, slave_id, + port_num, MLX4_PORT_CHANGE_SUBTYPE_ACTIVE); + } + } else { /* request to invalidate GUID */ + set_and_calc_slave_port_state(dev->dev, slave_id, port_num, + MLX4_PORT_STATE_IB_EVENT_GID_INVALID, + &gen_event); + pr_debug("sending PORT DOWN event to slave: %d, port: %d\n", + slave_id, port_num); + mlx4_gen_port_state_change_eqe(dev->dev, slave_id, port_num, + MLX4_PORT_CHANGE_SUBTYPE_DOWN); + } + } +} + +static void aliasguid_query_handler(int status, + struct ib_sa_guidinfo_rec *guid_rec, + void *context) +{ + struct mlx4_ib_dev *dev; + struct mlx4_alias_guid_work_context *cb_ctx = context; + u8 port_index; + int i; + struct mlx4_sriov_alias_guid_info_rec_det *rec; + unsigned long flags, flags1; + + if (!context) + return; + + dev = cb_ctx->dev; + port_index = cb_ctx->port - 1; + rec = &dev->sriov.alias_guid.ports_guid[port_index]. + all_rec_per_port[cb_ctx->block_num]; + + if (status) { + rec->status = MLX4_GUID_INFO_STATUS_IDLE; + pr_debug("(port: %d) failed: status = %d\n", + cb_ctx->port, status); + goto out; + } + + if (guid_rec->block_num != cb_ctx->block_num) { + pr_err("block num mismatch: %d != %d\n", + cb_ctx->block_num, guid_rec->block_num); + goto out; + } + + pr_debug("lid/port: %d/%d, block_num: %d\n", + be16_to_cpu(guid_rec->lid), cb_ctx->port, + guid_rec->block_num); + + rec = &dev->sriov.alias_guid.ports_guid[port_index]. + all_rec_per_port[guid_rec->block_num]; + + rec->status = MLX4_GUID_INFO_STATUS_SET; + rec->method = MLX4_GUID_INFO_RECORD_SET; + + for (i = 0 ; i < NUM_ALIAS_GUID_IN_REC; i++) { + __be64 tmp_cur_ag; + tmp_cur_ag = *(__be64 *)&guid_rec->guid_info_list[i * GUID_REC_SIZE]; + if ((cb_ctx->method == MLX4_GUID_INFO_RECORD_DELETE) + && (MLX4_NOT_SET_GUID == tmp_cur_ag)) { + pr_debug("%s:Record num %d in block_num:%d " + "was deleted by SM,ownership by %d " + "(0 = driver, 1=sysAdmin, 2=None)\n", + __func__, i, guid_rec->block_num, + rec->ownership); + rec->guid_indexes = rec->guid_indexes & + ~mlx4_ib_get_aguid_comp_mask_from_ix(i); + continue; + } + + /* check if the SM didn't assign one of the records. + * if it didn't, if it was not sysadmin request: + * ask the SM to give a new GUID, (instead of the driver request). + */ + if (tmp_cur_ag == MLX4_NOT_SET_GUID) { + mlx4_ib_warn(&dev->ib_dev, "%s:Record num %d in " + "block_num: %d was declined by SM, " + "ownership by %d (0 = driver, 1=sysAdmin," + " 2=None)\n", __func__, i, + guid_rec->block_num, rec->ownership); + if (rec->ownership == MLX4_GUID_DRIVER_ASSIGN) { + /* if it is driver assign, asks for new GUID from SM*/ + *(__be64 *)&rec->all_recs[i * GUID_REC_SIZE] = + MLX4_NOT_SET_GUID; + + /* Mark the record as not assigned, and let it + * be sent again in the next work sched.*/ + rec->status = MLX4_GUID_INFO_STATUS_IDLE; + rec->guid_indexes |= mlx4_ib_get_aguid_comp_mask_from_ix(i); + } + } else { + /* properly assigned record. */ + /* We save the GUID we just got from the SM in the + * admin_guid in order to be persistent, and in the + * request from the sm the process will ask for the same GUID */ + if (rec->ownership == MLX4_GUID_SYSADMIN_ASSIGN && + tmp_cur_ag != *(__be64 *)&rec->all_recs[i * GUID_REC_SIZE]) { + /* the sysadmin assignment failed.*/ + mlx4_ib_warn(&dev->ib_dev, "%s: Failed to set" + " admin guid after SysAdmin " + "configuration. " + "Record num %d in block_num:%d " + "was declined by SM, " + "new val(0x%llx) was kept\n", + __func__, i, + guid_rec->block_num, + (long long)be64_to_cpu(*(__be64 *) & + rec->all_recs[i * GUID_REC_SIZE])); + } else { + memcpy(&rec->all_recs[i * GUID_REC_SIZE], + &guid_rec->guid_info_list[i * GUID_REC_SIZE], + GUID_REC_SIZE); + } + } + } + /* + The func is call here to close the cases when the + sm doesn't send smp, so in the sa response the driver + notifies the slave. + */ + mlx4_ib_notify_slaves_on_guid_change(dev, guid_rec->block_num, + cb_ctx->port, + guid_rec->guid_info_list); +out: + spin_lock_irqsave(&dev->sriov.going_down_lock, flags); + spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); + if (!dev->sriov.is_going_down) + queue_delayed_work(dev->sriov.alias_guid.ports_guid[port_index].wq, + &dev->sriov.alias_guid.ports_guid[port_index]. + alias_guid_work, 0); + if (cb_ctx->sa_query) { + list_del(&cb_ctx->list); + kfree(cb_ctx); + } else + complete(&cb_ctx->done); + spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); + spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); +} + +static void invalidate_guid_record(struct mlx4_ib_dev *dev, u8 port, int index) +{ + int i; + u64 cur_admin_val; + ib_sa_comp_mask comp_mask = 0; + + dev->sriov.alias_guid.ports_guid[port - 1].all_rec_per_port[index].status + = MLX4_GUID_INFO_STATUS_IDLE; + dev->sriov.alias_guid.ports_guid[port - 1].all_rec_per_port[index].method + = MLX4_GUID_INFO_RECORD_SET; + + /* calculate the comp_mask for that record.*/ + for (i = 0; i < NUM_ALIAS_GUID_IN_REC; i++) { + cur_admin_val = + *(u64 *)&dev->sriov.alias_guid.ports_guid[port - 1]. + all_rec_per_port[index].all_recs[GUID_REC_SIZE * i]; + /* + check the admin value: if it's for delete (~00LL) or + it is the first guid of the first record (hw guid) or + the records is not in ownership of the sysadmin and the sm doesn't + need to assign GUIDs, then don't put it up for assignment. + */ + if (MLX4_GUID_FOR_DELETE_VAL == cur_admin_val || + (!index && !i) || + MLX4_GUID_NONE_ASSIGN == dev->sriov.alias_guid. + ports_guid[port - 1].all_rec_per_port[index].ownership) + continue; + comp_mask |= mlx4_ib_get_aguid_comp_mask_from_ix(i); + } + dev->sriov.alias_guid.ports_guid[port - 1]. + all_rec_per_port[index].guid_indexes = comp_mask; +} + +static int set_guid_rec(struct ib_device *ibdev, + u8 port, int index, + struct mlx4_sriov_alias_guid_info_rec_det *rec_det) +{ + int err; + struct mlx4_ib_dev *dev = to_mdev(ibdev); + struct ib_sa_guidinfo_rec guid_info_rec; + ib_sa_comp_mask comp_mask; + struct ib_port_attr attr; + struct mlx4_alias_guid_work_context *callback_context; + unsigned long resched_delay, flags, flags1; + struct list_head *head = + &dev->sriov.alias_guid.ports_guid[port - 1].cb_list; + + err = __mlx4_ib_query_port(ibdev, port, &attr, 1); + if (err) { + pr_debug("mlx4_ib_query_port failed (err: %d), port: %d\n", + err, port); + return err; + } + /*check the port was configured by the sm, otherwise no need to send */ + if (attr.state != IB_PORT_ACTIVE) { + pr_debug("port %d not active...rescheduling\n", port); + resched_delay = 5 * HZ; + err = -EAGAIN; + goto new_schedule; + } + + callback_context = kmalloc(sizeof *callback_context, GFP_KERNEL); + if (!callback_context) { + err = -ENOMEM; + resched_delay = HZ * 5; + goto new_schedule; + } + callback_context->port = port; + callback_context->dev = dev; + callback_context->block_num = index; + callback_context->method = rec_det->method; + memset(&guid_info_rec, 0, sizeof (struct ib_sa_guidinfo_rec)); + + guid_info_rec.lid = cpu_to_be16(attr.lid); + guid_info_rec.block_num = index; + + memcpy(guid_info_rec.guid_info_list, rec_det->all_recs, + GUID_REC_SIZE * NUM_ALIAS_GUID_IN_REC); + comp_mask = IB_SA_GUIDINFO_REC_LID | IB_SA_GUIDINFO_REC_BLOCK_NUM | + rec_det->guid_indexes; + + init_completion(&callback_context->done); + spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); + list_add_tail(&callback_context->list, head); + spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); + + callback_context->query_id = + ib_sa_guid_info_rec_query(dev->sriov.alias_guid.sa_client, + ibdev, port, &guid_info_rec, + comp_mask, rec_det->method, 1000, + GFP_KERNEL, aliasguid_query_handler, + callback_context, + &callback_context->sa_query); + if (callback_context->query_id < 0) { + pr_debug("ib_sa_guid_info_rec_query failed, query_id: " + "%d. will reschedule to the next 1 sec.\n", + callback_context->query_id); + spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); + list_del(&callback_context->list); + kfree(callback_context); + spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); + resched_delay = 1 * HZ; + err = -EAGAIN; + goto new_schedule; + } + err = 0; + goto out; + +new_schedule: + spin_lock_irqsave(&dev->sriov.going_down_lock, flags); + spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); + invalidate_guid_record(dev, port, index); + if (!dev->sriov.is_going_down) { + queue_delayed_work(dev->sriov.alias_guid.ports_guid[port - 1].wq, + &dev->sriov.alias_guid.ports_guid[port - 1].alias_guid_work, + resched_delay); + } + spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); + spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); + +out: + return err; +} + +void mlx4_ib_invalidate_all_guid_record(struct mlx4_ib_dev *dev, int port) +{ + int i; + unsigned long flags, flags1; + + pr_debug("port %d\n", port); + + spin_lock_irqsave(&dev->sriov.going_down_lock, flags); + spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); + for (i = 0; i < NUM_ALIAS_GUID_REC_IN_PORT; i++) + invalidate_guid_record(dev, port, i); + + if (mlx4_is_master(dev->dev) && !dev->sriov.is_going_down) { + /* + make sure no work waits in the queue, if the work is already + queued(not on the timer) the cancel will fail. That is not a problem + because we just want the work started. + */ + cancel_delayed_work(&dev->sriov.alias_guid. + ports_guid[port - 1].alias_guid_work); + queue_delayed_work(dev->sriov.alias_guid.ports_guid[port - 1].wq, + &dev->sriov.alias_guid.ports_guid[port - 1].alias_guid_work, + 0); + } + spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); + spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); +} + +/* The function returns the next record that was + * not configured (or failed to be configured) */ +static int get_next_record_to_update(struct mlx4_ib_dev *dev, u8 port, + struct mlx4_next_alias_guid_work *rec) +{ + int j; + unsigned long flags; + + for (j = 0; j < NUM_ALIAS_GUID_REC_IN_PORT; j++) { + spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags); + if (dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[j].status == + MLX4_GUID_INFO_STATUS_IDLE) { + memcpy(&rec->rec_det, + &dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[j], + sizeof (struct mlx4_sriov_alias_guid_info_rec_det)); + rec->port = port; + rec->block_num = j; + dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[j].status = + MLX4_GUID_INFO_STATUS_PENDING; + spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags); + return 0; + } + spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags); + } + return -ENOENT; +} + +static void set_administratively_guid_record(struct mlx4_ib_dev *dev, int port, + int rec_index, + struct mlx4_sriov_alias_guid_info_rec_det *rec_det) +{ + dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[rec_index].guid_indexes = + rec_det->guid_indexes; + memcpy(dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[rec_index].all_recs, + rec_det->all_recs, NUM_ALIAS_GUID_IN_REC * GUID_REC_SIZE); + dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[rec_index].status = + rec_det->status; +} + +static void set_all_slaves_guids(struct mlx4_ib_dev *dev, int port) +{ + int j; + struct mlx4_sriov_alias_guid_info_rec_det rec_det ; + + for (j = 0 ; j < NUM_ALIAS_GUID_REC_IN_PORT ; j++) { + memset(rec_det.all_recs, 0, NUM_ALIAS_GUID_IN_REC * GUID_REC_SIZE); + rec_det.guid_indexes = (!j ? 0 : IB_SA_GUIDINFO_REC_GID0) | + IB_SA_GUIDINFO_REC_GID1 | IB_SA_GUIDINFO_REC_GID2 | + IB_SA_GUIDINFO_REC_GID3 | IB_SA_GUIDINFO_REC_GID4 | + IB_SA_GUIDINFO_REC_GID5 | IB_SA_GUIDINFO_REC_GID6 | + IB_SA_GUIDINFO_REC_GID7; + rec_det.status = MLX4_GUID_INFO_STATUS_IDLE; + set_administratively_guid_record(dev, port, j, &rec_det); + } +} + +static void alias_guid_work(struct work_struct *work) +{ + struct delayed_work *delay = to_delayed_work(work); + int ret = 0; + struct mlx4_next_alias_guid_work *rec; + struct mlx4_sriov_alias_guid_port_rec_det *sriov_alias_port = + container_of(delay, struct mlx4_sriov_alias_guid_port_rec_det, + alias_guid_work); + struct mlx4_sriov_alias_guid *sriov_alias_guid = sriov_alias_port->parent; + struct mlx4_ib_sriov *ib_sriov = container_of(sriov_alias_guid, + struct mlx4_ib_sriov, + alias_guid); + struct mlx4_ib_dev *dev = container_of(ib_sriov, struct mlx4_ib_dev, sriov); + + rec = kzalloc(sizeof *rec, GFP_KERNEL); + if (!rec) { + pr_err("alias_guid_work: No Memory\n"); + return; + } + + pr_debug("starting [port: %d]...\n", sriov_alias_port->port + 1); + ret = get_next_record_to_update(dev, sriov_alias_port->port, rec); + if (ret) { + pr_debug("No more records to update.\n"); + goto out; + } + + set_guid_rec(&dev->ib_dev, rec->port + 1, rec->block_num, + &rec->rec_det); + +out: + kfree(rec); +} + + +void mlx4_ib_init_alias_guid_work(struct mlx4_ib_dev *dev, int port) +{ + unsigned long flags, flags1; + + if (!mlx4_is_master(dev->dev)) + return; + spin_lock_irqsave(&dev->sriov.going_down_lock, flags); + spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); + if (!dev->sriov.is_going_down) { + queue_delayed_work(dev->sriov.alias_guid.ports_guid[port].wq, + &dev->sriov.alias_guid.ports_guid[port].alias_guid_work, 0); + } + spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); + spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); +} + +void mlx4_ib_destroy_alias_guid_service(struct mlx4_ib_dev *dev) +{ + int i; + struct mlx4_ib_sriov *sriov = &dev->sriov; + struct mlx4_alias_guid_work_context *cb_ctx; + struct mlx4_sriov_alias_guid_port_rec_det *det; + struct ib_sa_query *sa_query; + unsigned long flags; + + for (i = 0 ; i < dev->num_ports; i++) { + cancel_delayed_work(&dev->sriov.alias_guid.ports_guid[i].alias_guid_work); + det = &sriov->alias_guid.ports_guid[i]; + spin_lock_irqsave(&sriov->alias_guid.ag_work_lock, flags); + while (!list_empty(&det->cb_list)) { + cb_ctx = list_entry(det->cb_list.next, + struct mlx4_alias_guid_work_context, + list); + sa_query = cb_ctx->sa_query; + cb_ctx->sa_query = NULL; + list_del(&cb_ctx->list); + spin_unlock_irqrestore(&sriov->alias_guid.ag_work_lock, flags); + ib_sa_cancel_query(cb_ctx->query_id, sa_query); + wait_for_completion(&cb_ctx->done); + kfree(cb_ctx); + spin_lock_irqsave(&sriov->alias_guid.ag_work_lock, flags); + } + spin_unlock_irqrestore(&sriov->alias_guid.ag_work_lock, flags); + } + for (i = 0 ; i < dev->num_ports; i++) { + flush_workqueue(dev->sriov.alias_guid.ports_guid[i].wq); + destroy_workqueue(dev->sriov.alias_guid.ports_guid[i].wq); + } + ib_sa_unregister_client(dev->sriov.alias_guid.sa_client); + kfree(dev->sriov.alias_guid.sa_client); +} + +int mlx4_ib_init_alias_guid_service(struct mlx4_ib_dev *dev) +{ + char alias_wq_name[15]; + int ret = 0; + int i, j, k; + union ib_gid gid; + + if (!mlx4_is_master(dev->dev)) + return 0; + dev->sriov.alias_guid.sa_client = + kzalloc(sizeof *dev->sriov.alias_guid.sa_client, GFP_KERNEL); + if (!dev->sriov.alias_guid.sa_client) + return -ENOMEM; + + ib_sa_register_client(dev->sriov.alias_guid.sa_client); + + spin_lock_init(&dev->sriov.alias_guid.ag_work_lock); + + for (i = 1; i <= dev->num_ports; ++i) { + if (dev->ib_dev.query_gid(&dev->ib_dev , i, 0, &gid)) { + ret = -EFAULT; + goto err_unregister; + } + } + + for (i = 0 ; i < dev->num_ports; i++) { + memset(&dev->sriov.alias_guid.ports_guid[i], 0, + sizeof (struct mlx4_sriov_alias_guid_port_rec_det)); + /*Check if the SM doesn't need to assign the GUIDs*/ + for (j = 0; j < NUM_ALIAS_GUID_REC_IN_PORT; j++) { + if (mlx4_ib_sm_guid_assign) { + dev->sriov.alias_guid.ports_guid[i]. + all_rec_per_port[j]. + ownership = MLX4_GUID_DRIVER_ASSIGN; + continue; + } + dev->sriov.alias_guid.ports_guid[i].all_rec_per_port[j]. + ownership = MLX4_GUID_NONE_ASSIGN; + /*mark each val as it was deleted, + till the sysAdmin will give it valid val*/ + for (k = 0; k < NUM_ALIAS_GUID_IN_REC; k++) { + *(__be64 *)&dev->sriov.alias_guid.ports_guid[i]. + all_rec_per_port[j].all_recs[GUID_REC_SIZE * k] = + cpu_to_be64(MLX4_GUID_FOR_DELETE_VAL); + } + } + INIT_LIST_HEAD(&dev->sriov.alias_guid.ports_guid[i].cb_list); + /*prepare the records, set them to be allocated by sm*/ + for (j = 0 ; j < NUM_ALIAS_GUID_REC_IN_PORT; j++) + invalidate_guid_record(dev, i + 1, j); + + dev->sriov.alias_guid.ports_guid[i].parent = &dev->sriov.alias_guid; + dev->sriov.alias_guid.ports_guid[i].port = i; + if (mlx4_ib_sm_guid_assign) + set_all_slaves_guids(dev, i); + + snprintf(alias_wq_name, sizeof alias_wq_name, "alias_guid%d", i); + dev->sriov.alias_guid.ports_guid[i].wq = + create_singlethread_workqueue(alias_wq_name); + if (!dev->sriov.alias_guid.ports_guid[i].wq) { + ret = -ENOMEM; + goto err_thread; + } + INIT_DELAYED_WORK(&dev->sriov.alias_guid.ports_guid[i].alias_guid_work, + alias_guid_work); + } + return 0; + +err_thread: + for (--i; i >= 0; i--) { + destroy_workqueue(dev->sriov.alias_guid.ports_guid[i].wq); + dev->sriov.alias_guid.ports_guid[i].wq = NULL; + } + +err_unregister: + ib_sa_unregister_client(dev->sriov.alias_guid.sa_client); + kfree(dev->sriov.alias_guid.sa_client); + dev->sriov.alias_guid.sa_client = NULL; + pr_err("init_alias_guid_service: Failed. (ret:%d)\n", ret); + return ret; +} Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_alias_GUID.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_cm.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_cm.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_cm.c (revision 329159) @@ -0,0 +1,484 @@ +/* + * Copyright (c) 2012 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include + +#include +#include +#include +#include + +#include "mlx4_ib.h" + +#define CM_CLEANUP_CACHE_TIMEOUT (5 * HZ) + +struct id_map_entry { + struct rb_node node; + + u32 sl_cm_id; + u32 pv_cm_id; + int slave_id; + int scheduled_delete; + struct mlx4_ib_dev *dev; + + struct list_head list; + struct delayed_work timeout; +}; + +struct cm_generic_msg { + struct ib_mad_hdr hdr; + + __be32 local_comm_id; + __be32 remote_comm_id; +}; + +struct cm_sidr_generic_msg { + struct ib_mad_hdr hdr; + __be32 request_id; +}; + +struct cm_req_msg { + unsigned char unused[0x60]; + union ib_gid primary_path_sgid; +}; + + +static void set_local_comm_id(struct ib_mad *mad, u32 cm_id) +{ + if (mad->mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID) { + struct cm_sidr_generic_msg *msg = + (struct cm_sidr_generic_msg *)mad; + msg->request_id = cpu_to_be32(cm_id); + } else if (mad->mad_hdr.attr_id == CM_SIDR_REP_ATTR_ID) { + pr_err("trying to set local_comm_id in SIDR_REP\n"); + return; + } else { + struct cm_generic_msg *msg = (struct cm_generic_msg *)mad; + msg->local_comm_id = cpu_to_be32(cm_id); + } +} + +static u32 get_local_comm_id(struct ib_mad *mad) +{ + if (mad->mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID) { + struct cm_sidr_generic_msg *msg = + (struct cm_sidr_generic_msg *)mad; + return be32_to_cpu(msg->request_id); + } else if (mad->mad_hdr.attr_id == CM_SIDR_REP_ATTR_ID) { + pr_err("trying to set local_comm_id in SIDR_REP\n"); + return -1; + } else { + struct cm_generic_msg *msg = (struct cm_generic_msg *)mad; + return be32_to_cpu(msg->local_comm_id); + } +} + +static void set_remote_comm_id(struct ib_mad *mad, u32 cm_id) +{ + if (mad->mad_hdr.attr_id == CM_SIDR_REP_ATTR_ID) { + struct cm_sidr_generic_msg *msg = + (struct cm_sidr_generic_msg *)mad; + msg->request_id = cpu_to_be32(cm_id); + } else if (mad->mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID) { + pr_err("trying to set remote_comm_id in SIDR_REQ\n"); + return; + } else { + struct cm_generic_msg *msg = (struct cm_generic_msg *)mad; + msg->remote_comm_id = cpu_to_be32(cm_id); + } +} + +static u32 get_remote_comm_id(struct ib_mad *mad) +{ + if (mad->mad_hdr.attr_id == CM_SIDR_REP_ATTR_ID) { + struct cm_sidr_generic_msg *msg = + (struct cm_sidr_generic_msg *)mad; + return be32_to_cpu(msg->request_id); + } else if (mad->mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID) { + pr_err("trying to set remote_comm_id in SIDR_REQ\n"); + return -1; + } else { + struct cm_generic_msg *msg = (struct cm_generic_msg *)mad; + return be32_to_cpu(msg->remote_comm_id); + } +} + +static union ib_gid gid_from_req_msg(struct ib_device *ibdev, struct ib_mad *mad) +{ + struct cm_req_msg *msg = (struct cm_req_msg *)mad; + + return msg->primary_path_sgid; +} + +/* Lock should be taken before called */ +static struct id_map_entry * +id_map_find_by_sl_id(struct ib_device *ibdev, u32 slave_id, u32 sl_cm_id) +{ + struct rb_root *sl_id_map = &to_mdev(ibdev)->sriov.sl_id_map; + struct rb_node *node = sl_id_map->rb_node; + + while (node) { + struct id_map_entry *id_map_entry = + rb_entry(node, struct id_map_entry, node); + + if (id_map_entry->sl_cm_id > sl_cm_id) + node = node->rb_left; + else if (id_map_entry->sl_cm_id < sl_cm_id) + node = node->rb_right; + else if (id_map_entry->slave_id > slave_id) + node = node->rb_left; + else if (id_map_entry->slave_id < slave_id) + node = node->rb_right; + else + return id_map_entry; + } + return NULL; +} + +static void id_map_ent_timeout(struct work_struct *work) +{ + struct delayed_work *delay = to_delayed_work(work); + struct id_map_entry *ent = container_of(delay, struct id_map_entry, timeout); + struct id_map_entry *db_ent, *found_ent; + struct mlx4_ib_dev *dev = ent->dev; + struct mlx4_ib_sriov *sriov = &dev->sriov; + struct rb_root *sl_id_map = &sriov->sl_id_map; + int pv_id = (int) ent->pv_cm_id; + + spin_lock(&sriov->id_map_lock); + db_ent = (struct id_map_entry *)idr_find(&sriov->pv_id_table, pv_id); + if (!db_ent) + goto out; + found_ent = id_map_find_by_sl_id(&dev->ib_dev, ent->slave_id, ent->sl_cm_id); + if (found_ent && found_ent == ent) + rb_erase(&found_ent->node, sl_id_map); + idr_remove(&sriov->pv_id_table, pv_id); + +out: + list_del(&ent->list); + spin_unlock(&sriov->id_map_lock); + kfree(ent); +} + +static void id_map_find_del(struct ib_device *ibdev, int pv_cm_id) +{ + struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; + struct rb_root *sl_id_map = &sriov->sl_id_map; + struct id_map_entry *ent, *found_ent; + + spin_lock(&sriov->id_map_lock); + ent = (struct id_map_entry *)idr_find(&sriov->pv_id_table, pv_cm_id); + if (!ent) + goto out; + found_ent = id_map_find_by_sl_id(ibdev, ent->slave_id, ent->sl_cm_id); + if (found_ent && found_ent == ent) + rb_erase(&found_ent->node, sl_id_map); + idr_remove(&sriov->pv_id_table, pv_cm_id); +out: + spin_unlock(&sriov->id_map_lock); +} + +static void sl_id_map_add(struct ib_device *ibdev, struct id_map_entry *new) +{ + struct rb_root *sl_id_map = &to_mdev(ibdev)->sriov.sl_id_map; + struct rb_node **link = &sl_id_map->rb_node, *parent = NULL; + struct id_map_entry *ent; + int slave_id = new->slave_id; + int sl_cm_id = new->sl_cm_id; + + ent = id_map_find_by_sl_id(ibdev, slave_id, sl_cm_id); + if (ent) { + pr_debug("overriding existing sl_id_map entry (cm_id = %x)\n", + sl_cm_id); + + rb_replace_node(&ent->node, &new->node, sl_id_map); + return; + } + + /* Go to the bottom of the tree */ + while (*link) { + parent = *link; + ent = rb_entry(parent, struct id_map_entry, node); + + if (ent->sl_cm_id > sl_cm_id || (ent->sl_cm_id == sl_cm_id && ent->slave_id > slave_id)) + link = &(*link)->rb_left; + else + link = &(*link)->rb_right; + } + + rb_link_node(&new->node, parent, link); + rb_insert_color(&new->node, sl_id_map); +} + +static struct id_map_entry * +id_map_alloc(struct ib_device *ibdev, int slave_id, u32 sl_cm_id) +{ + int ret, id; + static int next_id; + struct id_map_entry *ent; + struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; + + ent = kmalloc(sizeof (struct id_map_entry), GFP_KERNEL); + if (!ent) { + mlx4_ib_warn(ibdev, "Couldn't allocate id cache entry - out of memory\n"); + return ERR_PTR(-ENOMEM); + } + + ent->sl_cm_id = sl_cm_id; + ent->slave_id = slave_id; + ent->scheduled_delete = 0; + ent->dev = to_mdev(ibdev); + INIT_DELAYED_WORK(&ent->timeout, id_map_ent_timeout); + + do { + spin_lock(&to_mdev(ibdev)->sriov.id_map_lock); + ret = idr_get_new_above(&sriov->pv_id_table, ent, + next_id, &id); + if (!ret) { + next_id = ((unsigned) id + 1) & MAX_IDR_MASK; + ent->pv_cm_id = (u32)id; + sl_id_map_add(ibdev, ent); + } + + spin_unlock(&sriov->id_map_lock); + } while (ret == -EAGAIN && idr_pre_get(&sriov->pv_id_table, GFP_KERNEL)); + /*the function idr_get_new_above can return -ENOSPC, so don't insert in that case.*/ + if (!ret) { + spin_lock(&sriov->id_map_lock); + list_add_tail(&ent->list, &sriov->cm_list); + spin_unlock(&sriov->id_map_lock); + return ent; + } + /*error flow*/ + kfree(ent); + mlx4_ib_warn(ibdev, "No more space in the idr (err:0x%x)\n", ret); + return ERR_PTR(-ENOMEM); +} + +static struct id_map_entry * +id_map_get(struct ib_device *ibdev, int *pv_cm_id, int sl_cm_id, int slave_id) +{ + struct id_map_entry *ent; + struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; + + spin_lock(&sriov->id_map_lock); + if (*pv_cm_id == -1) { + ent = id_map_find_by_sl_id(ibdev, sl_cm_id, slave_id); + if (ent) + *pv_cm_id = (int) ent->pv_cm_id; + } else + ent = (struct id_map_entry *)idr_find(&sriov->pv_id_table, *pv_cm_id); + spin_unlock(&sriov->id_map_lock); + + return ent; +} + +static void schedule_delayed(struct ib_device *ibdev, struct id_map_entry *id) +{ + struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; + unsigned long flags; + + spin_lock(&sriov->id_map_lock); + spin_lock_irqsave(&sriov->going_down_lock, flags); + /*make sure that there is no schedule inside the scheduled work.*/ + if (!sriov->is_going_down) { + id->scheduled_delete = 1; + schedule_delayed_work(&id->timeout, CM_CLEANUP_CACHE_TIMEOUT); + } + spin_unlock_irqrestore(&sriov->going_down_lock, flags); + spin_unlock(&sriov->id_map_lock); +} + +int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id, + struct ib_mad *mad) +{ + struct id_map_entry *id; + u32 sl_cm_id; + int pv_cm_id = -1; + + if (mad->mad_hdr.attr_id == CM_REQ_ATTR_ID || + mad->mad_hdr.attr_id == CM_REP_ATTR_ID || + mad->mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID || + mad->mad_hdr.attr_id == CM_SIDR_REP_ATTR_ID) { + sl_cm_id = get_local_comm_id(mad); + id = id_map_alloc(ibdev, slave_id, sl_cm_id); + if (IS_ERR(id)) { + mlx4_ib_warn(ibdev, "%s: id{slave: %d, sl_cm_id: 0x%x} Failed to id_map_alloc\n", + __func__, slave_id, sl_cm_id); + return PTR_ERR(id); + } + } else if (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID || + mad->mad_hdr.attr_id == CM_SIDR_REP_ATTR_ID) { + return 0; + } else { + sl_cm_id = get_local_comm_id(mad); + id = id_map_get(ibdev, &pv_cm_id, slave_id, sl_cm_id); + } + + if (!id) { + pr_debug("id{slave: %d, sl_cm_id: 0x%x} is NULL!\n", + slave_id, sl_cm_id); + return -EINVAL; + } + + set_local_comm_id(mad, id->pv_cm_id); + + if (mad->mad_hdr.attr_id == CM_DREQ_ATTR_ID) + schedule_delayed(ibdev, id); + else if (mad->mad_hdr.attr_id == CM_DREP_ATTR_ID) + id_map_find_del(ibdev, pv_cm_id); + + return 0; +} + +int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave, + struct ib_mad *mad, int is_eth) +{ + u32 pv_cm_id; + struct id_map_entry *id; + + if (mad->mad_hdr.attr_id == CM_REQ_ATTR_ID || + mad->mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID) { + union ib_gid gid; + + if (is_eth) + return 0; + + gid = gid_from_req_msg(ibdev, mad); + *slave = mlx4_ib_find_real_gid(ibdev, port, gid.global.interface_id); + if (*slave < 0) { + mlx4_ib_warn(ibdev, "failed matching slave_id by gid (0x%llx)\n", + (unsigned long long)gid.global.interface_id); + return -ENOENT; + } + return 0; + } + + pv_cm_id = get_remote_comm_id(mad); + id = id_map_get(ibdev, (int *)&pv_cm_id, -1, -1); + + if (!id) { + pr_debug("Couldn't find an entry for pv_cm_id 0x%x\n", pv_cm_id); + return -ENOENT; + } + + if (!is_eth) + *slave = id->slave_id; + set_remote_comm_id(mad, id->sl_cm_id); + + if (mad->mad_hdr.attr_id == CM_DREQ_ATTR_ID) + schedule_delayed(ibdev, id); + else if (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID || + mad->mad_hdr.attr_id == CM_DREP_ATTR_ID) { + id_map_find_del(ibdev, (int) pv_cm_id); + } + + return 0; +} + +void mlx4_ib_cm_paravirt_init(struct mlx4_ib_dev *dev) +{ + spin_lock_init(&dev->sriov.id_map_lock); + INIT_LIST_HEAD(&dev->sriov.cm_list); + dev->sriov.sl_id_map = RB_ROOT; + idr_init(&dev->sriov.pv_id_table); + idr_pre_get(&dev->sriov.pv_id_table, GFP_KERNEL); +} + +/* slave = -1 ==> all slaves */ +/* TBD -- call paravirt clean for single slave. Need for slave RESET event */ +void mlx4_ib_cm_paravirt_clean(struct mlx4_ib_dev *dev, int slave) +{ + struct mlx4_ib_sriov *sriov = &dev->sriov; + struct rb_root *sl_id_map = &sriov->sl_id_map; + struct list_head lh; + struct rb_node *nd; + int need_flush = 1; + struct id_map_entry *map, *tmp_map; + /* cancel all delayed work queue entries */ + INIT_LIST_HEAD(&lh); + spin_lock(&sriov->id_map_lock); + list_for_each_entry_safe(map, tmp_map, &dev->sriov.cm_list, list) { + if (slave < 0 || slave == map->slave_id) { + if (map->scheduled_delete) + need_flush &= !!cancel_delayed_work(&map->timeout); + } + } + + spin_unlock(&sriov->id_map_lock); + + if (!need_flush) + flush_scheduled_work(); /* make sure all timers were flushed */ + + /* now, remove all leftover entries from databases*/ + spin_lock(&sriov->id_map_lock); + if (slave < 0) { + while (rb_first(sl_id_map)) { + struct id_map_entry *ent = + rb_entry(rb_first(sl_id_map), + struct id_map_entry, node); + + rb_erase(&ent->node, sl_id_map); + idr_remove(&sriov->pv_id_table, (int) ent->pv_cm_id); + } + list_splice_init(&dev->sriov.cm_list, &lh); + } else { + /* first, move nodes belonging to slave to db remove list */ + nd = rb_first(sl_id_map); + while (nd) { + struct id_map_entry *ent = + rb_entry(nd, struct id_map_entry, node); + nd = rb_next(nd); + if (ent->slave_id == slave) + list_move_tail(&ent->list, &lh); + } + /* remove those nodes from databases */ + list_for_each_entry_safe(map, tmp_map, &lh, list) { + rb_erase(&map->node, sl_id_map); + idr_remove(&sriov->pv_id_table, (int) map->pv_cm_id); + } + + /* add remaining nodes from cm_list */ + list_for_each_entry_safe(map, tmp_map, &dev->sriov.cm_list, list) { + if (slave == map->slave_id) + list_move_tail(&map->list, &lh); + } + } + + spin_unlock(&sriov->id_map_lock); + + /* free any map entries left behind due to cancel_delayed_work above */ + list_for_each_entry_safe(map, tmp_map, &lh, list) { + list_del(&map->list); + kfree(map); + } +} Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_cm.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_doorbell.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_doorbell.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_doorbell.c (revision 329159) @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include + +#include "mlx4_ib.h" + +struct mlx4_ib_user_db_page { + struct list_head list; + struct ib_umem *umem; + unsigned long user_virt; + int refcnt; +}; + +int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context, unsigned long virt, + struct mlx4_db *db) +{ + struct mlx4_ib_user_db_page *page; + int err = 0; + + mutex_lock(&context->db_page_mutex); + + list_for_each_entry(page, &context->db_page_list, list) + if (page->user_virt == (virt & PAGE_MASK)) + goto found; + + page = kmalloc(sizeof *page, GFP_KERNEL); + if (!page) { + err = -ENOMEM; + goto out; + } + + page->user_virt = (virt & PAGE_MASK); + page->refcnt = 0; + page->umem = ib_umem_get(&context->ibucontext, virt & PAGE_MASK, + PAGE_SIZE, 0, 0); + if (IS_ERR(page->umem)) { + err = PTR_ERR(page->umem); + kfree(page); + goto out; + } + + list_add(&page->list, &context->db_page_list); + +found: + db->dma = sg_dma_address(page->umem->sg_head.sgl) + (virt & ~PAGE_MASK); + db->u.user_page = page; + ++page->refcnt; + +out: + mutex_unlock(&context->db_page_mutex); + + return err; +} + +void mlx4_ib_db_unmap_user(struct mlx4_ib_ucontext *context, struct mlx4_db *db) +{ + mutex_lock(&context->db_page_mutex); + + if (!--db->u.user_page->refcnt) { + list_del(&db->u.user_page->list); + ib_umem_release(db->u.user_page->umem); + kfree(db->u.user_page); + } + + mutex_unlock(&context->db_page_mutex); +} Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_doorbell.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_exp.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_exp.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_exp.c (revision 329159) @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "mlx4_ib.h" +#include "mlx4_exp.h" +#include + +int mlx4_ib_exp_query_device(struct ib_device *ibdev, + struct ib_exp_device_attr *props) +{ + struct ib_device_attr *base = &props->base; + struct mlx4_ib_dev *dev = to_mdev(ibdev); + int ret = mlx4_ib_query_device(ibdev, &props->base); + + props->exp_comp_mask = IB_EXP_DEVICE_ATTR_INLINE_RECV_SZ; + props->inline_recv_sz = dev->dev->caps.max_rq_sg * sizeof(struct mlx4_wqe_data_seg); + props->device_cap_flags2 = 0; + + /* move RSS device cap from device_cap to device_cap_flags2 */ + if (base->device_cap_flags & IB_DEVICE_QPG) { + props->device_cap_flags2 |= IB_EXP_DEVICE_QPG; + if (base->device_cap_flags & IB_DEVICE_UD_RSS) + props->device_cap_flags2 |= IB_EXP_DEVICE_UD_RSS; + } + base->device_cap_flags &= ~(IB_DEVICE_QPG | + IB_DEVICE_UD_RSS | + IB_DEVICE_UD_TSS); + + if (base->max_rss_tbl_sz > 0) { + props->max_rss_tbl_sz = base->max_rss_tbl_sz; + props->exp_comp_mask |= IB_EXP_DEVICE_ATTR_RSS_TBL_SZ; + } else { + props->max_rss_tbl_sz = 0; + props->exp_comp_mask &= ~IB_EXP_DEVICE_ATTR_RSS_TBL_SZ; + } + + if (props->device_cap_flags2) + props->exp_comp_mask |= IB_EXP_DEVICE_ATTR_CAP_FLAGS2; + + return ret; +} + +/* + * Experimental functions + */ +struct ib_qp *mlx4_ib_exp_create_qp(struct ib_pd *pd, + struct ib_exp_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + int rwqe_size; + struct ib_qp *qp; + struct mlx4_ib_qp *mqp; + int use_inlr; + struct mlx4_ib_dev *dev; + + if (init_attr->max_inl_recv && !udata) + return ERR_PTR(-EINVAL); + + use_inlr = mlx4_ib_qp_has_rq((struct ib_qp_init_attr *)init_attr) && + init_attr->max_inl_recv && pd; + if (use_inlr) { + rwqe_size = roundup_pow_of_two(max(1U, init_attr->cap.max_recv_sge)) * + sizeof(struct mlx4_wqe_data_seg); + if (rwqe_size < init_attr->max_inl_recv) { + dev = to_mdev(pd->device); + init_attr->max_inl_recv = min(init_attr->max_inl_recv, + (u32)(dev->dev->caps.max_rq_sg * + sizeof(struct mlx4_wqe_data_seg))); + init_attr->cap.max_recv_sge = roundup_pow_of_two(init_attr->max_inl_recv) / + sizeof(struct mlx4_wqe_data_seg); + } + } else { + init_attr->max_inl_recv = 0; + } + qp = mlx4_ib_create_qp(pd, (struct ib_qp_init_attr *)init_attr, udata); + if (IS_ERR(qp)) + return qp; + + if (use_inlr) { + mqp = to_mqp(qp); + mqp->max_inlr_data = 1 << mqp->rq.wqe_shift; + init_attr->max_inl_recv = mqp->max_inlr_data; + } + + return qp; +} Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_exp.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_mcg.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_mcg.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_mcg.c (revision 329159) @@ -0,0 +1,1260 @@ +/* + * Copyright (c) 2012 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include + +#include +#include +#include + +#include "mlx4_ib.h" + +#define MAX_VFS 80 +#define MAX_PEND_REQS_PER_FUNC 4 +#define MAD_TIMEOUT_MS 2000 + +#define mcg_warn(fmt, arg...) pr_warn("MCG WARNING: " fmt, ##arg) +#define mcg_error(fmt, arg...) pr_err(fmt, ##arg) +#define mcg_warn_group(group, format, arg...) \ + pr_warn("%s-%d: %16s (port %d): WARNING: " format, __func__, __LINE__,\ + (group)->name, group->demux->port, ## arg) + +#define mcg_error_group(group, format, arg...) \ + pr_err(" %16s: " format, (group)->name, ## arg) + + +static union ib_gid mgid0; + +static struct workqueue_struct *clean_wq; + +enum mcast_state { + MCAST_NOT_MEMBER = 0, + MCAST_MEMBER, +}; + +enum mcast_group_state { + MCAST_IDLE, + MCAST_JOIN_SENT, + MCAST_LEAVE_SENT, + MCAST_RESP_READY +}; + +struct mcast_member { + enum mcast_state state; + uint8_t join_state; + int num_pend_reqs; + struct list_head pending; +}; + +struct ib_sa_mcmember_data { + union ib_gid mgid; + union ib_gid port_gid; + __be32 qkey; + __be16 mlid; + u8 mtusel_mtu; + u8 tclass; + __be16 pkey; + u8 ratesel_rate; + u8 lifetmsel_lifetm; + __be32 sl_flowlabel_hoplimit; + u8 scope_join_state; + u8 proxy_join; + u8 reserved[2]; +}; + +struct mcast_group { + struct ib_sa_mcmember_data rec; + struct rb_node node; + struct list_head mgid0_list; + struct mlx4_ib_demux_ctx *demux; + struct mcast_member func[MAX_VFS]; + struct mutex lock; + struct work_struct work; + struct list_head pending_list; + int members[3]; + enum mcast_group_state state; + enum mcast_group_state prev_state; + struct ib_sa_mad response_sa_mad; + __be64 last_req_tid; + + char name[33]; /* MGID string */ + struct device_attribute dentry; + + /* refcount is the reference count for the following: + 1. Each queued request + 2. Each invocation of the worker thread + 3. Membership of the port at the SA + */ + atomic_t refcount; + + /* delayed work to clean pending SM request */ + struct delayed_work timeout_work; + struct list_head cleanup_list; +}; + +struct mcast_req { + int func; + struct ib_sa_mad sa_mad; + struct list_head group_list; + struct list_head func_list; + struct mcast_group *group; + int clean; +}; + + +#define safe_atomic_dec(ref) \ + do {\ + if (atomic_dec_and_test(ref)) \ + mcg_warn_group(group, "did not expect to reach zero\n"); \ + } while (0) + +static const char *get_state_string(enum mcast_group_state state) +{ + switch (state) { + case MCAST_IDLE: + return "MCAST_IDLE"; + case MCAST_JOIN_SENT: + return "MCAST_JOIN_SENT"; + case MCAST_LEAVE_SENT: + return "MCAST_LEAVE_SENT"; + case MCAST_RESP_READY: + return "MCAST_RESP_READY"; + } + return "Invalid State"; +} + +static struct mcast_group *mcast_find(struct mlx4_ib_demux_ctx *ctx, + union ib_gid *mgid) +{ + struct rb_node *node = ctx->mcg_table.rb_node; + struct mcast_group *group; + int ret; + + while (node) { + group = rb_entry(node, struct mcast_group, node); + ret = memcmp(mgid->raw, group->rec.mgid.raw, sizeof *mgid); + if (!ret) + return group; + + if (ret < 0) + node = node->rb_left; + else + node = node->rb_right; + } + return NULL; +} + +static struct mcast_group *mcast_insert(struct mlx4_ib_demux_ctx *ctx, + struct mcast_group *group) +{ + struct rb_node **link = &ctx->mcg_table.rb_node; + struct rb_node *parent = NULL; + struct mcast_group *cur_group; + int ret; + + while (*link) { + parent = *link; + cur_group = rb_entry(parent, struct mcast_group, node); + + ret = memcmp(group->rec.mgid.raw, cur_group->rec.mgid.raw, + sizeof group->rec.mgid); + if (ret < 0) + link = &(*link)->rb_left; + else if (ret > 0) + link = &(*link)->rb_right; + else + return cur_group; + } + rb_link_node(&group->node, parent, link); + rb_insert_color(&group->node, &ctx->mcg_table); + return NULL; +} + +static int send_mad_to_wire(struct mlx4_ib_demux_ctx *ctx, struct ib_mad *mad) +{ + struct mlx4_ib_dev *dev = ctx->dev; + struct ib_ah_attr ah_attr; + + spin_lock(&dev->sm_lock); + if (!dev->sm_ah[ctx->port - 1]) { + /* port is not yet Active, sm_ah not ready */ + spin_unlock(&dev->sm_lock); + return -EAGAIN; + } + mlx4_ib_query_ah(dev->sm_ah[ctx->port - 1], &ah_attr); + spin_unlock(&dev->sm_lock); + return mlx4_ib_send_to_wire(dev, mlx4_master_func_num(dev->dev), ctx->port, + IB_QPT_GSI, 0, 1, IB_QP1_QKEY, &ah_attr, 0, mad); +} + +static int send_mad_to_slave(int slave, struct mlx4_ib_demux_ctx *ctx, + struct ib_mad *mad) +{ + struct mlx4_ib_dev *dev = ctx->dev; + struct ib_mad_agent *agent = dev->send_agent[ctx->port - 1][1]; + struct ib_wc wc; + struct ib_ah_attr ah_attr; + + /* Our agent might not yet be registered when mads start to arrive */ + if (!agent) + return -EAGAIN; + + ib_query_ah(dev->sm_ah[ctx->port - 1], &ah_attr); + + if (ib_find_cached_pkey(&dev->ib_dev, ctx->port, IB_DEFAULT_PKEY_FULL, &wc.pkey_index)) + return -EINVAL; + wc.sl = 0; + wc.dlid_path_bits = 0; + wc.port_num = ctx->port; + wc.slid = ah_attr.dlid; /* opensm lid */ + wc.src_qp = 1; + return mlx4_ib_send_to_slave(dev, slave, ctx->port, IB_QPT_GSI, &wc, NULL, mad); +} + +static int send_join_to_wire(struct mcast_group *group, struct ib_sa_mad *sa_mad) +{ + struct ib_sa_mad mad; + struct ib_sa_mcmember_data *sa_mad_data = (struct ib_sa_mcmember_data *)&mad.data; + int ret; + + /* we rely on a mad request as arrived from a VF */ + memcpy(&mad, sa_mad, sizeof mad); + + /* fix port GID to be the real one (slave 0) */ + sa_mad_data->port_gid.global.interface_id = group->demux->guid_cache[0]; + + /* assign our own TID */ + mad.mad_hdr.tid = mlx4_ib_get_new_demux_tid(group->demux); + group->last_req_tid = mad.mad_hdr.tid; /* keep it for later validation */ + + ret = send_mad_to_wire(group->demux, (struct ib_mad *)&mad); + /* set timeout handler */ + if (!ret) { + /* calls mlx4_ib_mcg_timeout_handler */ + queue_delayed_work(group->demux->mcg_wq, &group->timeout_work, + msecs_to_jiffies(MAD_TIMEOUT_MS)); + } + + return ret; +} + +static int send_leave_to_wire(struct mcast_group *group, u8 join_state) +{ + struct ib_sa_mad mad; + struct ib_sa_mcmember_data *sa_data = (struct ib_sa_mcmember_data *)&mad.data; + int ret; + + memset(&mad, 0, sizeof mad); + mad.mad_hdr.base_version = 1; + mad.mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM; + mad.mad_hdr.class_version = 2; + mad.mad_hdr.method = IB_SA_METHOD_DELETE; + mad.mad_hdr.status = cpu_to_be16(0); + mad.mad_hdr.class_specific = cpu_to_be16(0); + mad.mad_hdr.tid = mlx4_ib_get_new_demux_tid(group->demux); + group->last_req_tid = mad.mad_hdr.tid; /* keep it for later validation */ + mad.mad_hdr.attr_id = cpu_to_be16(IB_SA_ATTR_MC_MEMBER_REC); + mad.mad_hdr.attr_mod = cpu_to_be32(0); + mad.sa_hdr.sm_key = 0x0; + mad.sa_hdr.attr_offset = cpu_to_be16(7); + mad.sa_hdr.comp_mask = IB_SA_MCMEMBER_REC_MGID | + IB_SA_MCMEMBER_REC_PORT_GID | IB_SA_MCMEMBER_REC_JOIN_STATE; + + *sa_data = group->rec; + sa_data->scope_join_state = join_state; + + ret = send_mad_to_wire(group->demux, (struct ib_mad *)&mad); + if (ret) + group->state = MCAST_IDLE; + + /* set timeout handler */ + if (!ret) { + /* calls mlx4_ib_mcg_timeout_handler */ + queue_delayed_work(group->demux->mcg_wq, &group->timeout_work, + msecs_to_jiffies(MAD_TIMEOUT_MS)); + } + + return ret; +} + +static int send_reply_to_slave(int slave, struct mcast_group *group, + struct ib_sa_mad *req_sa_mad, u16 status) +{ + struct ib_sa_mad mad; + struct ib_sa_mcmember_data *sa_data = (struct ib_sa_mcmember_data *)&mad.data; + struct ib_sa_mcmember_data *req_sa_data = (struct ib_sa_mcmember_data *)&req_sa_mad->data; + int ret; + + memset(&mad, 0, sizeof mad); + mad.mad_hdr.base_version = 1; + mad.mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM; + mad.mad_hdr.class_version = 2; + mad.mad_hdr.method = IB_MGMT_METHOD_GET_RESP; + mad.mad_hdr.status = cpu_to_be16(status); + mad.mad_hdr.class_specific = cpu_to_be16(0); + mad.mad_hdr.tid = req_sa_mad->mad_hdr.tid; + *(u8 *)&mad.mad_hdr.tid = 0; /* resetting tid to 0 */ + mad.mad_hdr.attr_id = cpu_to_be16(IB_SA_ATTR_MC_MEMBER_REC); + mad.mad_hdr.attr_mod = cpu_to_be32(0); + mad.sa_hdr.sm_key = req_sa_mad->sa_hdr.sm_key; + mad.sa_hdr.attr_offset = cpu_to_be16(7); + mad.sa_hdr.comp_mask = 0; /* ignored on responses, see IBTA spec */ + + *sa_data = group->rec; + + /* reconstruct VF's requested join_state and port_gid */ + sa_data->scope_join_state &= 0xf0; + sa_data->scope_join_state |= (group->func[slave].join_state & 0x0f); + memcpy(&sa_data->port_gid, &req_sa_data->port_gid, sizeof req_sa_data->port_gid); + + ret = send_mad_to_slave(slave, group->demux, (struct ib_mad *)&mad); + return ret; +} + +static int check_selector(ib_sa_comp_mask comp_mask, + ib_sa_comp_mask selector_mask, + ib_sa_comp_mask value_mask, + u8 src_value, u8 dst_value) +{ + int err; + u8 selector = dst_value >> 6; + dst_value &= 0x3f; + src_value &= 0x3f; + + if (!(comp_mask & selector_mask) || !(comp_mask & value_mask)) + return 0; + + switch (selector) { + case IB_SA_GT: + err = (src_value <= dst_value); + break; + case IB_SA_LT: + err = (src_value >= dst_value); + break; + case IB_SA_EQ: + err = (src_value != dst_value); + break; + default: + err = 0; + break; + } + + return err; +} + +static u16 cmp_rec(struct ib_sa_mcmember_data *src, + struct ib_sa_mcmember_data *dst, ib_sa_comp_mask comp_mask) +{ + /* src is group record, dst is request record */ + /* MGID must already match */ + /* Port_GID we always replace to our Port_GID, so it is a match */ + +#define MAD_STATUS_REQ_INVALID 0x0200 + if (comp_mask & IB_SA_MCMEMBER_REC_QKEY && src->qkey != dst->qkey) + return MAD_STATUS_REQ_INVALID; + if (comp_mask & IB_SA_MCMEMBER_REC_MLID && src->mlid != dst->mlid) + return MAD_STATUS_REQ_INVALID; + if (check_selector(comp_mask, IB_SA_MCMEMBER_REC_MTU_SELECTOR, + IB_SA_MCMEMBER_REC_MTU, + src->mtusel_mtu, dst->mtusel_mtu)) + return MAD_STATUS_REQ_INVALID; + if (comp_mask & IB_SA_MCMEMBER_REC_TRAFFIC_CLASS && + src->tclass != dst->tclass) + return MAD_STATUS_REQ_INVALID; + if (comp_mask & IB_SA_MCMEMBER_REC_PKEY && src->pkey != dst->pkey) + return MAD_STATUS_REQ_INVALID; + if (check_selector(comp_mask, IB_SA_MCMEMBER_REC_RATE_SELECTOR, + IB_SA_MCMEMBER_REC_RATE, + src->ratesel_rate, dst->ratesel_rate)) + return MAD_STATUS_REQ_INVALID; + if (check_selector(comp_mask, + IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME_SELECTOR, + IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME, + src->lifetmsel_lifetm, dst->lifetmsel_lifetm)) + return MAD_STATUS_REQ_INVALID; + if (comp_mask & IB_SA_MCMEMBER_REC_SL && + (be32_to_cpu(src->sl_flowlabel_hoplimit) & 0xf0000000) != + (be32_to_cpu(dst->sl_flowlabel_hoplimit) & 0xf0000000)) + return MAD_STATUS_REQ_INVALID; + if (comp_mask & IB_SA_MCMEMBER_REC_FLOW_LABEL && + (be32_to_cpu(src->sl_flowlabel_hoplimit) & 0x0fffff00) != + (be32_to_cpu(dst->sl_flowlabel_hoplimit) & 0x0fffff00)) + return MAD_STATUS_REQ_INVALID; + if (comp_mask & IB_SA_MCMEMBER_REC_HOP_LIMIT && + (be32_to_cpu(src->sl_flowlabel_hoplimit) & 0x000000ff) != + (be32_to_cpu(dst->sl_flowlabel_hoplimit) & 0x000000ff)) + return MAD_STATUS_REQ_INVALID; + if (comp_mask & IB_SA_MCMEMBER_REC_SCOPE && + (src->scope_join_state & 0xf0) != + (dst->scope_join_state & 0xf0)) + return MAD_STATUS_REQ_INVALID; + + /* join_state checked separately, proxy_join ignored */ + + return 0; +} + +/* release group, return 1 if this was last release and group is destroyed + * timout work is canceled sync */ +static int release_group(struct mcast_group *group, int from_timeout_handler) +{ + struct mlx4_ib_demux_ctx *ctx = group->demux; + int nzgroup; + + mutex_lock(&ctx->mcg_table_lock); + mutex_lock(&group->lock); + if (atomic_dec_and_test(&group->refcount)) { + if (!from_timeout_handler) { + if (group->state != MCAST_IDLE && + !cancel_delayed_work(&group->timeout_work)) { + atomic_inc(&group->refcount); + mutex_unlock(&group->lock); + mutex_unlock(&ctx->mcg_table_lock); + return 0; + } + } + + nzgroup = memcmp(&group->rec.mgid, &mgid0, sizeof mgid0); + if (nzgroup) + del_sysfs_port_mcg_attr(ctx->dev, ctx->port, &group->dentry.attr); + if (!list_empty(&group->pending_list)) + mcg_warn_group(group, "releasing a group with non empty pending list\n"); + if (nzgroup) + rb_erase(&group->node, &ctx->mcg_table); + list_del_init(&group->mgid0_list); + mutex_unlock(&group->lock); + mutex_unlock(&ctx->mcg_table_lock); + kfree(group); + return 1; + } else { + mutex_unlock(&group->lock); + mutex_unlock(&ctx->mcg_table_lock); + } + return 0; +} + +static void adjust_membership(struct mcast_group *group, u8 join_state, int inc) +{ + int i; + + for (i = 0; i < 3; i++, join_state >>= 1) + if (join_state & 0x1) + group->members[i] += inc; +} + +static u8 get_leave_state(struct mcast_group *group) +{ + u8 leave_state = 0; + int i; + + for (i = 0; i < 3; i++) + if (!group->members[i]) + leave_state |= (1 << i); + + return leave_state & (group->rec.scope_join_state & 7); +} + +static int join_group(struct mcast_group *group, int slave, u8 join_mask) +{ + int ret = 0; + u8 join_state; + + /* remove bits that slave is already member of, and adjust */ + join_state = join_mask & (~group->func[slave].join_state); + adjust_membership(group, join_state, 1); + group->func[slave].join_state |= join_state; + if (group->func[slave].state != MCAST_MEMBER && join_state) { + group->func[slave].state = MCAST_MEMBER; + ret = 1; + } + return ret; +} + +static int leave_group(struct mcast_group *group, int slave, u8 leave_state) +{ + int ret = 0; + + adjust_membership(group, leave_state, -1); + group->func[slave].join_state &= ~leave_state; + if (!group->func[slave].join_state) { + group->func[slave].state = MCAST_NOT_MEMBER; + ret = 1; + } + return ret; +} + +static int check_leave(struct mcast_group *group, int slave, u8 leave_mask) +{ + if (group->func[slave].state != MCAST_MEMBER) + return MAD_STATUS_REQ_INVALID; + + /* make sure we're not deleting unset bits */ + if (~group->func[slave].join_state & leave_mask) + return MAD_STATUS_REQ_INVALID; + + if (!leave_mask) + return MAD_STATUS_REQ_INVALID; + + return 0; +} + +static void mlx4_ib_mcg_timeout_handler(struct work_struct *work) +{ + struct delayed_work *delay = to_delayed_work(work); + struct mcast_group *group; + struct mcast_req *req = NULL; + + group = container_of(delay, typeof(*group), timeout_work); + + mutex_lock(&group->lock); + if (group->state == MCAST_JOIN_SENT) { + if (!list_empty(&group->pending_list)) { + req = list_first_entry(&group->pending_list, struct mcast_req, group_list); + list_del(&req->group_list); + list_del(&req->func_list); + --group->func[req->func].num_pend_reqs; + mutex_unlock(&group->lock); + kfree(req); + if (memcmp(&group->rec.mgid, &mgid0, sizeof mgid0)) { + if (release_group(group, 1)) + return; + } else { + kfree(group); + return; + } + mutex_lock(&group->lock); + } else + mcg_warn_group(group, "DRIVER BUG\n"); + } else if (group->state == MCAST_LEAVE_SENT) { + if (group->rec.scope_join_state & 7) + group->rec.scope_join_state &= 0xf8; + group->state = MCAST_IDLE; + mutex_unlock(&group->lock); + if (release_group(group, 1)) + return; + mutex_lock(&group->lock); + } else + mcg_warn_group(group, "invalid state %s\n", get_state_string(group->state)); + group->state = MCAST_IDLE; + atomic_inc(&group->refcount); + if (!queue_work(group->demux->mcg_wq, &group->work)) + safe_atomic_dec(&group->refcount); + + mutex_unlock(&group->lock); +} + +static int handle_leave_req(struct mcast_group *group, u8 leave_mask, + struct mcast_req *req) +{ + u16 status; + + if (req->clean) + leave_mask = group->func[req->func].join_state; + + status = check_leave(group, req->func, leave_mask); + if (!status) + leave_group(group, req->func, leave_mask); + + if (!req->clean) + send_reply_to_slave(req->func, group, &req->sa_mad, status); + --group->func[req->func].num_pend_reqs; + list_del(&req->group_list); + list_del(&req->func_list); + kfree(req); + return 1; +} + +static int handle_join_req(struct mcast_group *group, u8 join_mask, + struct mcast_req *req) +{ + u8 group_join_state = group->rec.scope_join_state & 7; + int ref = 0; + u16 status; + struct ib_sa_mcmember_data *sa_data = (struct ib_sa_mcmember_data *)req->sa_mad.data; + + if (join_mask == (group_join_state & join_mask)) { + /* port's membership need not change */ + status = cmp_rec(&group->rec, sa_data, req->sa_mad.sa_hdr.comp_mask); + if (!status) + join_group(group, req->func, join_mask); + + --group->func[req->func].num_pend_reqs; + send_reply_to_slave(req->func, group, &req->sa_mad, status); + list_del(&req->group_list); + list_del(&req->func_list); + kfree(req); + ++ref; + } else { + /* port's membership needs to be updated */ + group->prev_state = group->state; + if (send_join_to_wire(group, &req->sa_mad)) { + --group->func[req->func].num_pend_reqs; + list_del(&req->group_list); + list_del(&req->func_list); + kfree(req); + ref = 1; + group->state = group->prev_state; + } else + group->state = MCAST_JOIN_SENT; + } + + return ref; +} + +static void mlx4_ib_mcg_work_handler(struct work_struct *work) +{ + struct mcast_group *group; + struct mcast_req *req = NULL; + struct ib_sa_mcmember_data *sa_data; + u8 req_join_state; + int rc = 1; /* release_count - this is for the scheduled work */ + u16 status; + u8 method; + + group = container_of(work, typeof(*group), work); + + mutex_lock(&group->lock); + + /* First, let's see if a response from SM is waiting regarding this group. + * If so, we need to update the group's REC. If this is a bad response, we + * may need to send a bad response to a VF waiting for it. If VF is waiting + * and this is a good response, the VF will be answered later in this func. */ + if (group->state == MCAST_RESP_READY) { + /* cancels mlx4_ib_mcg_timeout_handler */ + cancel_delayed_work(&group->timeout_work); + status = be16_to_cpu(group->response_sa_mad.mad_hdr.status); + method = group->response_sa_mad.mad_hdr.method; + if (group->last_req_tid != group->response_sa_mad.mad_hdr.tid) { + mcg_warn_group(group, "Got MAD response to existing MGID but wrong TID, dropping. Resp TID=%llx, group TID=%llx\n", + (long long)be64_to_cpu( + group->response_sa_mad.mad_hdr.tid), + (long long)be64_to_cpu(group->last_req_tid)); + group->state = group->prev_state; + goto process_requests; + } + if (status) { + if (!list_empty(&group->pending_list)) + req = list_first_entry(&group->pending_list, + struct mcast_req, group_list); + if (method == IB_MGMT_METHOD_GET_RESP) { + if (req) { + send_reply_to_slave(req->func, group, &req->sa_mad, status); + --group->func[req->func].num_pend_reqs; + list_del(&req->group_list); + list_del(&req->func_list); + kfree(req); + ++rc; + } else + mcg_warn_group(group, "no request for failed join\n"); + } else if (method == IB_SA_METHOD_DELETE_RESP && group->demux->flushing) + ++rc; + } else { + u8 resp_join_state; + u8 cur_join_state; + + resp_join_state = ((struct ib_sa_mcmember_data *) + group->response_sa_mad.data)->scope_join_state & 7; + cur_join_state = group->rec.scope_join_state & 7; + + if (method == IB_MGMT_METHOD_GET_RESP) { + /* successful join */ + if (!cur_join_state && resp_join_state) + --rc; + } else if (!resp_join_state) + ++rc; + memcpy(&group->rec, group->response_sa_mad.data, sizeof group->rec); + } + group->state = MCAST_IDLE; + } + +process_requests: + /* We should now go over pending join/leave requests, as long as we are idle. */ + while (!list_empty(&group->pending_list) && group->state == MCAST_IDLE) { + req = list_first_entry(&group->pending_list, struct mcast_req, + group_list); + sa_data = (struct ib_sa_mcmember_data *)req->sa_mad.data; + req_join_state = sa_data->scope_join_state & 0x7; + + /* For a leave request, we will immediately answer the VF, and + * update our internal counters. The actual leave will be sent + * to SM later, if at all needed. We dequeue the request now. */ + if (req->sa_mad.mad_hdr.method == IB_SA_METHOD_DELETE) + rc += handle_leave_req(group, req_join_state, req); + else + rc += handle_join_req(group, req_join_state, req); + } + + /* Handle leaves */ + if (group->state == MCAST_IDLE) { + req_join_state = get_leave_state(group); + if (req_join_state) { + group->rec.scope_join_state &= ~req_join_state; + group->prev_state = group->state; + if (send_leave_to_wire(group, req_join_state)) { + group->state = group->prev_state; + ++rc; + } else + group->state = MCAST_LEAVE_SENT; + } + } + + if (!list_empty(&group->pending_list) && group->state == MCAST_IDLE) + goto process_requests; + mutex_unlock(&group->lock); + + while (rc--) + release_group(group, 0); +} + +static struct mcast_group *search_relocate_mgid0_group(struct mlx4_ib_demux_ctx *ctx, + __be64 tid, + union ib_gid *new_mgid) +{ + struct mcast_group *group = NULL, *cur_group; + struct mcast_req *req; + struct list_head *pos; + struct list_head *n; + + mutex_lock(&ctx->mcg_table_lock); + list_for_each_safe(pos, n, &ctx->mcg_mgid0_list) { + group = list_entry(pos, struct mcast_group, mgid0_list); + mutex_lock(&group->lock); + if (group->last_req_tid == tid) { + if (memcmp(new_mgid, &mgid0, sizeof mgid0)) { + group->rec.mgid = *new_mgid; + sprintf(group->name, "%016llx%016llx", + (long long)be64_to_cpu(group->rec.mgid.global.subnet_prefix), + (long long)be64_to_cpu(group->rec.mgid.global.interface_id)); + list_del_init(&group->mgid0_list); + cur_group = mcast_insert(ctx, group); + if (cur_group) { + /* A race between our code and SM. Silently cleaning the new one */ + req = list_first_entry(&group->pending_list, + struct mcast_req, group_list); + --group->func[req->func].num_pend_reqs; + list_del(&req->group_list); + list_del(&req->func_list); + kfree(req); + mutex_unlock(&group->lock); + mutex_unlock(&ctx->mcg_table_lock); + release_group(group, 0); + return NULL; + } + + atomic_inc(&group->refcount); + add_sysfs_port_mcg_attr(ctx->dev, ctx->port, &group->dentry.attr); + mutex_unlock(&group->lock); + mutex_unlock(&ctx->mcg_table_lock); + return group; + } else { + struct mcast_req *tmp1, *tmp2; + + list_del(&group->mgid0_list); + if (!list_empty(&group->pending_list) && group->state != MCAST_IDLE) + cancel_delayed_work_sync(&group->timeout_work); + + list_for_each_entry_safe(tmp1, tmp2, &group->pending_list, group_list) { + list_del(&tmp1->group_list); + kfree(tmp1); + } + mutex_unlock(&group->lock); + mutex_unlock(&ctx->mcg_table_lock); + kfree(group); + return NULL; + } + } + mutex_unlock(&group->lock); + } + mutex_unlock(&ctx->mcg_table_lock); + + return NULL; +} + +static ssize_t sysfs_show_group(struct device *dev, + struct device_attribute *attr, char *buf); + +static struct mcast_group *acquire_group(struct mlx4_ib_demux_ctx *ctx, + union ib_gid *mgid, int create, + gfp_t gfp_mask) +{ + struct mcast_group *group, *cur_group; + int is_mgid0; + int i; + + is_mgid0 = !memcmp(&mgid0, mgid, sizeof mgid0); + if (!is_mgid0) { + group = mcast_find(ctx, mgid); + if (group) + goto found; + } + + if (!create) + return ERR_PTR(-ENOENT); + + group = kzalloc(sizeof *group, gfp_mask); + if (!group) + return ERR_PTR(-ENOMEM); + + group->demux = ctx; + group->rec.mgid = *mgid; + INIT_LIST_HEAD(&group->pending_list); + INIT_LIST_HEAD(&group->mgid0_list); + for (i = 0; i < MAX_VFS; ++i) + INIT_LIST_HEAD(&group->func[i].pending); + INIT_WORK(&group->work, mlx4_ib_mcg_work_handler); + INIT_DELAYED_WORK(&group->timeout_work, mlx4_ib_mcg_timeout_handler); + mutex_init(&group->lock); + sprintf(group->name, "%016llx%016llx", + (long long)be64_to_cpu( + group->rec.mgid.global.subnet_prefix), + (long long)be64_to_cpu( + group->rec.mgid.global.interface_id)); + sysfs_attr_init(&group->dentry.attr); + group->dentry.show = sysfs_show_group; + group->dentry.store = NULL; + group->dentry.attr.name = group->name; + group->dentry.attr.mode = 0400; + group->state = MCAST_IDLE; + + if (is_mgid0) { + list_add(&group->mgid0_list, &ctx->mcg_mgid0_list); + goto found; + } + + cur_group = mcast_insert(ctx, group); + if (cur_group) { + mcg_warn("group just showed up %s - confused\n", cur_group->name); + kfree(group); + return ERR_PTR(-EINVAL); + } + + add_sysfs_port_mcg_attr(ctx->dev, ctx->port, &group->dentry.attr); + +found: + atomic_inc(&group->refcount); + return group; +} + +static void queue_req(struct mcast_req *req) +{ + struct mcast_group *group = req->group; + + atomic_inc(&group->refcount); /* for the request */ + atomic_inc(&group->refcount); /* for scheduling the work */ + list_add_tail(&req->group_list, &group->pending_list); + list_add_tail(&req->func_list, &group->func[req->func].pending); + /* calls mlx4_ib_mcg_work_handler */ + if (!queue_work(group->demux->mcg_wq, &group->work)) + safe_atomic_dec(&group->refcount); +} + +int mlx4_ib_mcg_demux_handler(struct ib_device *ibdev, int port, int slave, + struct ib_sa_mad *mad) +{ + struct mlx4_ib_dev *dev = to_mdev(ibdev); + struct ib_sa_mcmember_data *rec = (struct ib_sa_mcmember_data *)mad->data; + struct mlx4_ib_demux_ctx *ctx = &dev->sriov.demux[port - 1]; + struct mcast_group *group; + + switch (mad->mad_hdr.method) { + case IB_MGMT_METHOD_GET_RESP: + case IB_SA_METHOD_DELETE_RESP: + mutex_lock(&ctx->mcg_table_lock); + group = acquire_group(ctx, &rec->mgid, 0, GFP_KERNEL); + mutex_unlock(&ctx->mcg_table_lock); + if (IS_ERR(group)) { + if (mad->mad_hdr.method == IB_MGMT_METHOD_GET_RESP) { + __be64 tid = mad->mad_hdr.tid; + *(u8 *)(&tid) = (u8)slave; /* in group we kept the modified TID */ + group = search_relocate_mgid0_group(ctx, tid, &rec->mgid); + } else + group = NULL; + } + + if (!group) + return 1; + + mutex_lock(&group->lock); + group->response_sa_mad = *mad; + group->prev_state = group->state; + group->state = MCAST_RESP_READY; + /* calls mlx4_ib_mcg_work_handler */ + atomic_inc(&group->refcount); + if (!queue_work(ctx->mcg_wq, &group->work)) + safe_atomic_dec(&group->refcount); + mutex_unlock(&group->lock); + release_group(group, 0); + return 1; /* consumed */ + case IB_MGMT_METHOD_SET: + case IB_SA_METHOD_GET_TABLE: + case IB_SA_METHOD_GET_TABLE_RESP: + case IB_SA_METHOD_DELETE: + return 0; /* not consumed, pass-through to guest over tunnel */ + default: + mcg_warn("In demux, port %d: unexpected MCMember method: 0x%x, dropping\n", + port, mad->mad_hdr.method); + return 1; /* consumed */ + } +} + +int mlx4_ib_mcg_multiplex_handler(struct ib_device *ibdev, int port, + int slave, struct ib_sa_mad *sa_mad) +{ + struct mlx4_ib_dev *dev = to_mdev(ibdev); + struct ib_sa_mcmember_data *rec = (struct ib_sa_mcmember_data *)sa_mad->data; + struct mlx4_ib_demux_ctx *ctx = &dev->sriov.demux[port - 1]; + struct mcast_group *group; + struct mcast_req *req; + int may_create = 0; + + if (ctx->flushing) + return -EAGAIN; + + switch (sa_mad->mad_hdr.method) { + case IB_MGMT_METHOD_SET: + may_create = 1; + case IB_SA_METHOD_DELETE: + req = kzalloc(sizeof *req, GFP_KERNEL); + if (!req) + return -ENOMEM; + + req->func = slave; + req->sa_mad = *sa_mad; + + mutex_lock(&ctx->mcg_table_lock); + group = acquire_group(ctx, &rec->mgid, may_create, GFP_KERNEL); + mutex_unlock(&ctx->mcg_table_lock); + if (IS_ERR(group)) { + kfree(req); + return PTR_ERR(group); + } + mutex_lock(&group->lock); + if (group->func[slave].num_pend_reqs > MAX_PEND_REQS_PER_FUNC) { + mutex_unlock(&group->lock); + mcg_warn_group(group, "Port %d, Func %d has too many pending requests (%d), dropping\n", + port, slave, MAX_PEND_REQS_PER_FUNC); + release_group(group, 0); + kfree(req); + return -ENOMEM; + } + ++group->func[slave].num_pend_reqs; + req->group = group; + queue_req(req); + mutex_unlock(&group->lock); + release_group(group, 0); + return 1; /* consumed */ + case IB_SA_METHOD_GET_TABLE: + case IB_MGMT_METHOD_GET_RESP: + case IB_SA_METHOD_GET_TABLE_RESP: + case IB_SA_METHOD_DELETE_RESP: + return 0; /* not consumed, pass-through */ + default: + mcg_warn("In multiplex, port %d, func %d: unexpected MCMember method: 0x%x, dropping\n", + port, slave, sa_mad->mad_hdr.method); + return 1; /* consumed */ + } +} + +static ssize_t sysfs_show_group(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct mcast_group *group = + container_of(attr, struct mcast_group, dentry); + struct mcast_req *req = NULL; + char pending_str[40]; + char state_str[40]; + ssize_t len = 0; + int f; + + if (group->state == MCAST_IDLE) + sprintf(state_str, "%s", get_state_string(group->state)); + else + sprintf(state_str, "%s(TID=0x%llx)", + get_state_string(group->state), + (long long)be64_to_cpu(group->last_req_tid)); + if (list_empty(&group->pending_list)) { + sprintf(pending_str, "No"); + } else { + req = list_first_entry(&group->pending_list, struct mcast_req, group_list); + sprintf(pending_str, "Yes(TID=0x%llx)", + (long long)be64_to_cpu( + req->sa_mad.mad_hdr.tid)); + } + len += sprintf(buf + len, "%1d [%02d,%02d,%02d] %4d %4s %5s ", + group->rec.scope_join_state & 0xf, + group->members[2], group->members[1], group->members[0], + atomic_read(&group->refcount), + pending_str, + state_str); + for (f = 0; f < MAX_VFS; ++f) + if (group->func[f].state == MCAST_MEMBER) + len += sprintf(buf + len, "%d[%1x] ", + f, group->func[f].join_state); + + len += sprintf(buf + len, "\t\t(%4hx %4x %2x %2x %2x %2x %2x " + "%4x %4x %2x %2x)\n", + be16_to_cpu(group->rec.pkey), + be32_to_cpu(group->rec.qkey), + (group->rec.mtusel_mtu & 0xc0) >> 6, + group->rec.mtusel_mtu & 0x3f, + group->rec.tclass, + (group->rec.ratesel_rate & 0xc0) >> 6, + group->rec.ratesel_rate & 0x3f, + (be32_to_cpu(group->rec.sl_flowlabel_hoplimit) & 0xf0000000) >> 28, + (be32_to_cpu(group->rec.sl_flowlabel_hoplimit) & 0x0fffff00) >> 8, + be32_to_cpu(group->rec.sl_flowlabel_hoplimit) & 0x000000ff, + group->rec.proxy_join); + + return len; +} + +int mlx4_ib_mcg_port_init(struct mlx4_ib_demux_ctx *ctx) +{ + char name[20]; + + atomic_set(&ctx->tid, 0); + sprintf(name, "mlx4_ib_mcg%d", ctx->port); + ctx->mcg_wq = create_singlethread_workqueue(name); + if (!ctx->mcg_wq) + return -ENOMEM; + + mutex_init(&ctx->mcg_table_lock); + ctx->mcg_table = RB_ROOT; + INIT_LIST_HEAD(&ctx->mcg_mgid0_list); + ctx->flushing = 0; + + return 0; +} + +static void force_clean_group(struct mcast_group *group) +{ + struct mcast_req *req, *tmp + ; + list_for_each_entry_safe(req, tmp, &group->pending_list, group_list) { + list_del(&req->group_list); + kfree(req); + } + del_sysfs_port_mcg_attr(group->demux->dev, group->demux->port, &group->dentry.attr); + rb_erase(&group->node, &group->demux->mcg_table); + kfree(group); +} + +static void _mlx4_ib_mcg_port_cleanup(struct mlx4_ib_demux_ctx *ctx, int destroy_wq) +{ + int i; + struct rb_node *p; + struct mcast_group *group; + unsigned long end; + int count; + + for (i = 0; i < MAX_VFS; ++i) + clean_vf_mcast(ctx, i); + + end = jiffies + msecs_to_jiffies(MAD_TIMEOUT_MS + 3000); + do { + count = 0; + mutex_lock(&ctx->mcg_table_lock); + for (p = rb_first(&ctx->mcg_table); p; p = rb_next(p)) + ++count; + mutex_unlock(&ctx->mcg_table_lock); + if (!count) + break; + + msleep(1); + } while (time_after(end, jiffies)); + + flush_workqueue(ctx->mcg_wq); + if (destroy_wq) + destroy_workqueue(ctx->mcg_wq); + + mutex_lock(&ctx->mcg_table_lock); + while ((p = rb_first(&ctx->mcg_table)) != NULL) { + group = rb_entry(p, struct mcast_group, node); + if (atomic_read(&group->refcount)) + mcg_warn_group(group, "group refcount %d!!! (pointer %p)\n", atomic_read(&group->refcount), group); + + force_clean_group(group); + } + mutex_unlock(&ctx->mcg_table_lock); +} + +struct clean_work { + struct work_struct work; + struct mlx4_ib_demux_ctx *ctx; + int destroy_wq; +}; + +static void mcg_clean_task(struct work_struct *work) +{ + struct clean_work *cw = container_of(work, struct clean_work, work); + + _mlx4_ib_mcg_port_cleanup(cw->ctx, cw->destroy_wq); + cw->ctx->flushing = 0; + kfree(cw); +} + +void mlx4_ib_mcg_port_cleanup(struct mlx4_ib_demux_ctx *ctx, int destroy_wq) +{ + struct clean_work *work; + + if (ctx->flushing) + return; + + ctx->flushing = 1; + + if (destroy_wq) { + _mlx4_ib_mcg_port_cleanup(ctx, destroy_wq); + ctx->flushing = 0; + return; + } + + work = kmalloc(sizeof *work, GFP_KERNEL); + if (!work) { + ctx->flushing = 0; + mcg_warn("failed allocating work for cleanup\n"); + return; + } + + work->ctx = ctx; + work->destroy_wq = destroy_wq; + INIT_WORK(&work->work, mcg_clean_task); + queue_work(clean_wq, &work->work); +} + +static void build_leave_mad(struct mcast_req *req) +{ + struct ib_sa_mad *mad = &req->sa_mad; + + mad->mad_hdr.method = IB_SA_METHOD_DELETE; +} + + +static void clear_pending_reqs(struct mcast_group *group, int vf) +{ + struct mcast_req *req, *tmp, *group_first = NULL; + int clear; + int pend = 0; + + if (!list_empty(&group->pending_list)) + group_first = list_first_entry(&group->pending_list, struct mcast_req, group_list); + + list_for_each_entry_safe(req, tmp, &group->func[vf].pending, func_list) { + clear = 1; + if (group_first == req && + (group->state == MCAST_JOIN_SENT || + group->state == MCAST_LEAVE_SENT)) { + clear = cancel_delayed_work(&group->timeout_work); + pend = !clear; + group->state = MCAST_IDLE; + } + if (clear) { + --group->func[vf].num_pend_reqs; + list_del(&req->group_list); + list_del(&req->func_list); + kfree(req); + atomic_dec(&group->refcount); + } + } + + if (!pend && (!list_empty(&group->func[vf].pending) || group->func[vf].num_pend_reqs)) { + mcg_warn_group(group, "DRIVER BUG: list_empty %d, num_pend_reqs %d\n", + list_empty(&group->func[vf].pending), group->func[vf].num_pend_reqs); + } +} + +static int push_deleteing_req(struct mcast_group *group, int slave) +{ + struct mcast_req *req; + struct mcast_req *pend_req; + + if (!group->func[slave].join_state) + return 0; + + req = kzalloc(sizeof *req, GFP_KERNEL); + if (!req) { + mcg_warn_group(group, "failed allocation - may leave stall groups\n"); + return -ENOMEM; + } + + if (!list_empty(&group->func[slave].pending)) { + pend_req = list_entry(group->func[slave].pending.prev, struct mcast_req, group_list); + if (pend_req->clean) { + kfree(req); + return 0; + } + } + + req->clean = 1; + req->func = slave; + req->group = group; + ++group->func[slave].num_pend_reqs; + build_leave_mad(req); + queue_req(req); + return 0; +} + +void clean_vf_mcast(struct mlx4_ib_demux_ctx *ctx, int slave) +{ + struct mcast_group *group; + struct rb_node *p; + + mutex_lock(&ctx->mcg_table_lock); + for (p = rb_first(&ctx->mcg_table); p; p = rb_next(p)) { + group = rb_entry(p, struct mcast_group, node); + mutex_lock(&group->lock); + if (atomic_read(&group->refcount)) { + /* clear pending requests of this VF */ + clear_pending_reqs(group, slave); + push_deleteing_req(group, slave); + } + mutex_unlock(&group->lock); + } + mutex_unlock(&ctx->mcg_table_lock); +} + + +int mlx4_ib_mcg_init(void) +{ + clean_wq = create_singlethread_workqueue("mlx4_ib_mcg"); + if (!clean_wq) + return -ENOMEM; + + return 0; +} + +void mlx4_ib_mcg_destroy(void) +{ + destroy_workqueue(clean_wq); +} Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_mcg.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_wc.c =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_wc.c (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_wc.c (revision 329159) @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2006-2007 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include "wc.h" + +#if defined(__i386__) || defined(__x86_64__) + +pgprot_t pgprot_wc(pgprot_t _prot) +{ + return pgprot_writecombine(_prot); +} + +int mlx4_wc_enabled(void) +{ + return 1; +} + +#elif defined(CONFIG_PPC64) + +pgprot_t pgprot_wc(pgprot_t _prot) +{ + return __pgprot((pgprot_val(_prot) | _PAGE_NO_CACHE) & + ~(pgprot_t)_PAGE_GUARDED); +} + +int mlx4_wc_enabled(void) +{ + return 1; +} + +#else /* !(defined(__i386__) || defined(__x86_64__)) */ + +pgprot_t pgprot_wc(pgprot_t _prot) +{ + return pgprot_noncached(_prot); +} + +int mlx4_wc_enabled(void) +{ + return 0; +} + +#endif Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/mlx4_ib_wc.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/user.h =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/user.h (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/user.h (revision 329159) @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MLX4_IB_USER_H +#define MLX4_IB_USER_H + +#include + +/* + * Increment this value if any changes that break userspace ABI + * compatibility are made. + */ + +#define MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION 3 +#define MLX4_IB_UVERBS_ABI_VERSION 4 + +/* + * Make sure that all structs defined in this file remain laid out so + * that they pack the same way on 32-bit and 64-bit architectures (to + * avoid incompatibility between 32-bit userspace and 64-bit kernels). + * In particular do not use pointer types -- pass pointers in __u64 + * instead. + */ + +struct mlx4_ib_alloc_ucontext_resp_v3 { + __u32 qp_tab_size; + __u16 bf_reg_size; + __u16 bf_regs_per_page; +}; + +struct mlx4_ib_alloc_ucontext_resp { + __u32 dev_caps; + __u32 qp_tab_size; + __u16 bf_reg_size; + __u16 bf_regs_per_page; + __u32 cqe_size; +}; + +struct mlx4_ib_alloc_pd_resp { + __u32 pdn; + __u32 reserved; +}; + +struct mlx4_ib_create_cq { + __u64 buf_addr; + __u64 db_addr; +}; + +struct mlx4_ib_create_cq_resp { + __u32 cqn; + __u32 reserved; +}; + +struct mlx4_ib_resize_cq { + __u64 buf_addr; +}; + +struct mlx4_ib_create_srq { + __u64 buf_addr; + __u64 db_addr; +}; + +struct mlx4_ib_create_srq_resp { + __u32 srqn; + __u32 reserved; +}; + +struct mlx4_ib_create_qp { + __u64 buf_addr; + __u64 db_addr; + __u8 log_sq_bb_count; + __u8 log_sq_stride; + __u8 sq_no_prefetch; + __u8 reserved[5]; +}; + +#endif /* MLX4_IB_USER_H */ Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/user.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/mlx4_ib/wc.h =================================================================== --- stable/11/sys/dev/mlx4/mlx4_ib/wc.h (nonexistent) +++ stable/11/sys/dev/mlx4/mlx4_ib/wc.h (revision 329159) @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2006-2007 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef mlx4_WC_H +#define mlx4_WC_H + +#include + +int mlx4_wc_enabled(void); +pgprot_t pgprot_wc(pgprot_t _prot); + +#endif Property changes on: stable/11/sys/dev/mlx4/mlx4_ib/wc.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/qp.h =================================================================== --- stable/11/sys/dev/mlx4/qp.h (nonexistent) +++ stable/11/sys/dev/mlx4/qp.h (revision 329159) @@ -0,0 +1,509 @@ +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MLX4_QP_H +#define MLX4_QP_H + +#include + +#include + +#define MLX4_INVALID_LKEY 0x100 +#define DS_SIZE_ALIGNMENT 16 + +#define SET_BYTE_COUNT(byte_count) cpu_to_be32(byte_count) +#define SET_LSO_MSS(mss_hdr_size) cpu_to_be32(mss_hdr_size) +#define DS_BYTE_COUNT_MASK cpu_to_be32(0x7fffffff) + +enum mlx4_qp_optpar { + MLX4_QP_OPTPAR_ALT_ADDR_PATH = 1 << 0, + MLX4_QP_OPTPAR_RRE = 1 << 1, + MLX4_QP_OPTPAR_RAE = 1 << 2, + MLX4_QP_OPTPAR_RWE = 1 << 3, + MLX4_QP_OPTPAR_PKEY_INDEX = 1 << 4, + MLX4_QP_OPTPAR_Q_KEY = 1 << 5, + MLX4_QP_OPTPAR_RNR_TIMEOUT = 1 << 6, + MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH = 1 << 7, + MLX4_QP_OPTPAR_SRA_MAX = 1 << 8, + MLX4_QP_OPTPAR_RRA_MAX = 1 << 9, + MLX4_QP_OPTPAR_PM_STATE = 1 << 10, + MLX4_QP_OPTPAR_RETRY_COUNT = 1 << 12, + MLX4_QP_OPTPAR_RNR_RETRY = 1 << 13, + MLX4_QP_OPTPAR_ACK_TIMEOUT = 1 << 14, + MLX4_QP_OPTPAR_SCHED_QUEUE = 1 << 16, + MLX4_QP_OPTPAR_COUNTER_INDEX = 1 << 20, + MLX4_QP_OPTPAR_VLAN_STRIPPING = 1 << 21, +}; + +enum mlx4_qp_state { + MLX4_QP_STATE_RST = 0, + MLX4_QP_STATE_INIT = 1, + MLX4_QP_STATE_RTR = 2, + MLX4_QP_STATE_RTS = 3, + MLX4_QP_STATE_SQER = 4, + MLX4_QP_STATE_SQD = 5, + MLX4_QP_STATE_ERR = 6, + MLX4_QP_STATE_SQ_DRAINING = 7, + MLX4_QP_NUM_STATE +}; + +enum { + MLX4_QP_ST_RC = 0x0, + MLX4_QP_ST_UC = 0x1, + MLX4_QP_ST_RD = 0x2, + MLX4_QP_ST_UD = 0x3, + MLX4_QP_ST_XRC = 0x6, + MLX4_QP_ST_MLX = 0x7 +}; + +enum { + MLX4_QP_PM_MIGRATED = 0x3, + MLX4_QP_PM_ARMED = 0x0, + MLX4_QP_PM_REARM = 0x1 +}; + +enum { + /* params1 */ + MLX4_QP_BIT_SRE = 1 << 15, + MLX4_QP_BIT_SWE = 1 << 14, + MLX4_QP_BIT_SAE = 1 << 13, + /* params2 */ + MLX4_QP_BIT_RRE = 1 << 15, + MLX4_QP_BIT_RWE = 1 << 14, + MLX4_QP_BIT_RAE = 1 << 13, + MLX4_QP_BIT_FPP = 1 << 3, + MLX4_QP_BIT_RIC = 1 << 4, +}; + +enum { + MLX4_RSS_HASH_XOR = 0, + MLX4_RSS_HASH_TOP = 1, + + MLX4_RSS_UDP_IPV6 = 1 << 0, + MLX4_RSS_UDP_IPV4 = 1 << 1, + MLX4_RSS_TCP_IPV6 = 1 << 2, + MLX4_RSS_IPV6 = 1 << 3, + MLX4_RSS_TCP_IPV4 = 1 << 4, + MLX4_RSS_IPV4 = 1 << 5, + + MLX4_RSS_BY_OUTER_HEADERS = 0 << 6, + MLX4_RSS_BY_INNER_HEADERS = 2 << 6, + MLX4_RSS_BY_INNER_HEADERS_IPONLY = 3 << 6, + + /* offset of mlx4_rss_context within mlx4_qp_context.pri_path */ + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH = 0x24, + /* offset of being RSS indirection QP within mlx4_qp_context.flags */ + MLX4_RSS_QPC_FLAG_OFFSET = 13, +}; + +#define MLX4_EN_RSS_KEY_SIZE 40 + +struct mlx4_rss_context { + __be32 base_qpn; + __be32 default_qpn; + u16 reserved; + u8 hash_fn; + u8 flags; + __be32 rss_key[MLX4_EN_RSS_KEY_SIZE / sizeof(__be32)]; + __be32 base_qpn_udp; +}; + +struct mlx4_qp_path { + u8 fl; + union { + u8 vlan_control; + u8 control; + }; + u8 disable_pkey_check; + u8 pkey_index; + u8 counter_index; + u8 grh_mylmc; + __be16 rlid; + u8 ackto; + u8 mgid_index; + u8 static_rate; + u8 hop_limit; + __be32 tclass_flowlabel; + u8 rgid[16]; + u8 sched_queue; + u8 vlan_index; + u8 feup; + u8 fvl_rx; + u8 reserved4[2]; + u8 dmac[ETH_ALEN]; +}; + +enum { /* fl */ + MLX4_FL_CV = 1 << 6, + MLX4_FL_SV = 1 << 5, + MLX4_FL_ETH_HIDE_CQE_VLAN = 1 << 2, + MLX4_FL_ETH_SRC_CHECK_MC_LB = 1 << 1, + MLX4_FL_ETH_SRC_CHECK_UC_LB = 1 << 0, +}; + +enum { /* control */ + MLX4_CTRL_ETH_SRC_CHECK_IF_COUNTER = 1 << 7, +}; + +enum { /* vlan_control */ + MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED = 1 << 6, + MLX4_VLAN_CTRL_ETH_TX_BLOCK_PRIO_TAGGED = 1 << 5, /* 802.1p priority tag */ + MLX4_VLAN_CTRL_ETH_TX_BLOCK_UNTAGGED = 1 << 4, + MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED = 1 << 2, + MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED = 1 << 1, /* 802.1p priority tag */ + MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED = 1 << 0 +}; + +enum { /* feup */ + MLX4_FEUP_FORCE_ETH_UP = 1 << 6, /* force Eth UP */ + MLX4_FSM_FORCE_ETH_SRC_MAC = 1 << 5, /* force Source MAC */ + MLX4_FVL_FORCE_ETH_VLAN = 1 << 3 /* force Eth vlan */ +}; + +enum { /* fvl_rx */ + MLX4_FVL_RX_FORCE_ETH_VLAN = 1 << 0 /* enforce Eth rx vlan */ +}; + +struct mlx4_qp_context { + __be32 flags; + __be32 pd; + u8 mtu_msgmax; + u8 rq_size_stride; + u8 sq_size_stride; + u8 rlkey_roce_mode; + __be32 usr_page; + __be32 local_qpn; + __be32 remote_qpn; + struct mlx4_qp_path pri_path; + struct mlx4_qp_path alt_path; + __be32 params1; + u32 reserved1; + __be32 next_send_psn; + __be32 cqn_send; + __be16 roce_entropy; + __be16 reserved2[3]; + __be32 last_acked_psn; + __be32 ssn; + __be32 params2; + __be32 rnr_nextrecvpsn; + __be32 xrcd; + __be32 cqn_recv; + __be64 db_rec_addr; + __be32 qkey; + __be32 srqn; + __be32 msn; + __be16 rq_wqe_counter; + __be16 sq_wqe_counter; + u32 reserved3; + __be16 rate_limit_params; + u8 reserved4; + u8 qos_vport; + __be32 param3; + __be32 nummmcpeers_basemkey; + u8 log_page_size; + u8 reserved5[2]; + u8 mtt_base_addr_h; + __be32 mtt_base_addr_l; + u32 reserved6[10]; +}; + +struct mlx4_update_qp_context { + __be64 qp_mask; + __be64 primary_addr_path_mask; + __be64 secondary_addr_path_mask; + u64 reserved1; + struct mlx4_qp_context qp_context; + u64 reserved2[58]; +}; + +enum { + MLX4_UPD_QP_MASK_PM_STATE = 32, + MLX4_UPD_QP_MASK_VSD = 33, + MLX4_UPD_QP_MASK_QOS_VPP = 34, + MLX4_UPD_QP_MASK_RATE_LIMIT = 35, +}; + +enum { + MLX4_UPD_QP_PATH_MASK_PKEY_INDEX = 0 + 32, + MLX4_UPD_QP_PATH_MASK_FSM = 1 + 32, + MLX4_UPD_QP_PATH_MASK_MAC_INDEX = 2 + 32, + MLX4_UPD_QP_PATH_MASK_FVL = 3 + 32, + MLX4_UPD_QP_PATH_MASK_CV = 4 + 32, + MLX4_UPD_QP_PATH_MASK_VLAN_INDEX = 5 + 32, + MLX4_UPD_QP_PATH_MASK_ETH_HIDE_CQE_VLAN = 6 + 32, + MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_UNTAGGED = 7 + 32, + MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_1P = 8 + 32, + MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_TAGGED = 9 + 32, + MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_UNTAGGED = 10 + 32, + MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_1P = 11 + 32, + MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_TAGGED = 12 + 32, + MLX4_UPD_QP_PATH_MASK_FEUP = 13 + 32, + MLX4_UPD_QP_PATH_MASK_SCHED_QUEUE = 14 + 32, + MLX4_UPD_QP_PATH_MASK_IF_COUNTER_INDEX = 15 + 32, + MLX4_UPD_QP_PATH_MASK_FVL_RX = 16 + 32, + MLX4_UPD_QP_PATH_MASK_ETH_SRC_CHECK_UC_LB = 18 + 32, + MLX4_UPD_QP_PATH_MASK_ETH_SRC_CHECK_MC_LB = 19 + 32, + MLX4_UPD_QP_PATH_MASK_SV = 22 + 32, +}; + +enum { /* param3 */ + MLX4_STRIP_VLAN = 1 << 30 +}; + +/* Which firmware version adds support for NEC (NoErrorCompletion) bit */ +#define MLX4_FW_VER_WQE_CTRL_NEC mlx4_fw_ver(2, 2, 232) + +enum { + MLX4_WQE_CTRL_OWN = 1 << 31, + MLX4_WQE_CTRL_NEC = 1 << 29, + MLX4_WQE_CTRL_RR = 1 << 6, + MLX4_WQE_CTRL_IIP = 1 << 28, + MLX4_WQE_CTRL_ILP = 1 << 27, + MLX4_WQE_CTRL_FENCE = 1 << 6, + MLX4_WQE_CTRL_CQ_UPDATE = 3 << 2, + MLX4_WQE_CTRL_SOLICITED = 1 << 1, + MLX4_WQE_CTRL_IP_CSUM = 1 << 4, + MLX4_WQE_CTRL_TCP_UDP_CSUM = 1 << 5, + MLX4_WQE_CTRL_INS_CVLAN = 1 << 6, + MLX4_WQE_CTRL_INS_SVLAN = 1 << 7, + MLX4_WQE_CTRL_STRONG_ORDER = 1 << 7, + MLX4_WQE_CTRL_FORCE_LOOPBACK = 1 << 0, +}; + +struct mlx4_wqe_ctrl_seg { + __be32 owner_opcode; + union { + struct { + __be16 vlan_tag; + u8 ins_vlan; + u8 fence_size; + }; + __be32 bf_qpn; + }; + /* + * High 24 bits are SRC remote buffer; low 8 bits are flags: + * [7] SO (strong ordering) + * [5] TCP/UDP checksum + * [4] IP checksum + * [3:2] C (generate completion queue entry) + * [1] SE (solicited event) + * [0] FL (force loopback) + */ + union { + __be32 srcrb_flags; + __be16 srcrb_flags16[2]; + }; + /* + * imm is immediate data for send/RDMA write w/ immediate; + * also invalidation key for send with invalidate; input + * modifier for WQEs on CCQs. + */ + __be32 imm; +}; + +enum { + MLX4_WQE_MLX_VL15 = 1 << 17, + MLX4_WQE_MLX_SLR = 1 << 16 +}; + +struct mlx4_wqe_mlx_seg { + u8 owner; + u8 reserved1[2]; + u8 opcode; + __be16 sched_prio; + u8 reserved2; + u8 size; + /* + * [17] VL15 + * [16] SLR + * [15:12] static rate + * [11:8] SL + * [4] ICRC + * [3:2] C + * [0] FL (force loopback) + */ + __be32 flags; + __be16 rlid; + u16 reserved3; +}; + +struct mlx4_wqe_datagram_seg { + __be32 av[8]; + __be32 dqpn; + __be32 qkey; + __be16 vlan; + u8 mac[ETH_ALEN]; +}; + +struct mlx4_wqe_lso_seg { + __be32 mss_hdr_size; + __be32 header[0]; +}; + +enum mlx4_wqe_bind_seg_flags2 { + MLX4_WQE_BIND_ZERO_BASED = (1 << 30), + MLX4_WQE_BIND_TYPE_2 = (1 << 31), +}; + +struct mlx4_wqe_bind_seg { + __be32 flags1; + __be32 flags2; + __be32 new_rkey; + __be32 lkey; + __be64 addr; + __be64 length; +}; + +enum { + MLX4_WQE_FMR_PERM_LOCAL_READ = 1 << 27, + MLX4_WQE_FMR_PERM_LOCAL_WRITE = 1 << 28, + MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_READ = 1 << 29, + MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_WRITE = 1 << 30, + MLX4_WQE_FMR_AND_BIND_PERM_ATOMIC = 1 << 31 +}; + +struct mlx4_wqe_fmr_seg { + __be32 flags; + __be32 mem_key; + __be64 buf_list; + __be64 start_addr; + __be64 reg_len; + __be32 offset; + __be32 page_size; + u32 reserved[2]; +}; + +struct mlx4_wqe_fmr_ext_seg { + u8 flags; + u8 reserved; + __be16 app_mask; + __be16 wire_app_tag; + __be16 mem_app_tag; + __be32 wire_ref_tag_base; + __be32 mem_ref_tag_base; +}; + +struct mlx4_wqe_local_inval_seg { + u64 reserved1; + __be32 mem_key; + u32 reserved2; + u64 reserved3[2]; +}; + +struct mlx4_wqe_raddr_seg { + __be64 raddr; + __be32 rkey; + u32 reserved; +}; + +struct mlx4_wqe_atomic_seg { + __be64 swap_add; + __be64 compare; +}; + +struct mlx4_wqe_masked_atomic_seg { + __be64 swap_add; + __be64 compare; + __be64 swap_add_mask; + __be64 compare_mask; +}; + +struct mlx4_wqe_data_seg { + __be32 byte_count; + __be32 lkey; + __be64 addr; +}; + +enum { + MLX4_INLINE_ALIGN = 64, + MLX4_INLINE_SEG = 1 << 31, +}; + +struct mlx4_wqe_inline_seg { + __be32 byte_count; +}; + +enum mlx4_update_qp_attr { + MLX4_UPDATE_QP_SMAC = 1 << 0, + MLX4_UPDATE_QP_VSD = 1 << 1, + MLX4_UPDATE_QP_RATE_LIMIT = 1 << 2, + MLX4_UPDATE_QP_QOS_VPORT = 1 << 3, + MLX4_UPDATE_QP_ETH_SRC_CHECK_MC_LB = 1 << 4, + MLX4_UPDATE_QP_SUPPORTED_ATTRS = (1 << 5) - 1 +}; + +enum mlx4_update_qp_params_flags { + MLX4_UPDATE_QP_PARAMS_FLAGS_ETH_CHECK_MC_LB = 1 << 0, + MLX4_UPDATE_QP_PARAMS_FLAGS_VSD_ENABLE = 1 << 1, +}; + +struct mlx4_update_qp_params { + u8 smac_index; + u8 qos_vport; + u32 flags; + u16 rate_unit; + u16 rate_val; +}; + +int mlx4_update_qp(struct mlx4_dev *dev, u32 qpn, + enum mlx4_update_qp_attr attr, + struct mlx4_update_qp_params *params); +int mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + enum mlx4_qp_state cur_state, enum mlx4_qp_state new_state, + struct mlx4_qp_context *context, enum mlx4_qp_optpar optpar, + int sqd_event, struct mlx4_qp *qp); + +int mlx4_qp_query(struct mlx4_dev *dev, struct mlx4_qp *qp, + struct mlx4_qp_context *context); + +int mlx4_qp_to_ready(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + struct mlx4_qp_context *context, + struct mlx4_qp *qp, enum mlx4_qp_state *qp_state); + +static inline struct mlx4_qp *__mlx4_qp_lookup(struct mlx4_dev *dev, u32 qpn) +{ + return radix_tree_lookup(&dev->qp_table_tree, qpn & (dev->caps.num_qps - 1)); +} + +void mlx4_qp_remove(struct mlx4_dev *dev, struct mlx4_qp *qp); + +static inline u16 folded_qp(u32 q) +{ + u16 res; + + res = ((q & 0xff) ^ ((q & 0xff0000) >> 16)) | (q & 0xff00); + return res; +} + +u16 mlx4_qp_roce_entropy(struct mlx4_dev *dev, u32 qpn); + +#endif /* MLX4_QP_H */ Property changes on: stable/11/sys/dev/mlx4/qp.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/stats.h =================================================================== --- stable/11/sys/dev/mlx4/stats.h (nonexistent) +++ stable/11/sys/dev/mlx4/stats.h (revision 329159) @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2014 Mellanox Technologies Ltd. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _MLX4_STATS_ +#define _MLX4_STATS_ + +#define NUM_PRIORITIES 9 +#define NUM_PRIORITY_STATS 2 + +struct mlx4_en_pkt_stats { + u64 rx_packets; + u64 rx_bytes; + u64 rx_multicast_packets; + u64 rx_broadcast_packets; + u64 rx_errors; + u64 rx_dropped; + u64 rx_length_errors; + u64 rx_over_errors; + u64 rx_crc_errors; + u64 rx_jabbers; + u64 rx_in_range_length_error; + u64 rx_out_range_length_error; + u64 rx_lt_64_bytes_packets; + u64 rx_127_bytes_packets; + u64 rx_255_bytes_packets; + u64 rx_511_bytes_packets; + u64 rx_1023_bytes_packets; + u64 rx_1518_bytes_packets; + u64 rx_1522_bytes_packets; + u64 rx_1548_bytes_packets; + u64 rx_gt_1548_bytes_packets; + u64 tx_packets; + u64 tx_bytes; + u64 tx_multicast_packets; + u64 tx_broadcast_packets; + u64 tx_errors; + u64 tx_dropped; + u64 tx_lt_64_bytes_packets; + u64 tx_127_bytes_packets; + u64 tx_255_bytes_packets; + u64 tx_511_bytes_packets; + u64 tx_1023_bytes_packets; + u64 tx_1518_bytes_packets; + u64 tx_1522_bytes_packets; + u64 tx_1548_bytes_packets; + u64 tx_gt_1548_bytes_packets; + u64 rx_prio[NUM_PRIORITIES][NUM_PRIORITY_STATS]; + u64 tx_prio[NUM_PRIORITIES][NUM_PRIORITY_STATS]; +}; + +struct mlx4_en_vf_stats { + u64 rx_frames; + u64 rx_bytes; + u64 tx_frames; + u64 tx_bytes; +}; + +struct mlx4_en_vport_stats { + u64 rx_frames; + u64 rx_bytes; + u64 tx_frames; + u64 tx_bytes; +}; + +struct mlx4_en_port_stats { + u64 tso_packets; + u64 queue_stopped; + u64 wake_queue; + u64 tx_timeout; + u64 oversized_packets; + u64 rx_alloc_failed; + u64 rx_chksum_good; + u64 rx_chksum_none; + u64 tx_chksum_offload; + u64 defrag_attempts; +}; + +struct mlx4_en_perf_stats { + u32 tx_poll; + u64 tx_pktsz_avg; + u32 inflight_avg; + u16 tx_coal_avg; + u16 rx_coal_avg; + u32 napi_quota; +}; + +#define MLX4_NUM_PRIORITIES 8 + +struct mlx4_en_flow_stats_rx { + u64 rx_pause; + u64 rx_pause_duration; + u64 rx_pause_transition; +}; + +struct mlx4_en_flow_stats_tx { + u64 tx_pause; + u64 tx_pause_duration; + u64 tx_pause_transition; +}; + +struct mlx4_en_stat_out_flow_control_mbox { + /* Total number of PAUSE frames received from the far-end port */ + __be64 rx_pause; + /* Total number of microseconds that far-end port requested to pause + * transmission of packets + */ + __be64 rx_pause_duration; + /* Number of received transmission from XOFF state to XON state */ + __be64 rx_pause_transition; + /* Total number of PAUSE frames sent from the far-end port */ + __be64 tx_pause; + /* Total time in microseconds that transmission of packets has been + * paused + */ + __be64 tx_pause_duration; + /* Number of transmitter transitions from XOFF state to XON state */ + __be64 tx_pause_transition; + /* Reserverd */ + __be64 reserved[2]; +}; + +enum { + MLX4_DUMP_ETH_STATS_FLOW_CONTROL = 1 << 12 +}; + +int mlx4_get_vport_ethtool_stats(struct mlx4_dev *dev, int port, + struct mlx4_en_vport_stats *vport_stats, + int reset, int *read_counters); + +#endif Property changes on: stable/11/sys/dev/mlx4/stats.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/doorbell.h =================================================================== --- stable/11/sys/dev/mlx4/doorbell.h (nonexistent) +++ stable/11/sys/dev/mlx4/doorbell.h (revision 329159) @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2004 Topspin Communications. All rights reserved. + * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. + * Copyright (c) 2005 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MLX4_DOORBELL_H +#define MLX4_DOORBELL_H + +#include +#include + +#define MLX4_SEND_DOORBELL 0x14 +#define MLX4_CQ_DOORBELL 0x20 + +#if BITS_PER_LONG == 64 +/* + * Assume that we can just write a 64-bit doorbell atomically. s390 + * actually doesn't have writeq() but S/390 systems don't even have + * PCI so we won't worry about it. + */ + +#define MLX4_DECLARE_DOORBELL_LOCK(name) +#define MLX4_INIT_DOORBELL_LOCK(ptr) do { } while (0) +#define MLX4_GET_DOORBELL_LOCK(ptr) (NULL) + +static inline void mlx4_write64(__be32 val[2], void __iomem *dest, + spinlock_t *doorbell_lock) +{ + __raw_writeq(*(u64 *) val, dest); +} + +#else + +/* + * Just fall back to a spinlock to protect the doorbell if + * BITS_PER_LONG is 32 -- there's no portable way to do atomic 64-bit + * MMIO writes. + */ + +#define MLX4_DECLARE_DOORBELL_LOCK(name) spinlock_t name; +#define MLX4_INIT_DOORBELL_LOCK(ptr) spin_lock_init(ptr) +#define MLX4_GET_DOORBELL_LOCK(ptr) (ptr) + +static inline void mlx4_write64(__be32 val[2], void __iomem *dest, + spinlock_t *doorbell_lock) +{ + unsigned long flags; + + spin_lock_irqsave(doorbell_lock, flags); + __raw_writel((__force u32) val[0], dest); + __raw_writel((__force u32) val[1], (u8 *)dest + 4); + spin_unlock_irqrestore(doorbell_lock, flags); +} + +#endif + +#endif /* MLX4_DOORBELL_H */ Property changes on: stable/11/sys/dev/mlx4/doorbell.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/dev/mlx4/srq.h =================================================================== --- stable/11/sys/dev/mlx4/srq.h (nonexistent) +++ stable/11/sys/dev/mlx4/srq.h (revision 329159) @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MLX4_SRQ_H +#define MLX4_SRQ_H + +struct mlx4_wqe_srq_next_seg { + u16 reserved1; + __be16 next_wqe_index; + u32 reserved2[3]; +}; + +struct mlx4_srq *mlx4_srq_lookup(struct mlx4_dev *dev, u32 srqn); + +#endif /* MLX4_SRQ_H */ Property changes on: stable/11/sys/dev/mlx4/srq.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/11/sys/i386/conf/NOTES =================================================================== --- stable/11/sys/i386/conf/NOTES (revision 329158) +++ stable/11/sys/i386/conf/NOTES (revision 329159) @@ -1,1080 +1,1081 @@ # # NOTES -- Lines that can be cut/pasted into kernel and hints configs. # # This file contains machine dependent kernel configuration notes. For # machine independent notes, look in /sys/conf/NOTES. # # $FreeBSD$ # # # We want LINT to cover profiling as well. profile 2 # # Enable the kernel DTrace hooks which are required to load the DTrace # kernel modules. # options KDTRACE_HOOKS # DTrace core # NOTE: introduces CDDL-licensed components into the kernel #device dtrace # DTrace modules #device dtrace_profile #device dtrace_sdt #device dtrace_fbt #device dtrace_systrace #device dtrace_prototype #device dtnfscl #device dtmalloc # Alternatively include all the DTrace modules #device dtraceall ##################################################################### # SMP OPTIONS: # # The apic device enables the use of the I/O APIC for interrupt delivery. # The apic device can be used in both UP and SMP kernels, but is required # for SMP kernels. Thus, the apic device is not strictly an SMP option, # but it is a prerequisite for SMP. # # Notes: # # HTT CPUs should only be used if they are enabled in the BIOS. For # the ACPI case, ACPI only correctly tells us about any HTT CPUs if # they are enabled. However, most HTT systems do not list HTT CPUs # in the MP Table if they are enabled, thus we guess at the HTT CPUs # for the MP Table case. However, we shouldn't try to guess and use # these CPUs if HTT is disabled. Thus, HTT guessing is only enabled # for the MP Table if the user explicitly asks for it via the # MPTABLE_FORCE_HTT option. Do NOT use this option if you have HTT # disabled in your BIOS. # # IPI_PREEMPTION instructs the kernel to preempt threads running on other # CPUS if needed. Relies on the PREEMPTION option # Mandatory: device apic # I/O apic # Optional: options MPTABLE_FORCE_HTT # Enable HTT CPUs with the MP Table options IPI_PREEMPTION # # Watchdog routines. # options MP_WATCHDOG # Debugging options. # options COUNT_XINVLTLB_HITS # Counters for TLB events options COUNT_IPIS # Per-CPU IPI interrupt counters ##################################################################### # CPU OPTIONS # # You must specify at least one CPU (the one you intend to run on); # deleting the specification for CPUs you don't need to use may make # parts of the system run faster. # cpu I486_CPU cpu I586_CPU # aka Pentium(tm) cpu I686_CPU # aka Pentium Pro(tm) # # Options for CPU features. # # CPU_ATHLON_SSE_HACK tries to enable SSE instructions when the BIOS has # forgotten to enable them. # # CPU_BLUELIGHTNING_3X enables triple-clock mode on IBM Blue Lightning # CPU if CPU supports it. The default is double-clock mode on # BlueLightning CPU box. # # CPU_BLUELIGHTNING_FPU_OP_CACHE enables FPU operand cache on IBM # BlueLightning CPU. It works only with Cyrix FPU, and this option # should not be used with Intel FPU. # # CPU_BTB_EN enables branch target buffer on Cyrix 5x86 (NOTE 1). # # CPU_CYRIX_NO_LOCK enables weak locking for the entire address space # of Cyrix 6x86 and 6x86MX CPUs by setting the NO_LOCK bit of CCR1. # Otherwise, the NO_LOCK bit of CCR1 is cleared. (NOTE 3) # # CPU_DIRECT_MAPPED_CACHE sets L1 cache of Cyrix 486DLC CPU in direct # mapped mode. Default is 2-way set associative mode. # # CPU_DISABLE_5X86_LSSER disables load store serialize (i.e., enables # reorder). This option should not be used if you use memory mapped # I/O device(s). # # CPU_ELAN enables support for AMDs ElanSC520 CPU. # CPU_ELAN_PPS enables precision timestamp code. # CPU_ELAN_XTAL sets the clock crystal frequency in Hz. # # CPU_ENABLE_LONGRUN enables support for Transmeta Crusoe LongRun # technology which allows to restrict power consumption of the CPU by # using group of hw.crusoe.* sysctls. # # CPU_FASTER_5X86_FPU enables faster FPU exception handler. # # CPU_GEODE is for the SC1100 Geode embedded processor. This option # is necessary because the i8254 timecounter is toast. # # CPU_I486_ON_386 enables CPU cache on i486 based CPU upgrade products # for i386 machines. # # CPU_IORT defines I/O clock delay time (NOTE 1). Default values of # I/O clock delay time on Cyrix 5x86 and 6x86 are 0 and 7,respectively # (no clock delay). # # CPU_L2_LATENCY specifies the L2 cache latency value. This option is used # only when CPU_PPRO2CELERON is defined and Mendocino Celeron is detected. # The default value is 5. # # CPU_LOOP_EN prevents flushing the prefetch buffer if the destination # of a jump is already present in the prefetch buffer on Cyrix 5x86(NOTE # 1). # # CPU_PPRO2CELERON enables L2 cache of Mendocino Celeron CPUs. This option # is useful when you use Socket 8 to Socket 370 converter, because most Pentium # Pro BIOSs do not enable L2 cache of Mendocino Celeron CPUs. # # CPU_RSTK_EN enables return stack on Cyrix 5x86 (NOTE 1). # # CPU_SOEKRIS enables support www.soekris.com hardware. # # CPU_SUSP_HLT enables suspend on HALT. If this option is set, CPU # enters suspend mode following execution of HALT instruction. # # CPU_UPGRADE_HW_CACHE eliminates unneeded cache flush instruction(s). # # CPU_WT_ALLOC enables write allocation on Cyrix 6x86/6x86MX and AMD # K5/K6/K6-2 CPUs. # # CYRIX_CACHE_WORKS enables CPU cache on Cyrix 486 CPUs with cache # flush at hold state. # # CYRIX_CACHE_REALLY_WORKS enables (1) CPU cache on Cyrix 486 CPUs # without cache flush at hold state, and (2) write-back CPU cache on # Cyrix 6x86 whose revision < 2.7 (NOTE 2). # # NO_F00F_HACK disables the hack that prevents Pentiums (and ONLY # Pentiums) from locking up when a LOCK CMPXCHG8B instruction is # executed. This option is only needed if I586_CPU is also defined, # and should be included for any non-Pentium CPU that defines it. # # NO_MEMORY_HOLE is an optimisation for systems with AMD K6 processors # which indicates that the 15-16MB range is *definitely* not being # occupied by an ISA memory hole. # # NOTE 1: The options, CPU_BTB_EN, CPU_LOOP_EN, CPU_IORT, # CPU_LOOP_EN and CPU_RSTK_EN should not be used because of CPU bugs. # These options may crash your system. # # NOTE 2: If CYRIX_CACHE_REALLY_WORKS is not set, CPU cache is enabled # in write-through mode when revision < 2.7. If revision of Cyrix # 6x86 >= 2.7, CPU cache is always enabled in write-back mode. # # NOTE 3: This option may cause failures for software that requires # locked cycles in order to operate correctly. # options CPU_ATHLON_SSE_HACK options CPU_BLUELIGHTNING_3X options CPU_BLUELIGHTNING_FPU_OP_CACHE options CPU_BTB_EN options CPU_DIRECT_MAPPED_CACHE options CPU_DISABLE_5X86_LSSER options CPU_ELAN options CPU_ELAN_PPS options CPU_ELAN_XTAL=32768000 options CPU_ENABLE_LONGRUN options CPU_FASTER_5X86_FPU options CPU_GEODE options CPU_I486_ON_386 options CPU_IORT options CPU_L2_LATENCY=5 options CPU_LOOP_EN options CPU_PPRO2CELERON options CPU_RSTK_EN options CPU_SOEKRIS options CPU_SUSP_HLT options CPU_UPGRADE_HW_CACHE options CPU_WT_ALLOC options CYRIX_CACHE_WORKS options CYRIX_CACHE_REALLY_WORKS #options NO_F00F_HACK # Debug options options NPX_DEBUG # enable npx debugging # # PERFMON causes the driver for Pentium/Pentium Pro performance counters # to be compiled. See perfmon(4) for more information. # options PERFMON # # XBOX causes the kernel to be bootable on the Microsoft XBox console system. # The resulting kernel will auto-detect whether it is being booted on a XBox, # so kernels compiled with this option will also work on an ordinary PC. # This option require I686_CPU. # # xboxfb includes support for the XBox frame buffer device. It is fully USB- # keyboard aware, and will only be used if an xbox is detected. This option # (obviously) requires XBOX support in your kernel. # # NOTE: xboxfb currently conflicts with syscons(4); if you have an XBOX and # include both in your kernel; you will not get any video output. Ordinary # PC's do not suffer from this. # options XBOX device xboxfb ##################################################################### # NETWORKING OPTIONS # # DEVICE_POLLING adds support for mixed interrupt-polling handling # of network device drivers, which has significant benefits in terms # of robustness to overloads and responsivity, as well as permitting # accurate scheduling of the CPU time between kernel network processing # and other activities. The drawback is a moderate (up to 1/HZ seconds) # potential increase in response times. # It is strongly recommended to use HZ=1000 or 2000 with DEVICE_POLLING # to achieve smoother behaviour. # Additionally, you can enable/disable polling at runtime with help of # the ifconfig(8) utility, and select the CPU fraction reserved to # userland with the sysctl variable kern.polling.user_frac # (default 50, range 0..100). # # Not all device drivers support this mode of operation at the time of # this writing. See polling(4) for more details. options DEVICE_POLLING # BPF_JITTER adds support for BPF just-in-time compiler. options BPF_JITTER # OpenFabrics Enterprise Distribution (Infiniband). options OFED options OFED_DEBUG_INIT # Sockets Direct Protocol options SDP options SDP_DEBUG # IP over Infiniband options IPOIB options IPOIB_DEBUG options IPOIB_CM ##################################################################### # CLOCK OPTIONS # Provide read/write access to the memory in the clock chip. device nvram # Access to rtc cmos via /dev/nvram ##################################################################### # MISCELLANEOUS DEVICES AND OPTIONS device speaker #Play IBM BASIC-style noises out your speaker hint.speaker.0.at="isa" hint.speaker.0.port="0x61" device gzip #Exec gzipped a.out's. REQUIRES COMPAT_AOUT! device apm_saver # Requires APM ##################################################################### # HARDWARE BUS CONFIGURATION # # ISA bus # device isa # # Options for `isa': # # AUTO_EOI_1 enables the `automatic EOI' feature for the master 8259A # interrupt controller. This saves about 0.7-1.25 usec for each interrupt. # This option breaks suspend/resume on some portables. # # AUTO_EOI_2 enables the `automatic EOI' feature for the slave 8259A # interrupt controller. This saves about 0.7-1.25 usec for each interrupt. # Automatic EOI is documented not to work for for the slave with the # original i8259A, but it works for some clones and some integrated # versions. # # MAXMEM specifies the amount of RAM on the machine; if this is not # specified, FreeBSD will first read the amount of memory from the CMOS # RAM, so the amount of memory will initially be limited to 64MB or 16MB # depending on the BIOS. If the BIOS reports 64MB, a memory probe will # then attempt to detect the installed amount of RAM. If this probe # fails to detect >64MB RAM you will have to use the MAXMEM option. # The amount is in kilobytes, so for a machine with 128MB of RAM, it would # be 131072 (128 * 1024). # # BROKEN_KEYBOARD_RESET disables the use of the keyboard controller to # reset the CPU for reboot. This is needed on some systems with broken # keyboard controllers. options AUTO_EOI_1 #options AUTO_EOI_2 options MAXMEM=(128*1024) #options BROKEN_KEYBOARD_RESET # # EISA bus # # The EISA bus device is `eisa'. It provides auto-detection and # configuration support for all devices on the EISA bus. device eisa # By default, only 10 EISA slots are probed, since the slot numbers # above clash with the configuration address space of the PCI subsystem, # and the EISA probe is not very smart about this. This is sufficient # for most machines, but in particular the HP NetServer LC series comes # with an onboard AIC7770 dual-channel SCSI controller on EISA slot #11, # thus you need to bump this figure to 12 for them. options EISA_SLOTS=12 # # MCA bus: # # The MCA bus device is `mca'. It provides auto-detection and # configuration support for all devices on the MCA bus. # No hints are required for MCA. device mca # # AGP GART support device agp # AGP debugging. options AGP_DEBUG ##################################################################### # HARDWARE DEVICE CONFIGURATION # To include support for VGA VESA video modes options VESA # Turn on extra debugging checks and output for VESA support. options VESA_DEBUG device dpms # DPMS suspend & resume via VESA BIOS # x86 real mode BIOS emulator, required by atkbdc/dpms/vesa options X86BIOS # # Hints for the non-optional Numeric Processing eXtension driver. hint.npx.0.flags="0x0" hint.npx.0.irq="13" # # `flags' for npx0: # 0x01 don't use the npx registers to optimize bcopy. # 0x02 don't use the npx registers to optimize bzero. # 0x04 don't use the npx registers to optimize copyin or copyout. # The npx registers are normally used to optimize copying and zeroing when # all of the following conditions are satisfied: # I586_CPU is an option # the cpu is an i586 (perhaps not a Pentium) # the probe for npx0 succeeds # INT 16 exception handling works. # Then copying and zeroing using the npx registers is normally 30-100% faster. # The flags can be used to control cases where it doesn't work or is slower. # Setting them at boot time using hints works right (the optimizations # are not used until later in the bootstrap when npx0 is attached). # Flag 0x08 automatically disables the i586 optimized routines. # # # Optional devices: # # PS/2 mouse device psm hint.psm.0.at="atkbdc" hint.psm.0.irq="12" # Options for psm: options PSM_HOOKRESUME #hook the system resume event, useful #for some laptops options PSM_RESETAFTERSUSPEND #reset the device at the resume event # The keyboard controller; it controls the keyboard and the PS/2 mouse. device atkbdc hint.atkbdc.0.at="isa" hint.atkbdc.0.port="0x060" # The AT keyboard device atkbd hint.atkbd.0.at="atkbdc" hint.atkbd.0.irq="1" # Options for atkbd: options ATKBD_DFLT_KEYMAP # specify the built-in keymap makeoptions ATKBD_DFLT_KEYMAP=fr.dvorak # `flags' for atkbd: # 0x01 Force detection of keyboard, else we always assume a keyboard # 0x02 Don't reset keyboard, useful for some newer ThinkPads # 0x03 Force detection and avoid reset, might help with certain # dockingstations # 0x04 Old-style (XT) keyboard support, useful for older ThinkPads # Video card driver for VGA adapters. device vga hint.vga.0.at="isa" # Options for vga: # Try the following option if the mouse pointer is not drawn correctly # or font does not seem to be loaded properly. May cause flicker on # some systems. options VGA_ALT_SEQACCESS # If you can dispense with some vga driver features, you may want to # use the following options to save some memory. #options VGA_NO_FONT_LOADING # don't save/load font #options VGA_NO_MODE_CHANGE # don't change video modes # Older video cards may require this option for proper operation. options VGA_SLOW_IOACCESS # do byte-wide i/o's to TS and GDC regs # The following option probably won't work with the LCD displays. options VGA_WIDTH90 # support 90 column modes # Debugging. options VGA_DEBUG # vt(4) drivers. device vt_vga # Linear framebuffer driver for S3 VESA 1.2 cards. Works on top of VESA. device s3pci # 3Dfx Voodoo Graphics, Voodoo II /dev/3dfx CDEV support. This will create # the /dev/3dfx0 device to work with glide implementations. This should get # linked to /dev/3dfx and /dev/voodoo. Note that this is not the same as # the tdfx DRI module from XFree86 and is completely unrelated. # # To enable Linuxulator support, one must also include COMPAT_LINUX in the # config as well. The other option is to load both as modules. device tdfx # Enable 3Dfx Voodoo support device tdfx_linux # Enable Linuxulator support # # ACPI support using the Intel ACPI Component Architecture reference # implementation. # # ACPI_DEBUG enables the use of the debug.acpi.level and debug.acpi.layer # kernel environment variables to select initial debugging levels for the # Intel ACPICA code. (Note that the Intel code must also have USE_DEBUGGER # defined when it is built). device acpi options ACPI_DEBUG options ACPI_DMAR # ACPI WMI Mapping driver device acpi_wmi # ACPI Asus Extras (LCD backlight/brightness, video output, etc.) device acpi_asus # ACPI Fujitsu Extras (Buttons) device acpi_fujitsu # ACPI extras driver for HP laptops device acpi_hp # ACPI extras driver for IBM laptops device acpi_ibm # ACPI Panasonic Extras (LCD backlight/brightness, video output, etc.) device acpi_panasonic # ACPI Sony extra (LCD brightness) device acpi_sony # ACPI Toshiba Extras (LCD backlight/brightness, video output, etc.) device acpi_toshiba # ACPI Video Extensions (LCD backlight/brightness, video output, etc.) device acpi_video # ACPI Docking Station device acpi_dock # ACPI ASOC ATK0110 ASUSTeK AI Booster (voltage, temperature and fan sensors) device aibs # The cpufreq(4) driver provides support for non-ACPI CPU frequency control device cpufreq # Direct Rendering modules for 3D acceleration. device drm # DRM core module required by DRM drivers device i915drm # Intel i830 through i915 device mach64drm # ATI Rage Pro, Rage Mobility P/M, Rage XL device mgadrm # AGP Matrox G200, G400, G450, G550 device r128drm # ATI Rage 128 device radeondrm # ATI Radeon device savagedrm # S3 Savage3D, Savage4 device sisdrm # SiS 300/305, 540, 630 device tdfxdrm # 3dfx Voodoo 3/4/5 and Banshee device viadrm # VIA options DRM_DEBUG # Include debug printfs (slow) # # mse: Logitech and ATI InPort bus mouse ports device mse hint.mse.0.at="isa" hint.mse.0.port="0x23c" hint.mse.0.irq="5" # # Network interfaces: # # bxe: Broadcom NetXtreme II (BCM5771X/BCM578XX) PCIe 10Gb Ethernet # adapters. # ce: Cronyx Tau-PCI/32 sync single/dual port G.703/E1 serial adaptor # with 32 HDLC subchannels (requires sppp (default), or NETGRAPH if # NETGRAPH_CRONYX is configured) # cp: Cronyx Tau-PCI sync single/dual/four port # V.35/RS-232/RS-530/RS-449/X.21/G.703/E1/E3/T3/STS-1 # serial adaptor (requires sppp (default), or NETGRAPH if # NETGRAPH_CRONYX is configured) # cs: IBM Etherjet and other Crystal Semi CS89x0-based adapters # ctau: Cronyx Tau sync dual port V.35/RS-232/RS-530/RS-449/X.21/G.703/E1 # serial adaptor (requires sppp (default), or NETGRAPH if # NETGRAPH_CRONYX is configured) # ed: Western Digital and SMC 80xx; Novell NE1000 and NE2000; 3Com 3C503 # HP PC Lan+, various PC Card devices # (requires miibus) # ie: AT&T StarLAN 10 and EN100; 3Com 3C507; unknown NI5210; # Intel EtherExpress # ipw: Intel PRO/Wireless 2100 IEEE 802.11 adapter # iwi: Intel PRO/Wireless 2200BG/2225BG/2915ABG IEEE 802.11 adapters # Requires the iwi firmware module # iwn: Intel Wireless WiFi Link 1000/105/135/2000/4965/5000/6000/6050 abgn # 802.11 network adapters # Requires the iwn firmware module # mlx4ib: Mellanox ConnectX HCA InfiniBand -# mlxen: Mellanox ConnectX HCA Ethernet +# mlx4en: Mellanox ConnectX HCA Ethernet # mthca: Mellanox HCA InfiniBand # nfe: nVidia nForce MCP on-board Ethernet Networking (BSD open source) # sbni: Granch SBNI12-xx ISA and PCI adapters # vmx: VMware VMXNET3 Ethernet (BSD open source) # wl: Lucent Wavelan (ISA card only). # wpi: Intel 3945ABG Wireless LAN controller # Requires the wpi firmware module # Order for ISA/EISA devices is important here device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE device ce device cp device cs # Crystal Semiconductor CS89x0 NIC hint.cs.0.at="isa" hint.cs.0.port="0x300" device ctau hint.ctau.0.at="isa" hint.ctau.0.port="0x240" hint.ctau.0.irq="15" hint.ctau.0.drq="7" #options NETGRAPH_CRONYX # Enable NETGRAPH support for Cronyx adapter(s) device ed # NE[12]000, SMC Ultra, 3c503, DS8390 cards options ED_3C503 options ED_HPP options ED_SIC hint.ed.0.at="isa" hint.ed.0.port="0x280" hint.ed.0.irq="5" hint.ed.0.maddr="0xd8000" device ie # EtherExpress 8/16, 3C507, StarLAN 10 etc. # Hints only required for Starlan hint.ie.2.at="isa" hint.ie.2.port="0x300" hint.ie.2.irq="5" hint.ie.2.maddr="0xd0000" device ipw # Intel 2100 wireless NICs. device iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. device iwn # Intel 4965/1000/5000/6000 wireless NICs. # Hint for the i386-only ISA front-end of le(4). hint.le.0.at="isa" hint.le.0.port="0x280" hint.le.0.irq="10" hint.le.0.drq="0" +device mlx4 # Shared code module between IB and Ethernet device mlx4ib # Mellanox ConnectX HCA InfiniBand -device mlxen # Mellanox ConnectX HCA Ethernet +device mlx4en # Mellanox ConnectX HCA Ethernet device mthca # Mellanox HCA InfiniBand device nfe # nVidia nForce MCP on-board Ethernet device sbni hint.sbni.0.at="isa" hint.sbni.0.port="0x210" hint.sbni.0.irq="0xefdead" hint.sbni.0.flags="0" device vmx # VMware VMXNET3 Ethernet device wl hint.wl.0.at="isa" hint.wl.0.port="0x300" options WLCACHE # enables the signal-strength cache options WLDEBUG # enables verbose debugging output device wpi # Intel 3945ABG wireless NICs. # IEEE 802.11 adapter firmware modules # Intel PRO/Wireless 2100 firmware: # ipwfw: BSS/IBSS/monitor mode firmware # ipwbssfw: BSS mode firmware # ipwibssfw: IBSS mode firmware # ipwmonitorfw: Monitor mode firmware # Intel PRO/Wireless 2200BG/2225BG/2915ABG firmware: # iwifw: BSS/IBSS/monitor mode firmware # iwibssfw: BSS mode firmware # iwiibssfw: IBSS mode firmware # iwimonitorfw: Monitor mode firmware # Intel Wireless WiFi Link 4965/1000/5000/6000 series firmware: # iwnfw: Single module to support all devices # iwn1000fw: Specific module for the 1000 only # iwn105fw: Specific module for the 105 only # iwn135fw: Specific module for the 135 only # iwn2000fw: Specific module for the 2000 only # iwn2030fw: Specific module for the 2030 only # iwn4965fw: Specific module for the 4965 only # iwn5000fw: Specific module for the 5000 only # iwn5150fw: Specific module for the 5150 only # iwn6000fw: Specific module for the 6000 only # iwn6000g2afw: Specific module for the 6000g2a only # iwn6000g2bfw: Specific module for the 6000g2b only # iwn6050fw: Specific module for the 6050 only # wpifw: Intel 3945ABG Wireless LAN Controller firmware device iwifw device iwibssfw device iwiibssfw device iwimonitorfw device ipwfw device ipwbssfw device ipwibssfw device ipwmonitorfw device iwnfw device iwn1000fw device iwn105fw device iwn135fw device iwn2000fw device iwn2030fw device iwn4965fw device iwn5000fw device iwn5150fw device iwn6000fw device iwn6000g2afw device iwn6000g2bfw device iwn6050fw device wpifw # # Non-Transparent Bridge (NTB) drivers # device if_ntb # Virtual NTB network interface device ntb_transport # NTB packet transport driver device ntb # NTB hardware interface device ntb_hw_intel # Intel NTB hardware driver device ntb_hw_plx # PLX NTB hardware driver # # ATA raid adapters # device pst # # Areca 11xx and 12xx series of SATA II RAID controllers. # CAM is required. # device arcmsr # Areca SATA II RAID # # 3ware 9000 series PATA/SATA RAID controller driver and options. # The driver is implemented as a SIM, and so, needs the CAM infrastructure. # options TWA_DEBUG # 0-10; 10 prints the most messages. options TWA_FLASH_FIRMWARE # firmware image bundled when defined. device twa # 3ware 9000 series PATA/SATA RAID # # SCSI host adapters: # # ncv: NCR 53C500 based SCSI host adapters. # nsp: Workbit Ninja SCSI-3 based PC Card SCSI host adapters. # stg: TMC 18C30, 18C50 based SCSI host adapters. device ncv device nsp device stg hint.stg.0.at="isa" hint.stg.0.port="0x140" hint.stg.0.port="11" # # Adaptec FSA RAID controllers, including integrated DELL controllers, # the Dell PERC 2/QC and the HP NetRAID-4M device aac device aacp # SCSI Passthrough interface (optional, CAM required) # # Adaptec by PMC RAID controllers, Series 6/7/8 and upcoming families device aacraid # Container interface, CAM required # # Highpoint RocketRAID 27xx. device hpt27xx # # Highpoint RocketRAID 182x. device hptmv # # Highpoint DC7280 and R750. device hptnr # # Highpoint RocketRAID. Supports RR172x, RR222x, RR2240, RR232x, RR2340, # RR2210, RR174x, RR2522, RR231x, RR230x. device hptrr # # Highpoint RocketRaid 3xxx series SATA RAID device hptiop # # IBM (now Adaptec) ServeRAID controllers device ips # # Intel C600 (Patsburg) integrated SAS controller device isci options ISCI_LOGGING # enable debugging in isci HAL # # NVM Express (NVMe) support device nvme # base NVMe driver device nvd # expose NVMe namespaces as disks, depends on nvme # # PMC-Sierra SAS/SATA controller device pmspcv # # SafeNet crypto driver: can be moved to the MI NOTES as soon as # it's tested on a big-endian machine # device safe # SafeNet 1141 options SAFE_DEBUG # enable debugging support: hw.safe.debug options SAFE_RNDTEST # enable rndtest support # # glxiic is an I2C driver for the AMD Geode LX CS5536 System Management Bus # controller. Requires 'device iicbus'. # device glxiic # AMD Geode LX CS5536 System Management Bus # # glxsb is a driver for the Security Block in AMD Geode LX processors. # Requires 'device crypto'. # device glxsb # AMD Geode LX Security Block # # VirtIO support # # The virtio entry provides a generic bus for use by the device drivers. # It must be combined with an interface that communicates with the host. # Multiple such interfaces defined by the VirtIO specification. FreeBSD # only has support for PCI. Therefore, virtio_pci must be statically # compiled in or loaded as a module for the device drivers to function. # device virtio # Generic VirtIO bus (required) device virtio_pci # VirtIO PCI Interface device vtnet # VirtIO Ethernet device device virtio_blk # VirtIO Block device device virtio_scsi # VirtIO SCSI device device virtio_balloon # VirtIO Memory Balloon device device virtio_random # VirtIO Entropy device device virtio_console # VirtIO Console device device hyperv # HyperV drivers ##################################################################### # # Miscellaneous hardware: # # apm: Laptop Advanced Power Management (experimental) # ipmi: Intelligent Platform Management Interface # smapi: System Management Application Program Interface driver # smbios: DMI/SMBIOS entry point # vpd: Vital Product Data kernel interface # pmtimer: Adjust system timer at wakeup time # pbio: Parallel (8255 PPI) basic I/O (mode 0) port (e.g. Advantech PCL-724) # spic: Sony Programmable I/O controller (VAIO notebooks) # asmc: Apple System Management Controller # si: Specialix International SI/XIO or SX intelligent serial card driver # tpm: Trusted Platform Module # Notes on APM # The flags takes the following meaning for apm0: # 0x0020 Statclock is broken. # Notes on the Specialix SI/XIO driver: # The host card is memory, not IO mapped. # The Rev 1 host cards use a 64K chunk, on a 32K boundary. # The Rev 2 host cards use a 32K chunk, on a 32K boundary. # The cards can use an IRQ of 11, 12 or 15. # Notes on the Sony Programmable I/O controller # This is a temporary driver that should someday be replaced by something # that hooks into the ACPI layer. The device is hooked to the PIIX4's # General Device 10 decoder, which means you have to fiddle with PCI # registers to map it in, even though it is otherwise treated here as # an ISA device. At the moment, the driver polls, although the device # is capable of generating interrupts. It largely undocumented. # The port location in the hint is where you WANT the device to be # mapped. 0x10a0 seems to be traditional. At the moment the jogdial # is the only thing truly supported, but apparently a fair percentage # of the Vaio extra features are controlled by this device. device apm hint.apm.0.flags="0x20" device ipmi device smapi device smbios device vpd device pmtimer device pbio hint.pbio.0.at="isa" hint.pbio.0.port="0x360" device spic hint.spic.0.at="isa" hint.spic.0.port="0x10a0" device asmc device si device tpm device padlock_rng # VIA Padlock RNG device rdrand_rng # Intel Bull Mountain RNG device aesni # AES-NI OpenCrypto module # # Laptop/Notebook options: # # See also: # apm under `Miscellaneous hardware' # above. # For older notebooks that signal a powerfail condition (external # power supply dropped, or battery state low) by issuing an NMI: options POWERFAIL_NMI # make it beep instead of panicing # # I2C Bus # # Philips i2c bus support is provided by the `iicbus' device. # # Supported interfaces: # pcf Philips PCF8584 ISA-bus controller # device pcf hint.pcf.0.at="isa" hint.pcf.0.port="0x320" hint.pcf.0.irq="5" # # Hardware watchdog timers: # # ichwd: Intel ICH watchdog timer # amdsbwd: AMD SB7xx watchdog timer # viawd: VIA south bridge watchdog timer # wbwd: Winbond watchdog timer # device ichwd device amdsbwd device viawd device wbwd # # Temperature sensors: # # coretemp: on-die sensor on Intel Core and newer CPUs # amdtemp: on-die sensor on AMD K8/K10/K11 CPUs # device coretemp device amdtemp # # CPU control pseudo-device. Provides access to MSRs, CPUID info and # microcode update feature. # device cpuctl # # System Management Bus (SMB) # options ENABLE_ALART # Control alarm on Intel intpm driver # # Set the number of PV entries per process. Increasing this can # stop panics related to heavy use of shared memory. However, that can # (combined with large amounts of physical memory) cause panics at # boot time due the kernel running out of VM space. # # If you're tweaking this, you might also want to increase the sysctls # "vm.v_free_min", "vm.v_free_reserved", and "vm.v_free_target". # # The value below is the one more than the default. # options PMAP_SHPGPERPROC=201 # # Change the size of the kernel virtual address space. Due to # constraints in loader(8) on i386, this must be a multiple of 4. # 256 = 1 GB of kernel address space. Increasing this also causes # a reduction of the address space in user processes. 512 splits # the 4GB cpu address space in half (2GB user, 2GB kernel). For PAE # kernels, the value will need to be double non-PAE. A value of 1024 # for PAE kernels is necessary to split the address space in half. # This will likely need to be increased to handle memory sizes >4GB. # PAE kernels default to a value of 512. # options KVA_PAGES=260 # # Number of initial kernel page table pages used for early bootstrap. # This number should include enough pages to map the kernel, any # modules or other data loaded with the kernel by the loader, and data # structures allocated before the VM system is initialized such as the # vm_page_t array. Each page table page maps 4MB (2MB with PAE). # options NKPT=31 ##################################################################### # ABI Emulation # Enable iBCS2 runtime support for SCO and ISC binaries #options IBCS2 # Emulate spx device for client side of SVR3 local X interface options SPX_HACK # Enable 32-bit runtime support for CloudABI binaries. options COMPAT_CLOUDABI32 # Enable Linux ABI emulation options COMPAT_LINUX # Enable i386 a.out binary support options COMPAT_AOUT # Enable the linux-like proc filesystem support (requires COMPAT_LINUX # and PSEUDOFS) options LINPROCFS #Enable the linux-like sys filesystem support (requires COMPAT_LINUX # and PSEUDOFS) options LINSYSFS # # SysVR4 ABI emulation # # The svr4 ABI emulator can be statically compiled into the kernel or loaded as # a KLD module. # The STREAMS network emulation code can also be compiled statically or as a # module. If loaded as a module, it must be loaded before the svr4 module # (the /usr/sbin/svr4 script does this for you). If compiling statically, # the `streams' device must be configured into any kernel which also # specifies COMPAT_SVR4. It is possible to have a statically-configured # STREAMS device and a dynamically loadable svr4 emulator; the /usr/sbin/svr4 # script understands that it doesn't need to load the `streams' module under # those circumstances. # Caveat: At this time, `options KTRACE' is required for the svr4 emulator # (whether static or dynamic). # options COMPAT_SVR4 # build emulator statically options DEBUG_SVR4 # enable verbose debugging device streams # STREAMS network driver (required for svr4). # Enable NDIS binary driver support options NDISAPI device ndis ##################################################################### # VM OPTIONS # Disable the 4 MByte page PSE CPU feature. The PSE feature allows the # kernel to use 4 MByte pages to map the kernel instead of 4k pages. # This saves on the amount of memory needed for page tables needed to # map the kernel. You should only disable this feature as a temporary # workaround if you are having problems with it enabled. # #options DISABLE_PSE # Disable the global pages PGE CPU feature. The PGE feature allows pages # to be marked with the PG_G bit. TLB entries for these pages are not # flushed from the cache when %cr3 is reloaded. This can make context # switches less expensive. You should only disable this feature as a # temporary workaround if you are having problems with it enabled. # #options DISABLE_PG_G # KSTACK_PAGES is the number of memory pages to assign to the kernel # stack of each thread. options KSTACK_PAGES=3 # Enable detailed accounting by the PV entry allocator. options PV_STATS ##################################################################### # More undocumented options for linting. # Note that documenting these are not considered an affront. options FB_INSTALL_CDEV # install a CDEV entry in /dev options I586_PMC_GUPROF=0x70000 options KBDIO_DEBUG=2 options KBD_MAXRETRY=4 options KBD_MAXWAIT=6 options KBD_RESETDELAY=201 options PSM_DEBUG=1 options TIMER_FREQ=((14318182+6)/12) options VM_KMEM_SIZE options VM_KMEM_SIZE_MAX options VM_KMEM_SIZE_SCALE Index: stable/11/sys/modules/mlxen/Makefile =================================================================== --- stable/11/sys/modules/mlxen/Makefile (revision 329158) +++ stable/11/sys/modules/mlxen/Makefile (nonexistent) @@ -1,13 +0,0 @@ -# $FreeBSD$ -.PATH: ${.CURDIR}/../../ofed/drivers/net/mlx4 - -KMOD = mlxen -SRCS = device_if.h bus_if.h pci_if.h vnode_if.h -SRCS += en_cq.c en_main.c en_netdev.c en_port.c en_resources.c -SRCS += en_rx.c en_tx.c -SRCS += opt_inet.h opt_inet6.h -CFLAGS+= -I${.CURDIR}/../../ofed/drivers/net/mlx4 -CFLAGS+= -I${.CURDIR}/../../ofed/include -CFLAGS+= -I${.CURDIR}/../../compat/linuxkpi/common/include - -.include Property changes on: stable/11/sys/modules/mlxen/Makefile ___________________________________________________________________ Deleted: svn:keywords ## -1 +0,0 ## -FreeBSD=%H \ No newline at end of property Index: stable/11/sys/modules/Makefile =================================================================== --- stable/11/sys/modules/Makefile (revision 329158) +++ stable/11/sys/modules/Makefile (revision 329159) @@ -1,848 +1,848 @@ # $FreeBSD$ SYSDIR?=${SRCTOP}/sys .include "${SYSDIR}/conf/kern.opts.mk" SUBDIR_PARALLEL= # Modules that include binary-only blobs of microcode should be selectable by # MK_SOURCELESS_UCODE option (see below). .if defined(MODULES_OVERRIDE) && !defined(ALL_MODULES) SUBDIR=${MODULES_OVERRIDE} .else SUBDIR= \ ${_3dfx} \ ${_3dfx_linux} \ ${_aac} \ ${_aacraid} \ accf_data \ accf_dns \ accf_http \ acl_nfs4 \ acl_posix1e \ ${_acpi} \ ae \ ${_aesni} \ age \ ${_agp} \ aha \ ${_ahb} \ ahci \ ${_aic} \ aic7xxx \ alc \ ale \ alq \ ${_amd_ecc_inject} \ ${_amdsbwd} \ ${_amdtemp} \ amr \ ${_an} \ ${_aout} \ ${_apm} \ ${_arcmsr} \ ${_arcnet} \ ${_asmc} \ ata \ ath \ ath_pci \ ${_autofs} \ ${_auxio} \ ${_bce} \ bfe \ bge \ bhnd \ ${_bxe} \ ${_bios} \ ${_bktr} \ ${_bm} \ bnxt \ bridgestp \ bwi \ bwn \ bwn_pci \ ${_bytgpio} \ cam \ ${_canbepm} \ ${_canbus} \ ${_cardbus} \ ${_carp} \ cas \ ${_cbb} \ cc \ cd9660 \ cd9660_iconv \ ${_ce} \ ${_cfi} \ ${_chromebook_platform} \ ${_ciss} \ cloudabi \ ${_cloudabi32} \ ${_cloudabi64} \ ${_cm} \ ${_cmx} \ ${_coff} \ ${_coretemp} \ ${_cp} \ ${_cpsw} \ ${_cpuctl} \ ${_cpufreq} \ ${_crypto} \ ${_cryptodev} \ ${_cs} \ ${_ct} \ ${_ctau} \ ctl \ ${_cxgb} \ ${_cxgbe} \ dc \ dcons \ dcons_crom \ de \ ${_dpms} \ ${_dpt} \ ${_drm} \ ${_drm2} \ dummynet \ ${_ed} \ ${_efirt} \ ${_elink} \ ${_em} \ ${_ena} \ en \ ${_ep} \ ${_epic} \ esp \ ${_et} \ evdev \ ${_ex} \ ${_exca} \ ext2fs \ ${_fatm} \ fdc \ fdescfs \ ${_fe} \ ${_ffec} \ filemon \ firewire \ firmware \ fuse \ ${_fxp} \ gem \ geom \ ${_glxiic} \ ${_glxsb} \ gpio \ hatm \ hifn \ hme \ ${_hpt27xx} \ ${_hptiop} \ ${_hptmv} \ ${_hptnr} \ ${_hptrr} \ hwpmc \ ${_hyperv} \ i2c \ ${_ibcore} \ ${_ibcs2} \ ${_ichwd} \ ${_ida} \ ${_ie} \ if_bridge \ if_disc \ if_edsc \ ${_if_enc} \ if_epair \ ${_if_gif} \ ${_if_gre} \ ${_if_me} \ if_lagg \ ${_if_ndis} \ ${_if_stf} \ if_tap \ if_tun \ if_vlan \ if_vxlan \ ${_igb} \ ${_iir} \ imgact_binmisc \ ${_intelspi} \ ${_io} \ ${_ioat} \ ${_ipoib} \ ${_ipdivert} \ ${_ipfilter} \ ${_ipfw} \ ipfw_nat \ ${_ipfw_nat64} \ ${_ipfw_nptv6} \ ${_ipfw_pmod} \ ${_ipmi} \ ip6_mroute_mod \ ip_mroute_mod \ ${_ips} \ ${_ipsec} \ ${_ipw} \ ${_ipwfw} \ ${_isci} \ ${_iser} \ isp \ ${_ispfw} \ ${_iwi} \ ${_iwifw} \ ${_iwm} \ ${_iwmfw} \ ${_iwn} \ ${_iwnfw} \ ${_ix} \ ${_ixv} \ ${_ixgb} \ ${_ixl} \ ${_ixlv} \ jme \ joy \ kbdmux \ kgssapi \ kgssapi_krb5 \ khelp \ krpc \ ksyms \ le \ lge \ libalias \ libiconv \ libmbpool \ libmchain \ ${_linprocfs} \ ${_linsysfs} \ ${_linux} \ ${_linux_common} \ ${_linux64} \ linuxkpi \ ${_lio} \ lmc \ lpt \ mac_biba \ mac_bsdextended \ mac_ifoff \ mac_lomac \ mac_mls \ mac_none \ mac_partition \ mac_portacl \ mac_seeotheruids \ mac_stub \ mac_test \ malo \ mcd \ md \ mdio \ mem \ mfi \ mii \ mlx \ ${_mlx4} \ ${_mlx4ib} \ - ${_mlxen} \ + ${_mlx4en} \ ${_mlx5} \ ${_mlx5en} \ ${_mlx5ib} \ ${_mly} \ mmc \ mmcsd \ mpr \ mps \ mpt \ mqueue \ mrsas \ msdosfs \ msdosfs_iconv \ ${_mse} \ msk \ ${_mthca} \ mvs \ mwl \ ${_mwlfw} \ mxge \ my \ ${_nandfs} \ ${_nandsim} \ ${_ncr} \ ${_nctgpio} \ ${_ncv} \ ${_ndis} \ netfpga10g \ ${_netgraph} \ ${_nfe} \ nfscl \ nfscommon \ nfsd \ nfslock \ nfslockd \ nfssvc \ nge \ nmdm \ ${_nsp} \ nullfs \ ${_ntb} \ ${_nvd} \ ${_nvme} \ ${_nvram} \ ${_nxge} \ oce \ otus \ ${_otusfw} \ ow \ ${_padlock} \ ${_padlock_rng} \ patm \ ${_pccard} \ ${_pcfclock} \ pcn \ ${_pf} \ ${_pflog} \ ${_pfsync} \ plip \ ${_pmc} \ ${_pms} \ ppbus \ ppc \ ppi \ pps \ procfs \ proto \ pseudofs \ ${_pst} \ pty \ puc \ ${_qlxge} \ ${_qlxgb} \ ${_qlxgbe} \ ${_qlnx} \ ral \ ${_ralfw} \ ${_random_fortuna} \ ${_random_yarrow} \ ${_random_other} \ rc4 \ ${_rdma} \ ${_rdrand_rng} \ re \ rl \ rtwn \ ${_rtwnfw} \ ${_s3} \ ${_safe} \ ${_sbni} \ scc \ scd \ ${_scsi_low} \ sdhci \ ${_sdhci_acpi} \ sdhci_pci \ sem \ send \ ${_sf} \ ${_sfxge} \ sge \ ${_si} \ siba_bwn \ siftr \ siis \ sis \ sk \ smbfs \ sn \ ${_snc} \ snp \ sound \ ${_speaker} \ ${_splash} \ ${_sppp} \ ste \ ${_stg} \ stge \ ${_streams} \ ${_svr4} \ ${_sym} \ ${_syscons} \ sysvipc \ tcp \ ${_ti} \ tl \ tmpfs \ ${_toecore} \ ${_tpm} \ trm \ ${_twa} \ twe \ tws \ tx \ ${_txp} \ uart \ ubsec \ udf \ udf_iconv \ ufs \ uinput \ unionfs \ urtwn \ ${_urtwnfw} \ usb \ utopia \ ${_vesa} \ ${_virtio} \ vge \ ${_viawd} \ videomode \ vkbd \ ${_vmm} \ ${_vmware} \ ${_vpo} \ vr \ vte \ vx \ ${_vxge} \ wb \ ${_wbwd} \ ${_wds} \ ${_wi} \ ${_wl} \ wlan \ wlan_acl \ wlan_amrr \ wlan_ccmp \ wlan_rssadapt \ wlan_tkip \ wlan_wep \ wlan_xauth \ ${_wpi} \ ${_wpifw} \ ${_x86bios} \ ${_xe} \ xl \ zlib .if ${MK_AUTOFS} != "no" || defined(ALL_MODULES) _autofs= autofs .endif .if ${MK_CDDL} != "no" || defined(ALL_MODULES) .if (${MACHINE_CPUARCH} != "arm" || ${MACHINE_ARCH:Marmv6*} != "") && \ ${MACHINE_CPUARCH} != "mips" && \ ${MACHINE_CPUARCH} != "sparc64" SUBDIR+= dtrace .endif SUBDIR+= opensolaris .endif .if ${MK_CRYPT} != "no" || defined(ALL_MODULES) .if exists(${SRCTOP}/sys/opencrypto) _crypto= crypto _cryptodev= cryptodev _random_fortuna=random_fortuna _random_yarrow= random_yarrow _random_other= random_other .endif .endif .if ${MK_CUSE} != "no" || defined(ALL_MODULES) SUBDIR+= cuse .endif .if (${MK_INET_SUPPORT} != "no" || ${MK_INET6_SUPPORT} != "no") || \ defined(ALL_MODULES) _carp= carp _toecore= toecore _if_enc= if_enc _if_gif= if_gif _if_gre= if_gre _ipfw_pmod= ipfw_pmod .if ${MK_IPSEC_SUPPORT} != "no" _ipsec= ipsec .endif .endif .if (${MK_INET_SUPPORT} != "no" && ${MK_INET6_SUPPORT} != "no") || \ defined(ALL_MODULES) _if_stf= if_stf .endif .if ${MK_INET_SUPPORT} != "no" || defined(ALL_MODULES) _if_me= if_me _ipdivert= ipdivert _ipfw= ipfw .if ${MK_INET6_SUPPORT} != "no" || defined(ALL_MODULES) _ipfw_nat64= ipfw_nat64 .endif .endif .if ${MK_INET6_SUPPORT} != "no" || defined(ALL_MODULES) _ipfw_nptv6= ipfw_nptv6 .endif .if ${MK_IPFILTER} != "no" || defined(ALL_MODULES) _ipfilter= ipfilter .endif .if ${MK_ISCSI} != "no" || defined(ALL_MODULES) SUBDIR+= cfiscsi SUBDIR+= iscsi SUBDIR+= iscsi_initiator .endif .if ${MK_NAND} != "no" || defined(ALL_MODULES) _nandfs= nandfs _nandsim= nandsim .endif .if ${MK_NETGRAPH} != "no" || defined(ALL_MODULES) _netgraph= netgraph .endif .if (${MK_PF} != "no" && (${MK_INET_SUPPORT} != "no" || \ ${MK_INET6_SUPPORT} != "no")) || defined(ALL_MODULES) _pf= pf _pflog= pflog .if ${MK_INET_SUPPORT} != "no" _pfsync= pfsync .endif .endif .if ${MK_SOURCELESS_UCODE} != "no" _bce= bce _fatm= fatm _fxp= fxp _ispfw= ispfw _mwlfw= mwlfw _otusfw= otusfw _ralfw= ralfw _rtwnfw= rtwnfw _urtwnfw= urtwnfw _sf= sf _ti= ti _txp= txp .endif .if ${MK_SOURCELESS_UCODE} != "no" && ${MACHINE_CPUARCH} != "arm" && \ ${MACHINE_ARCH:C/mips(el)?/mips/} != "mips" && \ ${MACHINE_ARCH} != "powerpc" && ${MACHINE_CPUARCH} != "riscv" _cxgbe= cxgbe .endif .if ${MK_TESTS} != "no" || defined(ALL_MODULES) SUBDIR+= tests .endif .if ${MK_ZFS} != "no" || defined(ALL_MODULES) SUBDIR+= zfs .endif .if ${MACHINE_CPUARCH} != "aarch64" && ${MACHINE_CPUARCH} != "arm" && \ ${MACHINE_CPUARCH} != "mips" && ${MACHINE_CPUARCH} != "powerpc" && \ ${MACHINE_CPUARCH} != "riscv" _syscons= syscons _vpo= vpo .endif .if ${MACHINE_CPUARCH} != "mips" # no BUS_SPACE_UNSPECIFIED # No barrier instruction support (specific to this driver) _sym= sym # intr_disable() is a macro, causes problems .if ${MK_SOURCELESS_UCODE} != "no" _cxgb= cxgb .endif .endif .if ${MACHINE_CPUARCH} == "aarch64" _em= em _igb= igb .endif .if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64" _agp= agp _an= an _aout= aout _bios= bios _bktr= bktr .if ${MK_SOURCELESS_UCODE} != "no" _bxe= bxe .endif _cardbus= cardbus _cbb= cbb _cpuctl= cpuctl _cpufreq= cpufreq _cs= cs _dpms= dpms _drm= drm _drm2= drm2 _ed= ed _em= em _ena= ena _ep= ep _et= et _exca= exca _fe= fe .if ${MK_OFED} != "no" || defined(ALL_MODULES) _ibcore= ibcore .endif _if_ndis= if_ndis _igb= igb _io= io .if ${MK_OFED} != "no" || defined(ALL_MODULES) _ipoib= ipoib _iser= iser .endif _ix= ix _ixv= ixv _linprocfs= linprocfs _linsysfs= linsysfs _linux= linux .if ${MK_SOURCELESS_UCODE} != "no" _lio= lio .endif _nctgpio= nctgpio _ndis= ndis _pccard= pccard .if ${MK_OFED} != "no" || defined(ALL_MODULES) _rdma= rdma .endif _safe= safe _scsi_low= scsi_low _si= si _speaker= speaker _splash= splash _sppp= sppp _vmware= vmware _vxge= vxge _wbwd= wbwd _wi= wi _xe= xe .if ${MACHINE} != "pc98" _aac= aac _aacraid= aacraid _acpi= acpi .if ${MK_CRYPT} != "no" || defined(ALL_MODULES) _aesni= aesni .endif _amd_ecc_inject=amd_ecc_inject _amdsbwd= amdsbwd _amdtemp= amdtemp _arcmsr= arcmsr _asmc= asmc _bytgpio= bytgpio _ciss= ciss _chromebook_platform= chromebook_platform _cmx= cmx _coretemp= coretemp .if ${MK_SOURCELESS_HOST} != "no" _hpt27xx= hpt27xx .endif _hptiop= hptiop .if ${MK_SOURCELESS_HOST} != "no" _hptmv= hptmv _hptnr= hptnr _hptrr= hptrr .endif _hyperv= hyperv _ichwd= ichwd _ida= ida _iir= iir _intelspi= intelspi _ipmi= ipmi _ips= ips _isci= isci _ipw= ipw _iwi= iwi _iwm= iwm _iwn= iwn _ixgb= ixgb .if ${MK_SOURCELESS_UCODE} != "no" _ipwfw= ipwfw _iwifw= iwifw _iwmfw= iwmfw _iwnfw= iwnfw .endif -.if ${MK_OFED} != "no" || defined(ALL_MODULES) _mlx4= mlx4 -_mlx4ib= mlx4ib -_mlx5ib= mlx5ib -_mlxen= mlxen -.endif _mlx5= mlx5 .if (${MK_INET_SUPPORT} != "no" && ${MK_INET6_SUPPORT} != "no") || \ defined(ALL_MODULES) +_mlx4en= mlx4en _mlx5en= mlx5en +.endif +.if ${MK_OFED} != "no" || defined(ALL_MODULES) +_mlx4ib= mlx4ib +_mlx5ib= mlx5ib .endif _mly= mly .if ${MK_OFED} != "no" || defined(ALL_MODULES) _mthca= mthca .endif _nfe= nfe _nvd= nvd _nvme= nvme _nvram= nvram _nxge= nxge .if ${MK_CRYPT} != "no" || defined(ALL_MODULES) _padlock= padlock _padlock_rng= padlock_rng _rdrand_rng= rdrand_rng .endif _s3= s3 _sdhci_acpi= sdhci_acpi _tpm= tpm _twa= twa _vesa= vesa _viawd= viawd _virtio= virtio _wpi= wpi .if ${MK_SOURCELESS_UCODE} != "no" _wpifw= wpifw .endif _x86bios= x86bios .endif .endif .if ${MACHINE_CPUARCH} == "amd64" _efirt= efirt _ioat= ioat _ixl= ixl _ixlv= ixlv _linux64= linux64 _linux_common= linux_common _ntb= ntb _pms= pms _qlxge= qlxge _qlxgb= qlxgb .if ${MK_SOURCELESS_UCODE} != "no" _qlxgbe= qlxgbe _qlnx= qlnx .endif _sfxge= sfxge .if ${MK_BHYVE} != "no" || defined(ALL_MODULES) _vmm= vmm .endif .endif .if ${MACHINE_CPUARCH} == "i386" # XXX some of these can move to the general case when de-i386'ed # XXX some of these can move now, but are untested on other architectures. _3dfx= 3dfx _3dfx_linux= 3dfx_linux _aic= aic _apm= apm _arcnet= arcnet .if ${MK_SOURCELESS_UCODE} != "no" _ce= ce .endif _coff= coff .if ${MK_SOURCELESS_UCODE} != "no" _cp= cp .endif _elink= elink _glxiic= glxiic _glxsb= glxsb #_ibcs2= ibcs2 _ie= ie _mse= mse _ncr= ncr _ncv= ncv _nsp= nsp _pcfclock= pcfclock _pst= pst _sbni= sbni _streams= streams _stg= stg _svr4= svr4 _wds= wds .if ${MACHINE} == "i386" .if ${MK_EISA} != "no" _ahb= ahb .endif _cm= cm .if ${MK_SOURCELESS_UCODE} != "no" _ctau= ctau .endif _dpt= dpt _ex= ex _wl= wl .elif ${MACHINE} == "pc98" _canbepm= canbepm _canbus= canbus _ct= ct _pmc= pmc _snc= snc .endif .endif .if ${MACHINE_CPUARCH} == "arm" _cfi= cfi _cpsw= cpsw .endif .if ${MACHINE_CPUARCH} == "powerpc" _agp= agp _an= an _bm= bm _cardbus= cardbus _cbb= cbb _cfi= cfi _cpufreq= cpufreq _drm= drm _exca= exca _nvram= powermac_nvram _ffec= ffec _pccard= pccard _wi= wi .endif .if ${MACHINE_ARCH} == "powerpc64" _drm2= drm2 .endif .if ${MACHINE_CPUARCH} == "sparc64" _auxio= auxio _em= em _epic= epic _igb= igb .endif .if (${MACHINE_CPUARCH} == "amd64" || ${MACHINE_ARCH} == "armv6" || \ ${MACHINE_CPUARCH} == "i386") _cloudabi32= cloudabi32 .endif .if ${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "amd64" _cloudabi64= cloudabi64 .endif .endif .if ${MACHINE_ARCH} == "armv6" _ffec= ffec .endif SUBDIR+=${MODULES_EXTRA} .for reject in ${WITHOUT_MODULES} SUBDIR:= ${SUBDIR:N${reject}} .endfor # Calling kldxref(8) for each module is expensive. .if !defined(NO_XREF) .MAKEFLAGS+= -DNO_XREF afterinstall: .PHONY @if type kldxref >/dev/null 2>&1; then \ ${ECHO} kldxref ${DESTDIR}${KMODDIR}; \ kldxref ${DESTDIR}${KMODDIR}; \ fi .endif .include "${SYSDIR}/conf/config.mk" SUBDIR:= ${SUBDIR:u:O} .include Index: stable/11/sys/modules/mlx4/Makefile =================================================================== --- stable/11/sys/modules/mlx4/Makefile (revision 329158) +++ stable/11/sys/modules/mlx4/Makefile (revision 329159) @@ -1,20 +1,33 @@ # $FreeBSD$ -.PATH: ${SRCTOP}/sys/ofed/drivers/net/mlx4 +.PATH: ${SRCTOP}/sys/dev/mlx4/mlx4_core KMOD= mlx4 SRCS= device_if.h bus_if.h vnode_if.h pci_if.h \ opt_inet.h opt_inet6.h \ - alloc.c catas.c cmd.c cq.c eq.c \ - fw.c icm.c intf.c main.c mcg.c mr.c \ - pd.c port.c profile.c qp.c reset.c sense.c \ - srq.c resource_tracker.c sys_tune.c + mlx4_alloc.c \ + mlx4_catas.c \ + mlx4_cmd.c \ + mlx4_cq.c \ + mlx4_eq.c \ + mlx4_fw.c \ + mlx4_fw_qos.c \ + mlx4_icm.c \ + mlx4_intf.c \ + mlx4_main.c \ + mlx4_mcg.c \ + mlx4_mr.c \ + mlx4_pd.c \ + mlx4_port.c \ + mlx4_profile.c \ + mlx4_qp.c \ + mlx4_reset.c \ + mlx4_sense.c \ + mlx4_srq.c \ + mlx4_resource_tracker.c -CFLAGS+= -I${SRCTOP}/sys/ofed/drivers/net/mlx4 CFLAGS+= -I${SRCTOP}/sys/ofed/include CFLAGS+= -I${SRCTOP}/sys/compat/linuxkpi/common/include .include CFLAGS+= -Wno-cast-qual -Wno-pointer-arith - -CWARNFLAGS.mcg.c= -Wno-unused Index: stable/11/sys/modules/mlx4en/Makefile =================================================================== --- stable/11/sys/modules/mlx4en/Makefile (nonexistent) +++ stable/11/sys/modules/mlx4en/Makefile (revision 329159) @@ -0,0 +1,18 @@ +# $FreeBSD$ +.PATH: ${SRCTOP}/sys/dev/mlx4/mlx4_en + +KMOD= mlx4en +SRCS= device_if.h bus_if.h pci_if.h vnode_if.h \ + opt_inet.h opt_inet6.h \ + mlx4_en_cq.c \ + mlx4_en_main.c \ + mlx4_en_netdev.c \ + mlx4_en_port.c \ + mlx4_en_resources.c \ + mlx4_en_rx.c \ + mlx4_en_tx.c + +CFLAGS+= -I${SRCTOP}/sys/ofed/include +CFLAGS+= -I${SRCTOP}/sys/compat/linuxkpi/common/include + +.include Property changes on: stable/11/sys/modules/mlx4en/Makefile ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: stable/11/sys/modules/mlx4ib/Makefile =================================================================== --- stable/11/sys/modules/mlx4ib/Makefile (revision 329158) +++ stable/11/sys/modules/mlx4ib/Makefile (revision 329159) @@ -1,19 +1,29 @@ # $FreeBSD$ -.PATH: ${SRCTOP}/sys/ofed/drivers/infiniband/hw/mlx4 +.PATH: ${SRCTOP}/sys/dev/mlx4/mlx4_ib KMOD= mlx4ib SRCS= device_if.h bus_if.h vnode_if.h pci_if.h \ opt_inet.h opt_inet6.h \ - alias_GUID.c mcg.c sysfs.c ah.c cq.c \ - mlx4_exp.c \ - doorbell.c mad.c main.c mr.c qp.c srq.c wc.c cm.c + mlx4_ib_alias_GUID.c \ + mlx4_ib_mcg.c \ + mlx4_ib_sysfs.c \ + mlx4_ib_ah.c \ + mlx4_ib_cq.c \ + mlx4_ib_exp.c \ + mlx4_ib_doorbell.c \ + mlx4_ib_mad.c \ + mlx4_ib_main.c \ + mlx4_ib_mr.c \ + mlx4_ib_qp.c \ + mlx4_ib_srq.c \ + mlx4_ib_wc.c \ + mlx4_ib_cm.c -CFLAGS+= -I${SRCTOP}/sys/ofed/drivers/infiniband/hw/mlx4 CFLAGS+= -I${SRCTOP}/sys/ofed/include CFLAGS+= -I${SRCTOP}/sys/compat/linuxkpi/common/include CFLAGS+= -DCONFIG_INFINIBAND_USER_MEM CFLAGS+= -DINET6 -DINET .include CFLAGS+= -Wno-cast-qual -Wno-pointer-arith Index: stable/11/sys/ofed/drivers/net/mlx4/en_port.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/en_port.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/en_port.c (nonexistent) @@ -1,635 +0,0 @@ -/* - * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include -#include - -#include -#include - -#include "en_port.h" -#include "mlx4_en.h" -#define EN_IFQ_MIN_INTERVAL 3000 - - -int mlx4_SET_VLAN_FLTR(struct mlx4_dev *dev, struct mlx4_en_priv *priv) -{ - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_set_vlan_fltr_mbox *filter; - int i; - int j; - int index = 0; - u32 entry; - int err = 0; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - filter = mailbox->buf; - memset(filter, 0, sizeof(*filter)); - for (i = VLAN_FLTR_SIZE - 1; i >= 0; i--) { - entry = 0; - for (j = 0; j < 32; j++) { - if (test_bit(index, priv->active_vlans)) - entry |= 1 << j; - index++; - } - filter->entry[i] = cpu_to_be32(entry); - } - err = mlx4_cmd(dev, mailbox->dma, priv->port, 0, MLX4_CMD_SET_VLAN_FLTR, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - -int mlx4_en_QUERY_PORT(struct mlx4_en_dev *mdev, u8 port) -{ - struct mlx4_en_query_port_context *qport_context; - struct mlx4_en_priv *priv = netdev_priv(mdev->pndev[port]); - struct mlx4_en_port_state *state = &priv->port_state; - struct mlx4_cmd_mailbox *mailbox; - int err; - - mailbox = mlx4_alloc_cmd_mailbox(mdev->dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - memset(mailbox->buf, 0, sizeof(*qport_context)); - err = mlx4_cmd_box(mdev->dev, 0, mailbox->dma, port, 0, - MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_WRAPPED); - if (err) - goto out; - qport_context = mailbox->buf; - - /* This command is always accessed from Ethtool context - * already synchronized, no need in locking */ - state->link_state = !!(qport_context->link_up & MLX4_EN_LINK_UP_MASK); - switch (qport_context->link_speed & MLX4_EN_SPEED_MASK) { - case MLX4_EN_1G_SPEED: - state->link_speed = 1000; - break; - case MLX4_EN_10G_SPEED_XAUI: - case MLX4_EN_10G_SPEED_XFI: - state->link_speed = 10000; - break; - case MLX4_EN_20G_SPEED: - state->link_speed = 20000; - break; - case MLX4_EN_40G_SPEED: - state->link_speed = 40000; - break; - case MLX4_EN_56G_SPEED: - state->link_speed = 56000; - break; - default: - state->link_speed = -1; - break; - } - state->transciver = qport_context->transceiver; - state->autoneg = !!(qport_context->autoneg & MLX4_EN_AUTONEG_MASK); - -out: - mlx4_free_cmd_mailbox(mdev->dev, mailbox); - return err; -} - -static void mlx4_en_fold_software_stats(struct net_device *dev) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - u64 packets, bytes; - int i; - - if (!priv->port_up || mlx4_is_master(mdev->dev)) - return; - - packets = 0; - bytes = 0; - for (i = 0; i < priv->rx_ring_num; i++) { - const struct mlx4_en_rx_ring *ring = priv->rx_ring[i]; - - packets += READ_ONCE(ring->packets); - bytes += READ_ONCE(ring->bytes); - } - priv->pkstats.rx_packets = packets; - priv->pkstats.rx_bytes = bytes; - - packets = 0; - bytes = 0; - for (i = 0; i < priv->tx_ring_num; i++) { - const struct mlx4_en_tx_ring *ring = priv->tx_ring[i]; - - packets += READ_ONCE(ring->packets); - bytes += READ_ONCE(ring->bytes); - } - priv->pkstats.tx_packets = packets; - priv->pkstats.tx_bytes = bytes; -} - -int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset) -{ - struct mlx4_en_stat_out_mbox *mlx4_en_stats; - struct mlx4_en_stat_out_flow_control_mbox *flowstats; - struct net_device *dev = mdev->pndev[port]; - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_vport_stats *vport_stats = &priv->vport_stats; - struct mlx4_cmd_mailbox *mailbox = NULL; - struct mlx4_cmd_mailbox *mailbox_flow = NULL; - u64 in_mod = reset << 8 | port; - int err; - int i; - int do_if_stat = 1; - unsigned long period = (unsigned long) (jiffies - priv->last_ifq_jiffies); - struct mlx4_en_vport_stats tmp_vport_stats; - - if (jiffies_to_msecs(period) < EN_IFQ_MIN_INTERVAL || - priv->counter_index == 0xff) - do_if_stat = 0; - - mailbox = mlx4_alloc_cmd_mailbox(mdev->dev); - if (IS_ERR(mailbox)) { - err = PTR_ERR(mailbox); - goto mailbox_out; - } - - mailbox_flow = mlx4_alloc_cmd_mailbox(mdev->dev); - if (IS_ERR(mailbox_flow)) { - mlx4_free_cmd_mailbox(mdev->dev, mailbox); - err = PTR_ERR(mailbox_flow); - goto mailbox_out; - } - - /* 0xffs indicates invalid value */ - memset(mailbox_flow->buf, 0xff, sizeof(*flowstats) * - MLX4_NUM_PRIORITIES); - - if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) { - memset(mailbox_flow->buf, 0, sizeof(*flowstats)); - err = mlx4_cmd_box(mdev->dev, 0, mailbox_flow->dma, - in_mod | 1<<12, 0, MLX4_CMD_DUMP_ETH_STATS, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); - - if (err) - goto out; - } - - err = mlx4_cmd_box(mdev->dev, 0, mailbox->dma, in_mod, 0, - MLX4_CMD_DUMP_ETH_STATS, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_NATIVE); - if (err) - goto out; - - mlx4_en_stats = mailbox->buf; - - spin_lock(&priv->stats_lock); - - priv->port_stats.rx_chksum_good = 0; - priv->port_stats.rx_chksum_none = 0; - for (i = 0; i < priv->rx_ring_num; i++) { - priv->port_stats.rx_chksum_good += priv->rx_ring[i]->csum_ok; - priv->port_stats.rx_chksum_none += priv->rx_ring[i]->csum_none; - } - - priv->port_stats.tx_chksum_offload = 0; - priv->port_stats.queue_stopped = 0; - priv->port_stats.wake_queue = 0; - priv->port_stats.oversized_packets = 0; - priv->port_stats.tso_packets = 0; - priv->port_stats.defrag_attempts = 0; - for (i = 0; i < priv->tx_ring_num; i++) { - priv->port_stats.tx_chksum_offload += priv->tx_ring[i]->tx_csum; - priv->port_stats.queue_stopped += priv->tx_ring[i]->queue_stopped; - priv->port_stats.wake_queue += priv->tx_ring[i]->wake_queue; - priv->port_stats.oversized_packets += priv->tx_ring[i]->oversized_packets; - priv->port_stats.tso_packets += priv->tx_ring[i]->tso_packets; - priv->port_stats.defrag_attempts += priv->tx_ring[i]->defrag_attempts; - } - /* RX Statistics */ - priv->pkstats.rx_packets = be64_to_cpu(mlx4_en_stats->RTOT_prio_0) + - be64_to_cpu(mlx4_en_stats->RTOT_prio_1) + - be64_to_cpu(mlx4_en_stats->RTOT_prio_2) + - be64_to_cpu(mlx4_en_stats->RTOT_prio_3) + - be64_to_cpu(mlx4_en_stats->RTOT_prio_4) + - be64_to_cpu(mlx4_en_stats->RTOT_prio_5) + - be64_to_cpu(mlx4_en_stats->RTOT_prio_6) + - be64_to_cpu(mlx4_en_stats->RTOT_prio_7) + - be64_to_cpu(mlx4_en_stats->RTOT_novlan); - priv->pkstats.rx_bytes = be64_to_cpu(mlx4_en_stats->ROCT_prio_0) + - be64_to_cpu(mlx4_en_stats->ROCT_prio_1) + - be64_to_cpu(mlx4_en_stats->ROCT_prio_2) + - be64_to_cpu(mlx4_en_stats->ROCT_prio_3) + - be64_to_cpu(mlx4_en_stats->ROCT_prio_4) + - be64_to_cpu(mlx4_en_stats->ROCT_prio_5) + - be64_to_cpu(mlx4_en_stats->ROCT_prio_6) + - be64_to_cpu(mlx4_en_stats->ROCT_prio_7) + - be64_to_cpu(mlx4_en_stats->ROCT_novlan); - priv->pkstats.rx_multicast_packets = be64_to_cpu(mlx4_en_stats->MCAST_prio_0) + - be64_to_cpu(mlx4_en_stats->MCAST_prio_1) + - be64_to_cpu(mlx4_en_stats->MCAST_prio_2) + - be64_to_cpu(mlx4_en_stats->MCAST_prio_3) + - be64_to_cpu(mlx4_en_stats->MCAST_prio_4) + - be64_to_cpu(mlx4_en_stats->MCAST_prio_5) + - be64_to_cpu(mlx4_en_stats->MCAST_prio_6) + - be64_to_cpu(mlx4_en_stats->MCAST_prio_7) + - be64_to_cpu(mlx4_en_stats->MCAST_novlan); - priv->pkstats.rx_broadcast_packets = be64_to_cpu(mlx4_en_stats->RBCAST_prio_0) + - be64_to_cpu(mlx4_en_stats->RBCAST_prio_1) + - be64_to_cpu(mlx4_en_stats->RBCAST_prio_2) + - be64_to_cpu(mlx4_en_stats->RBCAST_prio_3) + - be64_to_cpu(mlx4_en_stats->RBCAST_prio_4) + - be64_to_cpu(mlx4_en_stats->RBCAST_prio_5) + - be64_to_cpu(mlx4_en_stats->RBCAST_prio_6) + - be64_to_cpu(mlx4_en_stats->RBCAST_prio_7) + - be64_to_cpu(mlx4_en_stats->RBCAST_novlan); - priv->pkstats.rx_errors = be64_to_cpu(mlx4_en_stats->PCS) + - be32_to_cpu(mlx4_en_stats->RJBBR) + - be32_to_cpu(mlx4_en_stats->RCRC) + - be32_to_cpu(mlx4_en_stats->RRUNT) + - be64_to_cpu(mlx4_en_stats->RInRangeLengthErr) + - be64_to_cpu(mlx4_en_stats->ROutRangeLengthErr) + - be32_to_cpu(mlx4_en_stats->RSHORT) + - be64_to_cpu(mlx4_en_stats->RGIANT_prio_0) + - be64_to_cpu(mlx4_en_stats->RGIANT_prio_1) + - be64_to_cpu(mlx4_en_stats->RGIANT_prio_2) + - be64_to_cpu(mlx4_en_stats->RGIANT_prio_3) + - be64_to_cpu(mlx4_en_stats->RGIANT_prio_4) + - be64_to_cpu(mlx4_en_stats->RGIANT_prio_5) + - be64_to_cpu(mlx4_en_stats->RGIANT_prio_6) + - be64_to_cpu(mlx4_en_stats->RGIANT_prio_7) + - be64_to_cpu(mlx4_en_stats->RGIANT_novlan); - priv->pkstats.rx_dropped = be32_to_cpu(mlx4_en_stats->RdropOvflw); - priv->pkstats.rx_length_errors = be32_to_cpu(mlx4_en_stats->RdropLength); - priv->pkstats.rx_over_errors = be32_to_cpu(mlx4_en_stats->RdropOvflw); - priv->pkstats.rx_crc_errors = be32_to_cpu(mlx4_en_stats->RCRC); - priv->pkstats.rx_jabbers = be32_to_cpu(mlx4_en_stats->RJBBR); - priv->pkstats.rx_in_range_length_error = be64_to_cpu(mlx4_en_stats->RInRangeLengthErr); - priv->pkstats.rx_out_range_length_error = be64_to_cpu(mlx4_en_stats->ROutRangeLengthErr); - priv->pkstats.rx_lt_64_bytes_packets = be64_to_cpu(mlx4_en_stats->R64_prio_0) + - be64_to_cpu(mlx4_en_stats->R64_prio_1) + - be64_to_cpu(mlx4_en_stats->R64_prio_2) + - be64_to_cpu(mlx4_en_stats->R64_prio_3) + - be64_to_cpu(mlx4_en_stats->R64_prio_4) + - be64_to_cpu(mlx4_en_stats->R64_prio_5) + - be64_to_cpu(mlx4_en_stats->R64_prio_6) + - be64_to_cpu(mlx4_en_stats->R64_prio_7) + - be64_to_cpu(mlx4_en_stats->R64_novlan); - priv->pkstats.rx_127_bytes_packets = be64_to_cpu(mlx4_en_stats->R127_prio_0) + - be64_to_cpu(mlx4_en_stats->R127_prio_1) + - be64_to_cpu(mlx4_en_stats->R127_prio_2) + - be64_to_cpu(mlx4_en_stats->R127_prio_3) + - be64_to_cpu(mlx4_en_stats->R127_prio_4) + - be64_to_cpu(mlx4_en_stats->R127_prio_5) + - be64_to_cpu(mlx4_en_stats->R127_prio_6) + - be64_to_cpu(mlx4_en_stats->R127_prio_7) + - be64_to_cpu(mlx4_en_stats->R127_novlan); - priv->pkstats.rx_255_bytes_packets = be64_to_cpu(mlx4_en_stats->R255_prio_0) + - be64_to_cpu(mlx4_en_stats->R255_prio_1) + - be64_to_cpu(mlx4_en_stats->R255_prio_2) + - be64_to_cpu(mlx4_en_stats->R255_prio_3) + - be64_to_cpu(mlx4_en_stats->R255_prio_4) + - be64_to_cpu(mlx4_en_stats->R255_prio_5) + - be64_to_cpu(mlx4_en_stats->R255_prio_6) + - be64_to_cpu(mlx4_en_stats->R255_prio_7) + - be64_to_cpu(mlx4_en_stats->R255_novlan); - priv->pkstats.rx_511_bytes_packets = be64_to_cpu(mlx4_en_stats->R511_prio_0) + - be64_to_cpu(mlx4_en_stats->R511_prio_1) + - be64_to_cpu(mlx4_en_stats->R511_prio_2) + - be64_to_cpu(mlx4_en_stats->R511_prio_3) + - be64_to_cpu(mlx4_en_stats->R511_prio_4) + - be64_to_cpu(mlx4_en_stats->R511_prio_5) + - be64_to_cpu(mlx4_en_stats->R511_prio_6) + - be64_to_cpu(mlx4_en_stats->R511_prio_7) + - be64_to_cpu(mlx4_en_stats->R511_novlan); - priv->pkstats.rx_1023_bytes_packets = be64_to_cpu(mlx4_en_stats->R1023_prio_0) + - be64_to_cpu(mlx4_en_stats->R1023_prio_1) + - be64_to_cpu(mlx4_en_stats->R1023_prio_2) + - be64_to_cpu(mlx4_en_stats->R1023_prio_3) + - be64_to_cpu(mlx4_en_stats->R1023_prio_4) + - be64_to_cpu(mlx4_en_stats->R1023_prio_5) + - be64_to_cpu(mlx4_en_stats->R1023_prio_6) + - be64_to_cpu(mlx4_en_stats->R1023_prio_7) + - be64_to_cpu(mlx4_en_stats->R1023_novlan); - priv->pkstats.rx_1518_bytes_packets = be64_to_cpu(mlx4_en_stats->R1518_prio_0) + - be64_to_cpu(mlx4_en_stats->R1518_prio_1) + - be64_to_cpu(mlx4_en_stats->R1518_prio_2) + - be64_to_cpu(mlx4_en_stats->R1518_prio_3) + - be64_to_cpu(mlx4_en_stats->R1518_prio_4) + - be64_to_cpu(mlx4_en_stats->R1518_prio_5) + - be64_to_cpu(mlx4_en_stats->R1518_prio_6) + - be64_to_cpu(mlx4_en_stats->R1518_prio_7) + - be64_to_cpu(mlx4_en_stats->R1518_novlan); - priv->pkstats.rx_1522_bytes_packets = be64_to_cpu(mlx4_en_stats->R1522_prio_0) + - be64_to_cpu(mlx4_en_stats->R1522_prio_1) + - be64_to_cpu(mlx4_en_stats->R1522_prio_2) + - be64_to_cpu(mlx4_en_stats->R1522_prio_3) + - be64_to_cpu(mlx4_en_stats->R1522_prio_4) + - be64_to_cpu(mlx4_en_stats->R1522_prio_5) + - be64_to_cpu(mlx4_en_stats->R1522_prio_6) + - be64_to_cpu(mlx4_en_stats->R1522_prio_7) + - be64_to_cpu(mlx4_en_stats->R1522_novlan); - priv->pkstats.rx_1548_bytes_packets = be64_to_cpu(mlx4_en_stats->R1548_prio_0) + - be64_to_cpu(mlx4_en_stats->R1548_prio_1) + - be64_to_cpu(mlx4_en_stats->R1548_prio_2) + - be64_to_cpu(mlx4_en_stats->R1548_prio_3) + - be64_to_cpu(mlx4_en_stats->R1548_prio_4) + - be64_to_cpu(mlx4_en_stats->R1548_prio_5) + - be64_to_cpu(mlx4_en_stats->R1548_prio_6) + - be64_to_cpu(mlx4_en_stats->R1548_prio_7) + - be64_to_cpu(mlx4_en_stats->R1548_novlan); - priv->pkstats.rx_gt_1548_bytes_packets = be64_to_cpu(mlx4_en_stats->R2MTU_prio_0) + - be64_to_cpu(mlx4_en_stats->R2MTU_prio_1) + - be64_to_cpu(mlx4_en_stats->R2MTU_prio_2) + - be64_to_cpu(mlx4_en_stats->R2MTU_prio_3) + - be64_to_cpu(mlx4_en_stats->R2MTU_prio_4) + - be64_to_cpu(mlx4_en_stats->R2MTU_prio_5) + - be64_to_cpu(mlx4_en_stats->R2MTU_prio_6) + - be64_to_cpu(mlx4_en_stats->R2MTU_prio_7) + - be64_to_cpu(mlx4_en_stats->R2MTU_novlan); - - /* Tx Stats */ - priv->pkstats.tx_packets = be64_to_cpu(mlx4_en_stats->TTOT_prio_0) + - be64_to_cpu(mlx4_en_stats->TTOT_prio_1) + - be64_to_cpu(mlx4_en_stats->TTOT_prio_2) + - be64_to_cpu(mlx4_en_stats->TTOT_prio_3) + - be64_to_cpu(mlx4_en_stats->TTOT_prio_4) + - be64_to_cpu(mlx4_en_stats->TTOT_prio_5) + - be64_to_cpu(mlx4_en_stats->TTOT_prio_6) + - be64_to_cpu(mlx4_en_stats->TTOT_prio_7) + - be64_to_cpu(mlx4_en_stats->TTOT_novlan); - priv->pkstats.tx_bytes = be64_to_cpu(mlx4_en_stats->TOCT_prio_0) + - be64_to_cpu(mlx4_en_stats->TOCT_prio_1) + - be64_to_cpu(mlx4_en_stats->TOCT_prio_2) + - be64_to_cpu(mlx4_en_stats->TOCT_prio_3) + - be64_to_cpu(mlx4_en_stats->TOCT_prio_4) + - be64_to_cpu(mlx4_en_stats->TOCT_prio_5) + - be64_to_cpu(mlx4_en_stats->TOCT_prio_6) + - be64_to_cpu(mlx4_en_stats->TOCT_prio_7) + - be64_to_cpu(mlx4_en_stats->TOCT_novlan); - priv->pkstats.tx_multicast_packets = be64_to_cpu(mlx4_en_stats->TMCAST_prio_0) + - be64_to_cpu(mlx4_en_stats->TMCAST_prio_1) + - be64_to_cpu(mlx4_en_stats->TMCAST_prio_2) + - be64_to_cpu(mlx4_en_stats->TMCAST_prio_3) + - be64_to_cpu(mlx4_en_stats->TMCAST_prio_4) + - be64_to_cpu(mlx4_en_stats->TMCAST_prio_5) + - be64_to_cpu(mlx4_en_stats->TMCAST_prio_6) + - be64_to_cpu(mlx4_en_stats->TMCAST_prio_7) + - be64_to_cpu(mlx4_en_stats->TMCAST_novlan); - priv->pkstats.tx_broadcast_packets = be64_to_cpu(mlx4_en_stats->TBCAST_prio_0) + - be64_to_cpu(mlx4_en_stats->TBCAST_prio_1) + - be64_to_cpu(mlx4_en_stats->TBCAST_prio_2) + - be64_to_cpu(mlx4_en_stats->TBCAST_prio_3) + - be64_to_cpu(mlx4_en_stats->TBCAST_prio_4) + - be64_to_cpu(mlx4_en_stats->TBCAST_prio_5) + - be64_to_cpu(mlx4_en_stats->TBCAST_prio_6) + - be64_to_cpu(mlx4_en_stats->TBCAST_prio_7) + - be64_to_cpu(mlx4_en_stats->TBCAST_novlan); - priv->pkstats.tx_errors = be64_to_cpu(mlx4_en_stats->TGIANT_prio_0) + - be64_to_cpu(mlx4_en_stats->TGIANT_prio_1) + - be64_to_cpu(mlx4_en_stats->TGIANT_prio_2) + - be64_to_cpu(mlx4_en_stats->TGIANT_prio_3) + - be64_to_cpu(mlx4_en_stats->TGIANT_prio_4) + - be64_to_cpu(mlx4_en_stats->TGIANT_prio_5) + - be64_to_cpu(mlx4_en_stats->TGIANT_prio_6) + - be64_to_cpu(mlx4_en_stats->TGIANT_prio_7) + - be64_to_cpu(mlx4_en_stats->TGIANT_novlan); - priv->pkstats.tx_dropped = be32_to_cpu(mlx4_en_stats->TDROP) - - priv->pkstats.tx_errors; - priv->pkstats.tx_lt_64_bytes_packets = be64_to_cpu(mlx4_en_stats->T64_prio_0) + - be64_to_cpu(mlx4_en_stats->T64_prio_1) + - be64_to_cpu(mlx4_en_stats->T64_prio_2) + - be64_to_cpu(mlx4_en_stats->T64_prio_3) + - be64_to_cpu(mlx4_en_stats->T64_prio_4) + - be64_to_cpu(mlx4_en_stats->T64_prio_5) + - be64_to_cpu(mlx4_en_stats->T64_prio_6) + - be64_to_cpu(mlx4_en_stats->T64_prio_7) + - be64_to_cpu(mlx4_en_stats->T64_novlan); - priv->pkstats.tx_127_bytes_packets = be64_to_cpu(mlx4_en_stats->T127_prio_0) + - be64_to_cpu(mlx4_en_stats->T127_prio_1) + - be64_to_cpu(mlx4_en_stats->T127_prio_2) + - be64_to_cpu(mlx4_en_stats->T127_prio_3) + - be64_to_cpu(mlx4_en_stats->T127_prio_4) + - be64_to_cpu(mlx4_en_stats->T127_prio_5) + - be64_to_cpu(mlx4_en_stats->T127_prio_6) + - be64_to_cpu(mlx4_en_stats->T127_prio_7) + - be64_to_cpu(mlx4_en_stats->T127_novlan); - priv->pkstats.tx_255_bytes_packets = be64_to_cpu(mlx4_en_stats->T255_prio_0) + - be64_to_cpu(mlx4_en_stats->T255_prio_1) + - be64_to_cpu(mlx4_en_stats->T255_prio_2) + - be64_to_cpu(mlx4_en_stats->T255_prio_3) + - be64_to_cpu(mlx4_en_stats->T255_prio_4) + - be64_to_cpu(mlx4_en_stats->T255_prio_5) + - be64_to_cpu(mlx4_en_stats->T255_prio_6) + - be64_to_cpu(mlx4_en_stats->T255_prio_7) + - be64_to_cpu(mlx4_en_stats->T255_novlan); - priv->pkstats.tx_511_bytes_packets = be64_to_cpu(mlx4_en_stats->T511_prio_0) + - be64_to_cpu(mlx4_en_stats->T511_prio_1) + - be64_to_cpu(mlx4_en_stats->T511_prio_2) + - be64_to_cpu(mlx4_en_stats->T511_prio_3) + - be64_to_cpu(mlx4_en_stats->T511_prio_4) + - be64_to_cpu(mlx4_en_stats->T511_prio_5) + - be64_to_cpu(mlx4_en_stats->T511_prio_6) + - be64_to_cpu(mlx4_en_stats->T511_prio_7) + - be64_to_cpu(mlx4_en_stats->T511_novlan); - priv->pkstats.tx_1023_bytes_packets = be64_to_cpu(mlx4_en_stats->T1023_prio_0) + - be64_to_cpu(mlx4_en_stats->T1023_prio_1) + - be64_to_cpu(mlx4_en_stats->T1023_prio_2) + - be64_to_cpu(mlx4_en_stats->T1023_prio_3) + - be64_to_cpu(mlx4_en_stats->T1023_prio_4) + - be64_to_cpu(mlx4_en_stats->T1023_prio_5) + - be64_to_cpu(mlx4_en_stats->T1023_prio_6) + - be64_to_cpu(mlx4_en_stats->T1023_prio_7) + - be64_to_cpu(mlx4_en_stats->T1023_novlan); - priv->pkstats.tx_1518_bytes_packets = be64_to_cpu(mlx4_en_stats->T1518_prio_0) + - be64_to_cpu(mlx4_en_stats->T1518_prio_1) + - be64_to_cpu(mlx4_en_stats->T1518_prio_2) + - be64_to_cpu(mlx4_en_stats->T1518_prio_3) + - be64_to_cpu(mlx4_en_stats->T1518_prio_4) + - be64_to_cpu(mlx4_en_stats->T1518_prio_5) + - be64_to_cpu(mlx4_en_stats->T1518_prio_6) + - be64_to_cpu(mlx4_en_stats->T1518_prio_7) + - be64_to_cpu(mlx4_en_stats->T1518_novlan); - priv->pkstats.tx_1522_bytes_packets = be64_to_cpu(mlx4_en_stats->T1522_prio_0) + - be64_to_cpu(mlx4_en_stats->T1522_prio_1) + - be64_to_cpu(mlx4_en_stats->T1522_prio_2) + - be64_to_cpu(mlx4_en_stats->T1522_prio_3) + - be64_to_cpu(mlx4_en_stats->T1522_prio_4) + - be64_to_cpu(mlx4_en_stats->T1522_prio_5) + - be64_to_cpu(mlx4_en_stats->T1522_prio_6) + - be64_to_cpu(mlx4_en_stats->T1522_prio_7) + - be64_to_cpu(mlx4_en_stats->T1522_novlan); - priv->pkstats.tx_1548_bytes_packets = be64_to_cpu(mlx4_en_stats->T1548_prio_0) + - be64_to_cpu(mlx4_en_stats->T1548_prio_1) + - be64_to_cpu(mlx4_en_stats->T1548_prio_2) + - be64_to_cpu(mlx4_en_stats->T1548_prio_3) + - be64_to_cpu(mlx4_en_stats->T1548_prio_4) + - be64_to_cpu(mlx4_en_stats->T1548_prio_5) + - be64_to_cpu(mlx4_en_stats->T1548_prio_6) + - be64_to_cpu(mlx4_en_stats->T1548_prio_7) + - be64_to_cpu(mlx4_en_stats->T1548_novlan); - priv->pkstats.tx_gt_1548_bytes_packets = be64_to_cpu(mlx4_en_stats->T2MTU_prio_0) + - be64_to_cpu(mlx4_en_stats->T2MTU_prio_1) + - be64_to_cpu(mlx4_en_stats->T2MTU_prio_2) + - be64_to_cpu(mlx4_en_stats->T2MTU_prio_3) + - be64_to_cpu(mlx4_en_stats->T2MTU_prio_4) + - be64_to_cpu(mlx4_en_stats->T2MTU_prio_5) + - be64_to_cpu(mlx4_en_stats->T2MTU_prio_6) + - be64_to_cpu(mlx4_en_stats->T2MTU_prio_7) + - be64_to_cpu(mlx4_en_stats->T2MTU_novlan); - - priv->pkstats.rx_prio[0][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_0); - priv->pkstats.rx_prio[0][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_0); - priv->pkstats.rx_prio[1][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_1); - priv->pkstats.rx_prio[1][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_1); - priv->pkstats.rx_prio[2][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_2); - priv->pkstats.rx_prio[2][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_2); - priv->pkstats.rx_prio[3][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_3); - priv->pkstats.rx_prio[3][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_3); - priv->pkstats.rx_prio[4][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_4); - priv->pkstats.rx_prio[4][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_4); - priv->pkstats.rx_prio[5][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_5); - priv->pkstats.rx_prio[5][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_5); - priv->pkstats.rx_prio[6][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_6); - priv->pkstats.rx_prio[6][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_6); - priv->pkstats.rx_prio[7][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_7); - priv->pkstats.rx_prio[7][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_7); - priv->pkstats.rx_prio[8][0] = be64_to_cpu(mlx4_en_stats->RTOT_novlan); - priv->pkstats.rx_prio[8][1] = be64_to_cpu(mlx4_en_stats->ROCT_novlan); - priv->pkstats.tx_prio[0][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_0); - priv->pkstats.tx_prio[0][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_0); - priv->pkstats.tx_prio[1][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_1); - priv->pkstats.tx_prio[1][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_1); - priv->pkstats.tx_prio[2][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_2); - priv->pkstats.tx_prio[2][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_2); - priv->pkstats.tx_prio[3][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_3); - priv->pkstats.tx_prio[3][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_3); - priv->pkstats.tx_prio[4][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_4); - priv->pkstats.tx_prio[4][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_4); - priv->pkstats.tx_prio[5][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_5); - priv->pkstats.tx_prio[5][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_5); - priv->pkstats.tx_prio[6][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_6); - priv->pkstats.tx_prio[6][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_6); - priv->pkstats.tx_prio[7][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_7); - priv->pkstats.tx_prio[7][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_7); - priv->pkstats.tx_prio[8][0] = be64_to_cpu(mlx4_en_stats->TTOT_novlan); - priv->pkstats.tx_prio[8][1] = be64_to_cpu(mlx4_en_stats->TOCT_novlan); - - flowstats = mailbox_flow->buf; - - for (i = 0; i < MLX4_NUM_PRIORITIES; i++) { - priv->flowstats[i].rx_pause = - be64_to_cpu(flowstats[i].rx_pause); - priv->flowstats[i].rx_pause_duration = - be64_to_cpu(flowstats[i].rx_pause_duration); - priv->flowstats[i].rx_pause_transition = - be64_to_cpu(flowstats[i].rx_pause_transition); - priv->flowstats[i].tx_pause = - be64_to_cpu(flowstats[i].tx_pause); - priv->flowstats[i].tx_pause_duration = - be64_to_cpu(flowstats[i].tx_pause_duration); - priv->flowstats[i].tx_pause_transition = - be64_to_cpu(flowstats[i].tx_pause_transition); - } - - mlx4_en_fold_software_stats(dev); - - spin_unlock(&priv->stats_lock); - - memset(&tmp_vport_stats, 0, sizeof(tmp_vport_stats)); - - err = mlx4_get_vport_ethtool_stats(mdev->dev, port, - &tmp_vport_stats, reset); - spin_lock(&priv->stats_lock); - if (!err) { - /* ethtool stats format */ - vport_stats->rx_unicast_packets = tmp_vport_stats.rx_unicast_packets; - vport_stats->rx_unicast_bytes = tmp_vport_stats.rx_unicast_bytes; - vport_stats->rx_multicast_packets = tmp_vport_stats.rx_multicast_packets; - vport_stats->rx_multicast_bytes = tmp_vport_stats.rx_multicast_bytes; - vport_stats->rx_broadcast_packets = tmp_vport_stats.rx_broadcast_packets; - vport_stats->rx_broadcast_bytes = tmp_vport_stats.rx_broadcast_bytes; - vport_stats->rx_dropped = tmp_vport_stats.rx_dropped; - vport_stats->rx_errors = tmp_vport_stats.rx_errors; - vport_stats->tx_unicast_packets = tmp_vport_stats.tx_unicast_packets; - vport_stats->tx_unicast_bytes = tmp_vport_stats.tx_unicast_bytes; - vport_stats->tx_multicast_packets = tmp_vport_stats.tx_multicast_packets; - vport_stats->tx_multicast_bytes = tmp_vport_stats.tx_multicast_bytes; - vport_stats->tx_broadcast_packets = tmp_vport_stats.tx_broadcast_packets; - vport_stats->tx_broadcast_bytes = tmp_vport_stats.tx_broadcast_bytes; - vport_stats->tx_errors = tmp_vport_stats.tx_errors; - } - -#if __FreeBSD_version >= 1100000 - if (reset == 0) { - if_inc_counter(dev, IFCOUNTER_IPACKETS, - priv->pkstats.rx_packets - priv->pkstats_last.rx_packets); - if_inc_counter(dev, IFCOUNTER_OPACKETS, - priv->pkstats.tx_packets - priv->pkstats_last.tx_packets); - if_inc_counter(dev, IFCOUNTER_IBYTES, - priv->pkstats.rx_bytes - priv->pkstats_last.rx_bytes); - if_inc_counter(dev, IFCOUNTER_OBYTES, - priv->pkstats.tx_bytes - priv->pkstats_last.tx_bytes); - if_inc_counter(dev, IFCOUNTER_IERRORS, - priv->pkstats.rx_errors - priv->pkstats_last.rx_errors); - if_inc_counter(dev, IFCOUNTER_IQDROPS, - priv->pkstats.rx_dropped - priv->pkstats_last.rx_dropped); - if_inc_counter(dev, IFCOUNTER_IMCASTS, - priv->pkstats.rx_multicast_packets - priv->pkstats_last.rx_multicast_packets); - if_inc_counter(dev, IFCOUNTER_OMCASTS, - priv->pkstats.tx_multicast_packets - priv->pkstats_last.tx_multicast_packets); - } - priv->pkstats_last = priv->pkstats; -#else - dev->if_ipackets = priv->pkstats.rx_packets; - dev->if_opackets = priv->pkstats.tx_packets; - dev->if_ibytes = priv->pkstats.rx_bytes; - dev->if_obytes = priv->pkstats.tx_bytes; - dev->if_ierrors = priv->pkstats.rx_errors; - dev->if_iqdrops = priv->pkstats.rx_dropped; - dev->if_imcasts = priv->pkstats.rx_multicast_packets; - dev->if_omcasts = priv->pkstats.tx_multicast_packets; - dev->if_collisions = 0; -#endif - - spin_unlock(&priv->stats_lock); - -out: - mlx4_free_cmd_mailbox(mdev->dev, mailbox_flow); - mlx4_free_cmd_mailbox(mdev->dev, mailbox); - -mailbox_out: - if (do_if_stat) - priv->last_ifq_jiffies = jiffies; - - return err; -} Property changes on: stable/11/sys/ofed/drivers/net/mlx4/en_port.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/catas.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/catas.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/catas.c (nonexistent) @@ -1,175 +0,0 @@ -/* - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#define LINUXKPI_PARAM_PREFIX mlx4_ - -#include -#include - -#include - -#include "mlx4.h" - -#define MLX4_CATAS_POLL_INTERVAL (5 * HZ) - -static DEFINE_SPINLOCK(catas_lock); - -static LIST_HEAD(catas_list); -static struct work_struct catas_work; - -static int internal_err_reset = 1; -module_param(internal_err_reset, int, 0644); -MODULE_PARM_DESC(internal_err_reset, - "Reset device on internal errors if non-zero" - " (default 1, in SRIOV mode default is 0)"); - -static void dump_err_buf(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - int i; - - mlx4_err(dev, "Internal error detected:\n"); - for (i = 0; i < priv->fw.catas_size; ++i) - mlx4_err(dev, " buf[%02x]: %08x\n", - i, swab32(readl(priv->catas_err.map + i))); -} - -static void poll_catas(unsigned long dev_ptr) -{ - struct mlx4_dev *dev = (struct mlx4_dev *) dev_ptr; - struct mlx4_priv *priv = mlx4_priv(dev); - - if (readl(priv->catas_err.map)) { - /* If the device is off-line, we cannot try to recover it */ - if (pci_channel_offline(dev->pdev)) - mod_timer(&priv->catas_err.timer, - round_jiffies(jiffies + MLX4_CATAS_POLL_INTERVAL)); - else { - dump_err_buf(dev); - mlx4_dispatch_event(dev, MLX4_DEV_EVENT_CATASTROPHIC_ERROR, 0); - - if (internal_err_reset) { - spin_lock(&catas_lock); - list_add(&priv->catas_err.list, &catas_list); - spin_unlock(&catas_lock); - - queue_work(mlx4_wq, &catas_work); - } - } - } else - mod_timer(&priv->catas_err.timer, - round_jiffies(jiffies + MLX4_CATAS_POLL_INTERVAL)); -} - -static void catas_reset(struct work_struct *work) -{ - struct mlx4_priv *priv, *tmppriv; - struct mlx4_dev *dev; - - LIST_HEAD(tlist); - int ret; - - spin_lock_irq(&catas_lock); - list_splice_init(&catas_list, &tlist); - spin_unlock_irq(&catas_lock); - - list_for_each_entry_safe(priv, tmppriv, &tlist, catas_err.list) { - struct pci_dev *pdev = priv->dev.pdev; - - /* If the device is off-line, we cannot reset it */ - if (pci_channel_offline(pdev)) - continue; - - ret = mlx4_restart_one(priv->dev.pdev); - /* 'priv' now is not valid */ - if (ret) - pr_err("mlx4 %s: Reset failed (%d)\n", - pci_name(pdev), ret); - else { - dev = pci_get_drvdata(pdev); - mlx4_dbg(dev, "Reset succeeded\n"); - } - } -} - -void mlx4_start_catas_poll(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - phys_addr_t addr; - - /*If we are in SRIOV the default of the module param must be 0*/ - if (mlx4_is_mfunc(dev)) - internal_err_reset = 0; - - INIT_LIST_HEAD(&priv->catas_err.list); - init_timer(&priv->catas_err.timer); - priv->catas_err.map = NULL; - - addr = pci_resource_start(dev->pdev, priv->fw.catas_bar) + - priv->fw.catas_offset; - - priv->catas_err.map = ioremap(addr, priv->fw.catas_size * 4); - if (!priv->catas_err.map) { - mlx4_warn(dev, "Failed to map internal error buffer at 0x%llx\n", - (unsigned long long) addr); - return; - } - - priv->catas_err.timer.data = (unsigned long) dev; - priv->catas_err.timer.function = poll_catas; - priv->catas_err.timer.expires = - round_jiffies(jiffies + MLX4_CATAS_POLL_INTERVAL); - add_timer(&priv->catas_err.timer); -} - -void mlx4_stop_catas_poll(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - del_timer_sync(&priv->catas_err.timer); - - if (priv->catas_err.map) { - iounmap(priv->catas_err.map); - priv->catas_err.map = NULL; - } - - spin_lock_irq(&catas_lock); - list_del_init(&priv->catas_err.list); - spin_unlock_irq(&catas_lock); -} - -void __init mlx4_catas_init(void) -{ - INIT_WORK(&catas_work, catas_reset); -} Property changes on: stable/11/sys/ofed/drivers/net/mlx4/catas.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/mcg.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/mcg.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/mcg.c (nonexistent) @@ -1,1543 +0,0 @@ -/* - * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include - -#include -#include -#include - -#include "mlx4.h" - -int mlx4_get_mgm_entry_size(struct mlx4_dev *dev) -{ - return 1 << dev->oper_log_mgm_entry_size; -} - -int mlx4_get_qp_per_mgm(struct mlx4_dev *dev) -{ - return 4 * (mlx4_get_mgm_entry_size(dev) / 16 - 2); -} - -static int mlx4_QP_FLOW_STEERING_ATTACH(struct mlx4_dev *dev, - struct mlx4_cmd_mailbox *mailbox, - u32 size, - u64 *reg_id) -{ - u64 imm; - int err = 0; - - err = mlx4_cmd_imm(dev, mailbox->dma, &imm, size, 0, - MLX4_QP_FLOW_STEERING_ATTACH, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - if (err) - return err; - *reg_id = imm; - - return err; -} - -static int mlx4_QP_FLOW_STEERING_DETACH(struct mlx4_dev *dev, u64 regid) -{ - int err = 0; - - err = mlx4_cmd(dev, regid, 0, 0, - MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - - return err; -} - -static int mlx4_READ_ENTRY(struct mlx4_dev *dev, int index, - struct mlx4_cmd_mailbox *mailbox) -{ - return mlx4_cmd_box(dev, 0, mailbox->dma, index, 0, MLX4_CMD_READ_MCG, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); -} - -static int mlx4_WRITE_ENTRY(struct mlx4_dev *dev, int index, - struct mlx4_cmd_mailbox *mailbox) -{ - return mlx4_cmd(dev, mailbox->dma, index, 0, MLX4_CMD_WRITE_MCG, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); -} - -static int mlx4_WRITE_PROMISC(struct mlx4_dev *dev, u8 port, u8 steer, - struct mlx4_cmd_mailbox *mailbox) -{ - u32 in_mod; - - in_mod = (u32) port << 16 | steer << 1; - return mlx4_cmd(dev, mailbox->dma, in_mod, 0x1, - MLX4_CMD_WRITE_MCG, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); -} - -static int mlx4_GID_HASH(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, - u16 *hash, u8 op_mod) -{ - u64 imm; - int err; - - err = mlx4_cmd_imm(dev, mailbox->dma, &imm, 0, op_mod, - MLX4_CMD_MGID_HASH, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - - if (!err) - *hash = imm; - - return err; -} - -static struct mlx4_promisc_qp *get_promisc_qp(struct mlx4_dev *dev, u8 port, - enum mlx4_steer_type steer, - u32 qpn) -{ - struct mlx4_steer *s_steer; - struct mlx4_promisc_qp *pqp; - - if (port < 1 || port > dev->caps.num_ports) - return NULL; - - s_steer = &mlx4_priv(dev)->steer[port - 1]; - - list_for_each_entry(pqp, &s_steer->promisc_qps[steer], list) { - if (pqp->qpn == qpn) - return pqp; - } - /* not found */ - return NULL; -} - -/* - * Add new entry to steering data structure. - * All promisc QPs should be added as well - */ -static int new_steering_entry(struct mlx4_dev *dev, u8 port, - enum mlx4_steer_type steer, - unsigned int index, u32 qpn) -{ - struct mlx4_steer *s_steer; - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_mgm *mgm; - u32 members_count; - struct mlx4_steer_index *new_entry; - struct mlx4_promisc_qp *pqp; - struct mlx4_promisc_qp *dqp = NULL; - u32 prot; - int err; - - if (port < 1 || port > dev->caps.num_ports) - return -EINVAL; - - s_steer = &mlx4_priv(dev)->steer[port - 1]; - new_entry = kzalloc(sizeof *new_entry, GFP_KERNEL); - if (!new_entry) - return -ENOMEM; - - INIT_LIST_HEAD(&new_entry->duplicates); - new_entry->index = index; - list_add_tail(&new_entry->list, &s_steer->steer_entries[steer]); - - /* If the given qpn is also a promisc qp, - * it should be inserted to duplicates list - */ - pqp = get_promisc_qp(dev, port, steer, qpn); - if (pqp) { - dqp = kmalloc(sizeof *dqp, GFP_KERNEL); - if (!dqp) { - err = -ENOMEM; - goto out_alloc; - } - dqp->qpn = qpn; - list_add_tail(&dqp->list, &new_entry->duplicates); - } - - /* if no promisc qps for this vep, we are done */ - if (list_empty(&s_steer->promisc_qps[steer])) - return 0; - - /* now need to add all the promisc qps to the new - * steering entry, as they should also receive the packets - * destined to this address */ - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) { - err = -ENOMEM; - goto out_alloc; - } - mgm = mailbox->buf; - - err = mlx4_READ_ENTRY(dev, index, mailbox); - if (err) - goto out_mailbox; - - members_count = be32_to_cpu(mgm->members_count) & 0xffffff; - prot = be32_to_cpu(mgm->members_count) >> 30; - list_for_each_entry(pqp, &s_steer->promisc_qps[steer], list) { - /* don't add already existing qpn */ - if (pqp->qpn == qpn) - continue; - if (members_count == dev->caps.num_qp_per_mgm) { - /* out of space */ - err = -ENOMEM; - goto out_mailbox; - } - - /* add the qpn */ - mgm->qp[members_count++] = cpu_to_be32(pqp->qpn & MGM_QPN_MASK); - } - /* update the qps count and update the entry with all the promisc qps*/ - mgm->members_count = cpu_to_be32(members_count | (prot << 30)); - err = mlx4_WRITE_ENTRY(dev, index, mailbox); - -out_mailbox: - mlx4_free_cmd_mailbox(dev, mailbox); - if (!err) - return 0; -out_alloc: - if (dqp) { - list_del(&dqp->list); - kfree(dqp); - } - list_del(&new_entry->list); - kfree(new_entry); - return err; -} - -/* update the data structures with existing steering entry */ -static int existing_steering_entry(struct mlx4_dev *dev, u8 port, - enum mlx4_steer_type steer, - unsigned int index, u32 qpn) -{ - struct mlx4_steer *s_steer; - struct mlx4_steer_index *tmp_entry, *entry = NULL; - struct mlx4_promisc_qp *pqp; - struct mlx4_promisc_qp *dqp; - - if (port < 1 || port > dev->caps.num_ports) - return -EINVAL; - - s_steer = &mlx4_priv(dev)->steer[port - 1]; - - pqp = get_promisc_qp(dev, port, steer, qpn); - if (!pqp) - return 0; /* nothing to do */ - - list_for_each_entry(tmp_entry, &s_steer->steer_entries[steer], list) { - if (tmp_entry->index == index) { - entry = tmp_entry; - break; - } - } - if (unlikely(!entry)) { - mlx4_warn(dev, "Steering entry at index %x is not registered\n", index); - return -EINVAL; - } - - /* the given qpn is listed as a promisc qpn - * we need to add it as a duplicate to this entry - * for future references */ - list_for_each_entry(dqp, &entry->duplicates, list) { - if (qpn == dqp->qpn) - return 0; /* qp is already duplicated */ - } - - /* add the qp as a duplicate on this index */ - dqp = kmalloc(sizeof *dqp, GFP_KERNEL); - if (!dqp) - return -ENOMEM; - dqp->qpn = qpn; - list_add_tail(&dqp->list, &entry->duplicates); - - return 0; -} - -/* Check whether a qpn is a duplicate on steering entry - * If so, it should not be removed from mgm */ -static bool check_duplicate_entry(struct mlx4_dev *dev, u8 port, - enum mlx4_steer_type steer, - unsigned int index, u32 qpn) -{ - struct mlx4_steer *s_steer; - struct mlx4_steer_index *tmp_entry, *entry = NULL; - struct mlx4_promisc_qp *dqp, *tmp_dqp; - - if (port < 1 || port > dev->caps.num_ports) - return NULL; - - s_steer = &mlx4_priv(dev)->steer[port - 1]; - - /* if qp is not promisc, it cannot be duplicated */ - if (!get_promisc_qp(dev, port, steer, qpn)) - return false; - - /* The qp is promisc qp so it is a duplicate on this index - * Find the index entry, and remove the duplicate */ - list_for_each_entry(tmp_entry, &s_steer->steer_entries[steer], list) { - if (tmp_entry->index == index) { - entry = tmp_entry; - break; - } - } - if (unlikely(!entry)) { - mlx4_warn(dev, "Steering entry for index %x is not registered\n", index); - return false; - } - list_for_each_entry_safe(dqp, tmp_dqp, &entry->duplicates, list) { - if (dqp->qpn == qpn) { - list_del(&dqp->list); - kfree(dqp); - } - } - return true; -} - -/* - * returns true if all the QPs != tqpn contained in this entry - * are Promisc QPs. return false otherwise. - */ -static bool promisc_steering_entry(struct mlx4_dev *dev, u8 port, - enum mlx4_steer_type steer, - unsigned int index, u32 tqpn, u32 *members_count) -{ - struct mlx4_steer *s_steer; - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_mgm *mgm; - u32 m_count; - bool ret = false; - int i; - - if (port < 1 || port > dev->caps.num_ports) - return false; - - s_steer = &mlx4_priv(dev)->steer[port - 1]; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return false; - mgm = mailbox->buf; - - if (mlx4_READ_ENTRY(dev, index, mailbox)) - goto out; - m_count = be32_to_cpu(mgm->members_count) & 0xffffff; - if (members_count) - *members_count = m_count; - - for (i = 0; i < m_count; i++) { - u32 qpn = be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK; - if (!get_promisc_qp(dev, port, steer, qpn) && qpn != tqpn) { - /* the qp is not promisc, the entry can't be removed */ - goto out; - } - } - ret = true; -out: - mlx4_free_cmd_mailbox(dev, mailbox); - return ret; -} - -/* IF a steering entry contains only promisc QPs, it can be removed. */ -static bool can_remove_steering_entry(struct mlx4_dev *dev, u8 port, - enum mlx4_steer_type steer, - unsigned int index, u32 tqpn) -{ - struct mlx4_steer *s_steer; - struct mlx4_steer_index *entry = NULL, *tmp_entry; - u32 members_count; - bool ret = false; - - if (port < 1 || port > dev->caps.num_ports) - return NULL; - - s_steer = &mlx4_priv(dev)->steer[port - 1]; - - if (!promisc_steering_entry(dev, port, steer, index, tqpn, &members_count)) - goto out; - - /* All the qps currently registered for this entry are promiscuous, - * Checking for duplicates */ - ret = true; - list_for_each_entry_safe(entry, tmp_entry, &s_steer->steer_entries[steer], list) { - if (entry->index == index) { - if (list_empty(&entry->duplicates) || members_count == 1) { - struct mlx4_promisc_qp *pqp, *tmp_pqp; - /* - * If there is only 1 entry in duplicates than - * this is the QP we want to delete, going over - * the list and deleting the entry. - */ - list_del(&entry->list); - list_for_each_entry_safe(pqp, tmp_pqp, - &entry->duplicates, - list) { - list_del(&pqp->list); - kfree(pqp); - } - kfree(entry); - } else { - /* This entry contains duplicates so it shouldn't be removed */ - ret = false; - goto out; - } - } - } - -out: - return ret; -} - -static int add_promisc_qp(struct mlx4_dev *dev, u8 port, - enum mlx4_steer_type steer, u32 qpn) -{ - struct mlx4_steer *s_steer; - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_mgm *mgm; - struct mlx4_steer_index *entry; - struct mlx4_promisc_qp *pqp; - struct mlx4_promisc_qp *dqp; - u32 members_count; - u32 prot; - int i; - bool found; - int err; - struct mlx4_priv *priv = mlx4_priv(dev); - - if (port < 1 || port > dev->caps.num_ports) - return -EINVAL; - - s_steer = &mlx4_priv(dev)->steer[port - 1]; - - mutex_lock(&priv->mcg_table.mutex); - - if (get_promisc_qp(dev, port, steer, qpn)) { - err = 0; /* Noting to do, already exists */ - goto out_mutex; - } - - pqp = kmalloc(sizeof *pqp, GFP_KERNEL); - if (!pqp) { - err = -ENOMEM; - goto out_mutex; - } - pqp->qpn = qpn; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) { - err = -ENOMEM; - goto out_alloc; - } - mgm = mailbox->buf; - - if (!(mlx4_is_mfunc(dev) && steer == MLX4_UC_STEER)) { - /* the promisc qp needs to be added for each one of the steering - * entries, if it already exists, needs to be added as a duplicate - * for this entry */ - list_for_each_entry(entry, &s_steer->steer_entries[steer], list) { - err = mlx4_READ_ENTRY(dev, entry->index, mailbox); - if (err) - goto out_mailbox; - - members_count = be32_to_cpu(mgm->members_count) & 0xffffff; - prot = be32_to_cpu(mgm->members_count) >> 30; - found = false; - for (i = 0; i < members_count; i++) { - if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qpn) { - /* Entry already exists, add to duplicates */ - dqp = kmalloc(sizeof *dqp, GFP_KERNEL); - if (!dqp) { - err = -ENOMEM; - goto out_mailbox; - } - dqp->qpn = qpn; - list_add_tail(&dqp->list, &entry->duplicates); - found = true; - } - } - if (!found) { - /* Need to add the qpn to mgm */ - if (members_count == dev->caps.num_qp_per_mgm) { - /* entry is full */ - err = -ENOMEM; - goto out_mailbox; - } - mgm->qp[members_count++] = cpu_to_be32(qpn & MGM_QPN_MASK); - mgm->members_count = cpu_to_be32(members_count | (prot << 30)); - err = mlx4_WRITE_ENTRY(dev, entry->index, mailbox); - if (err) - goto out_mailbox; - } - } - } - - /* add the new qpn to list of promisc qps */ - list_add_tail(&pqp->list, &s_steer->promisc_qps[steer]); - /* now need to add all the promisc qps to default entry */ - memset(mgm, 0, sizeof *mgm); - members_count = 0; - list_for_each_entry(dqp, &s_steer->promisc_qps[steer], list) { - if (members_count == dev->caps.num_qp_per_mgm) { - /* entry is full */ - err = -ENOMEM; - goto out_list; - } - mgm->qp[members_count++] = cpu_to_be32(dqp->qpn & MGM_QPN_MASK); - } - mgm->members_count = cpu_to_be32(members_count | MLX4_PROT_ETH << 30); - - err = mlx4_WRITE_PROMISC(dev, port, steer, mailbox); - if (err) - goto out_list; - - mlx4_free_cmd_mailbox(dev, mailbox); - mutex_unlock(&priv->mcg_table.mutex); - return 0; - -out_list: - list_del(&pqp->list); -out_mailbox: - mlx4_free_cmd_mailbox(dev, mailbox); -out_alloc: - kfree(pqp); -out_mutex: - mutex_unlock(&priv->mcg_table.mutex); - return err; -} - -static int remove_promisc_qp(struct mlx4_dev *dev, u8 port, - enum mlx4_steer_type steer, u32 qpn) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_steer *s_steer; - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_mgm *mgm; - struct mlx4_steer_index *entry, *tmp_entry; - struct mlx4_promisc_qp *pqp; - struct mlx4_promisc_qp *dqp; - u32 members_count; - bool found; - bool back_to_list = false; - int i, loc = -1; - int err; - - if (port < 1 || port > dev->caps.num_ports) - return -EINVAL; - - s_steer = &mlx4_priv(dev)->steer[port - 1]; - mutex_lock(&priv->mcg_table.mutex); - - pqp = get_promisc_qp(dev, port, steer, qpn); - if (unlikely(!pqp)) { - mlx4_warn(dev, "QP %x is not promiscuous QP\n", qpn); - /* nothing to do */ - err = 0; - goto out_mutex; - } - - /*remove from list of promisc qps */ - list_del(&pqp->list); - - /* set the default entry not to include the removed one */ - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) { - err = -ENOMEM; - back_to_list = true; - goto out_list; - } - mgm = mailbox->buf; - memset(mgm, 0, sizeof *mgm); - members_count = 0; - list_for_each_entry(dqp, &s_steer->promisc_qps[steer], list) - mgm->qp[members_count++] = cpu_to_be32(dqp->qpn & MGM_QPN_MASK); - mgm->members_count = cpu_to_be32(members_count | MLX4_PROT_ETH << 30); - - err = mlx4_WRITE_PROMISC(dev, port, steer, mailbox); - if (err) - goto out_mailbox; - - if (!(mlx4_is_mfunc(dev) && steer == MLX4_UC_STEER)) { - /* remove the qp from all the steering entries*/ - list_for_each_entry_safe(entry, tmp_entry, &s_steer->steer_entries[steer], list) { - found = false; - list_for_each_entry(dqp, &entry->duplicates, list) { - if (dqp->qpn == qpn) { - found = true; - break; - } - } - if (found) { - /* a duplicate, no need to change the mgm, - * only update the duplicates list */ - list_del(&dqp->list); - kfree(dqp); - } else { - err = mlx4_READ_ENTRY(dev, entry->index, mailbox); - if (err) - goto out_mailbox; - members_count = be32_to_cpu(mgm->members_count) & 0xffffff; - if (!members_count) { - mlx4_warn(dev, "QP %06x wasn't found in entry %x mcount=0." - " deleting entry...\n", qpn, entry->index); - list_del(&entry->list); - kfree(entry); - continue; - } - - for (i = 0; i < members_count; ++i) - if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qpn) { - loc = i; - break; - } - - if (loc < 0) { - mlx4_err(dev, "QP %06x wasn't found in entry %d\n", - qpn, entry->index); - err = -EINVAL; - goto out_mailbox; - } - - /* copy the last QP in this MGM over removed QP */ - mgm->qp[loc] = mgm->qp[members_count - 1]; - mgm->qp[members_count - 1] = 0; - mgm->members_count = cpu_to_be32(--members_count | - (MLX4_PROT_ETH << 30)); - - err = mlx4_WRITE_ENTRY(dev, entry->index, mailbox); - if (err) - goto out_mailbox; - } - } - } - -out_mailbox: - mlx4_free_cmd_mailbox(dev, mailbox); -out_list: - if (back_to_list) - list_add_tail(&pqp->list, &s_steer->promisc_qps[steer]); - else - kfree(pqp); -out_mutex: - mutex_unlock(&priv->mcg_table.mutex); - return err; -} - -/* - * Caller must hold MCG table semaphore. gid and mgm parameters must - * be properly aligned for command interface. - * - * Returns 0 unless a firmware command error occurs. - * - * If GID is found in MGM or MGM is empty, *index = *hash, *prev = -1 - * and *mgm holds MGM entry. - * - * if GID is found in AMGM, *index = index in AMGM, *prev = index of - * previous entry in hash chain and *mgm holds AMGM entry. - * - * If no AMGM exists for given gid, *index = -1, *prev = index of last - * entry in hash chain and *mgm holds end of hash chain. - */ -static int find_entry(struct mlx4_dev *dev, u8 port, - u8 *gid, enum mlx4_protocol prot, - struct mlx4_cmd_mailbox *mgm_mailbox, - int *prev, int *index) -{ - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_mgm *mgm = mgm_mailbox->buf; - u8 *mgid; - int err; - u16 hash; - u8 op_mod = (prot == MLX4_PROT_ETH) ? - !!(dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) : 0; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return -ENOMEM; - mgid = mailbox->buf; - - memcpy(mgid, gid, 16); - - err = mlx4_GID_HASH(dev, mailbox, &hash, op_mod); - mlx4_free_cmd_mailbox(dev, mailbox); - if (err) - return err; - - if (0) { - mlx4_dbg(dev, "Hash for "GID_PRINT_FMT" is %04x\n", - GID_PRINT_ARGS(gid), hash); - } - - *index = hash; - *prev = -1; - - do { - err = mlx4_READ_ENTRY(dev, *index, mgm_mailbox); - if (err) - return err; - - if (!(be32_to_cpu(mgm->members_count) & 0xffffff)) { - if (*index != hash) { - mlx4_err(dev, "Found zero MGID in AMGM.\n"); - err = -EINVAL; - } - return err; - } - - if (!memcmp(mgm->gid, gid, 16) && - be32_to_cpu(mgm->members_count) >> 30 == prot) - return err; - - *prev = *index; - *index = be32_to_cpu(mgm->next_gid_index) >> 6; - } while (*index); - - *index = -1; - return err; -} - -static const u8 __promisc_mode[] = { - [MLX4_FS_REGULAR] = 0x0, - [MLX4_FS_ALL_DEFAULT] = 0x1, - [MLX4_FS_MC_DEFAULT] = 0x3, - [MLX4_FS_UC_SNIFFER] = 0x4, - [MLX4_FS_MC_SNIFFER] = 0x5, -}; - -int map_sw_to_hw_steering_mode(struct mlx4_dev *dev, - enum mlx4_net_trans_promisc_mode flow_type) -{ - if (flow_type >= MLX4_FS_MODE_NUM || flow_type < 0) { - mlx4_err(dev, "Invalid flow type. type = %d\n", flow_type); - return -EINVAL; - } - return __promisc_mode[flow_type]; -} -EXPORT_SYMBOL_GPL(map_sw_to_hw_steering_mode); - -static void trans_rule_ctrl_to_hw(struct mlx4_net_trans_rule *ctrl, - struct mlx4_net_trans_rule_hw_ctrl *hw) -{ - u8 flags = 0; - - flags = ctrl->queue_mode == MLX4_NET_TRANS_Q_LIFO ? 1 : 0; - flags |= ctrl->exclusive ? (1 << 2) : 0; - flags |= ctrl->allow_loopback ? (1 << 3) : 0; - - hw->flags = flags; - hw->type = __promisc_mode[ctrl->promisc_mode]; - hw->prio = cpu_to_be16(ctrl->priority); - hw->port = ctrl->port; - hw->qpn = cpu_to_be32(ctrl->qpn); -} - -const u16 __sw_id_hw[] = { - [MLX4_NET_TRANS_RULE_ID_ETH] = 0xE001, - [MLX4_NET_TRANS_RULE_ID_IB] = 0xE005, - [MLX4_NET_TRANS_RULE_ID_IPV6] = 0xE003, - [MLX4_NET_TRANS_RULE_ID_IPV4] = 0xE002, - [MLX4_NET_TRANS_RULE_ID_TCP] = 0xE004, - [MLX4_NET_TRANS_RULE_ID_UDP] = 0xE006 -}; - -int map_sw_to_hw_steering_id(struct mlx4_dev *dev, - enum mlx4_net_trans_rule_id id) -{ - if (id >= MLX4_NET_TRANS_RULE_NUM || id < 0) { - mlx4_err(dev, "Invalid network rule id. id = %d\n", id); - return -EINVAL; - } - return __sw_id_hw[id]; -} -EXPORT_SYMBOL_GPL(map_sw_to_hw_steering_id); - -static const int __rule_hw_sz[] = { - [MLX4_NET_TRANS_RULE_ID_ETH] = - sizeof(struct mlx4_net_trans_rule_hw_eth), - [MLX4_NET_TRANS_RULE_ID_IB] = - sizeof(struct mlx4_net_trans_rule_hw_ib), - [MLX4_NET_TRANS_RULE_ID_IPV6] = 0, - [MLX4_NET_TRANS_RULE_ID_IPV4] = - sizeof(struct mlx4_net_trans_rule_hw_ipv4), - [MLX4_NET_TRANS_RULE_ID_TCP] = - sizeof(struct mlx4_net_trans_rule_hw_tcp_udp), - [MLX4_NET_TRANS_RULE_ID_UDP] = - sizeof(struct mlx4_net_trans_rule_hw_tcp_udp) -}; - -int hw_rule_sz(struct mlx4_dev *dev, - enum mlx4_net_trans_rule_id id) -{ - if (id >= MLX4_NET_TRANS_RULE_NUM || id < 0) { - mlx4_err(dev, "Invalid network rule id. id = %d\n", id); - return -EINVAL; - } - - return __rule_hw_sz[id]; -} -EXPORT_SYMBOL_GPL(hw_rule_sz); - -static int parse_trans_rule(struct mlx4_dev *dev, struct mlx4_spec_list *spec, - struct _rule_hw *rule_hw) -{ - if (hw_rule_sz(dev, spec->id) < 0) - return -EINVAL; - memset(rule_hw, 0, hw_rule_sz(dev, spec->id)); - rule_hw->id = cpu_to_be16(__sw_id_hw[spec->id]); - rule_hw->size = hw_rule_sz(dev, spec->id) >> 2; - - switch (spec->id) { - case MLX4_NET_TRANS_RULE_ID_ETH: - memcpy(rule_hw->eth.dst_mac, spec->eth.dst_mac, ETH_ALEN); - memcpy(rule_hw->eth.dst_mac_msk, spec->eth.dst_mac_msk, - ETH_ALEN); - memcpy(rule_hw->eth.src_mac, spec->eth.src_mac, ETH_ALEN); - memcpy(rule_hw->eth.src_mac_msk, spec->eth.src_mac_msk, - ETH_ALEN); - if (spec->eth.ether_type_enable) { - rule_hw->eth.ether_type_enable = 1; - rule_hw->eth.ether_type = spec->eth.ether_type; - } - rule_hw->eth.vlan_tag = spec->eth.vlan_id; - rule_hw->eth.vlan_tag_msk = spec->eth.vlan_id_msk; - break; - - case MLX4_NET_TRANS_RULE_ID_IB: - rule_hw->ib.l3_qpn = spec->ib.l3_qpn; - rule_hw->ib.qpn_mask = spec->ib.qpn_msk; - memcpy(&rule_hw->ib.dst_gid, &spec->ib.dst_gid, 16); - memcpy(&rule_hw->ib.dst_gid_msk, &spec->ib.dst_gid_msk, 16); - break; - - case MLX4_NET_TRANS_RULE_ID_IPV6: - return -EOPNOTSUPP; - - case MLX4_NET_TRANS_RULE_ID_IPV4: - rule_hw->ipv4.src_ip = spec->ipv4.src_ip; - rule_hw->ipv4.src_ip_msk = spec->ipv4.src_ip_msk; - rule_hw->ipv4.dst_ip = spec->ipv4.dst_ip; - rule_hw->ipv4.dst_ip_msk = spec->ipv4.dst_ip_msk; - break; - - case MLX4_NET_TRANS_RULE_ID_TCP: - case MLX4_NET_TRANS_RULE_ID_UDP: - rule_hw->tcp_udp.dst_port = spec->tcp_udp.dst_port; - rule_hw->tcp_udp.dst_port_msk = spec->tcp_udp.dst_port_msk; - rule_hw->tcp_udp.src_port = spec->tcp_udp.src_port; - rule_hw->tcp_udp.src_port_msk = spec->tcp_udp.src_port_msk; - break; - - default: - return -EINVAL; - } - - return __rule_hw_sz[spec->id]; -} - -static void mlx4_err_rule(struct mlx4_dev *dev, char *str, - struct mlx4_net_trans_rule *rule) -{ -#define BUF_SIZE 256 - struct mlx4_spec_list *cur; - char buf[BUF_SIZE]; - int len = 0; - - mlx4_err(dev, "%s", str); - len += snprintf(buf + len, BUF_SIZE - len, - "port = %d prio = 0x%x qp = 0x%x ", - rule->port, rule->priority, rule->qpn); - - list_for_each_entry(cur, &rule->list, list) { - switch (cur->id) { - case MLX4_NET_TRANS_RULE_ID_ETH: - len += snprintf(buf + len, BUF_SIZE - len, - "dmac = %pM ", &cur->eth.dst_mac); - if (cur->eth.ether_type) - len += snprintf(buf + len, BUF_SIZE - len, - "ethertype = 0x%x ", - be16_to_cpu(cur->eth.ether_type)); - if (cur->eth.vlan_id) - len += snprintf(buf + len, BUF_SIZE - len, - "vlan-id = %d ", - be16_to_cpu(cur->eth.vlan_id)); - break; - - case MLX4_NET_TRANS_RULE_ID_IPV4: - if (cur->ipv4.src_ip) - len += snprintf(buf + len, BUF_SIZE - len, - "src-ip = %pI4 ", - &cur->ipv4.src_ip); - if (cur->ipv4.dst_ip) - len += snprintf(buf + len, BUF_SIZE - len, - "dst-ip = %pI4 ", - &cur->ipv4.dst_ip); - break; - - case MLX4_NET_TRANS_RULE_ID_TCP: - case MLX4_NET_TRANS_RULE_ID_UDP: - if (cur->tcp_udp.src_port) - len += snprintf(buf + len, BUF_SIZE - len, - "src-port = %d ", - be16_to_cpu(cur->tcp_udp.src_port)); - if (cur->tcp_udp.dst_port) - len += snprintf(buf + len, BUF_SIZE - len, - "dst-port = %d ", - be16_to_cpu(cur->tcp_udp.dst_port)); - break; - - case MLX4_NET_TRANS_RULE_ID_IB: - len += snprintf(buf + len, BUF_SIZE - len, - "dst-gid = "GID_PRINT_FMT"\n", - GID_PRINT_ARGS(cur->ib.dst_gid)); - len += snprintf(buf + len, BUF_SIZE - len, - "dst-gid-mask = "GID_PRINT_FMT"\n", - GID_PRINT_ARGS(cur->ib.dst_gid_msk)); - break; - - case MLX4_NET_TRANS_RULE_ID_IPV6: - break; - - default: - break; - } - } - len += snprintf(buf + len, BUF_SIZE - len, "\n"); - mlx4_err(dev, "%s", buf); - - if (len >= BUF_SIZE) - mlx4_err(dev, "Network rule error message was truncated, print buffer is too small.\n"); -} - -int mlx4_flow_attach(struct mlx4_dev *dev, - struct mlx4_net_trans_rule *rule, u64 *reg_id) -{ - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_spec_list *cur; - u32 size = 0; - int ret; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - memset(mailbox->buf, 0, sizeof(struct mlx4_net_trans_rule_hw_ctrl)); - trans_rule_ctrl_to_hw(rule, mailbox->buf); - - size += sizeof(struct mlx4_net_trans_rule_hw_ctrl); - - list_for_each_entry(cur, &rule->list, list) { - ret = parse_trans_rule(dev, cur, mailbox->buf + size); - if (ret < 0) { - mlx4_free_cmd_mailbox(dev, mailbox); - return -EINVAL; - } - size += ret; - } - - ret = mlx4_QP_FLOW_STEERING_ATTACH(dev, mailbox, size >> 2, reg_id); - if (ret == -ENOMEM) - mlx4_err_rule(dev, - "mcg table is full. Fail to register network rule.\n", - rule); - else if (ret) - mlx4_err_rule(dev, "Fail to register network rule.\n", rule); - - mlx4_free_cmd_mailbox(dev, mailbox); - - return ret; -} -EXPORT_SYMBOL_GPL(mlx4_flow_attach); - -int mlx4_flow_detach(struct mlx4_dev *dev, u64 reg_id) -{ - int err; - - err = mlx4_QP_FLOW_STEERING_DETACH(dev, reg_id); - if (err) - mlx4_err(dev, "Fail to detach network rule. registration id = 0x%llx\n", - (unsigned long long)reg_id); - return err; -} -EXPORT_SYMBOL_GPL(mlx4_flow_detach); - -int mlx4_FLOW_STEERING_IB_UC_QP_RANGE(struct mlx4_dev *dev, u32 min_range_qpn, u32 max_range_qpn) -{ - int err; - u64 in_param; - - in_param = ((u64) min_range_qpn) << 32; - in_param |= ((u64) max_range_qpn) & 0xFFFFFFFF; - - err = mlx4_cmd(dev, in_param, 0, 0, - MLX4_FLOW_STEERING_IB_UC_QP_RANGE, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); - - return err; -} -EXPORT_SYMBOL_GPL(mlx4_FLOW_STEERING_IB_UC_QP_RANGE); - -int mlx4_qp_attach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], - int block_mcast_loopback, enum mlx4_protocol prot, - enum mlx4_steer_type steer) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_mgm *mgm; - u32 members_count; - int index, prev; - int link = 0; - int i; - int err; - u8 port = gid[5]; - u8 new_entry = 0; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - mgm = mailbox->buf; - - mutex_lock(&priv->mcg_table.mutex); - err = find_entry(dev, port, gid, prot, - mailbox, &prev, &index); - if (err) - goto out; - - if (index != -1) { - if (!(be32_to_cpu(mgm->members_count) & 0xffffff)) { - new_entry = 1; - memcpy(mgm->gid, gid, 16); - } - } else { - link = 1; - - index = mlx4_bitmap_alloc(&priv->mcg_table.bitmap); - if (index == -1) { - mlx4_err(dev, "No AMGM entries left\n"); - err = -ENOMEM; - goto out; - } - index += dev->caps.num_mgms; - - new_entry = 1; - memset(mgm, 0, sizeof *mgm); - memcpy(mgm->gid, gid, 16); - } - - members_count = be32_to_cpu(mgm->members_count) & 0xffffff; - if (members_count == dev->caps.num_qp_per_mgm) { - mlx4_err(dev, "MGM at index %x is full.\n", index); - err = -ENOMEM; - goto out; - } - - for (i = 0; i < members_count; ++i) - if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qp->qpn) { - mlx4_dbg(dev, "QP %06x already a member of MGM\n", qp->qpn); - err = 0; - goto out; - } - - mgm->qp[members_count++] = cpu_to_be32((qp->qpn & MGM_QPN_MASK) | - (!!mlx4_blck_lb << MGM_BLCK_LB_BIT)); - - mgm->members_count = cpu_to_be32(members_count | (u32) prot << 30); - - err = mlx4_WRITE_ENTRY(dev, index, mailbox); - if (err) - goto out; - - /* if !link, still add the new entry. */ - if (!link) - goto skip_link; - - err = mlx4_READ_ENTRY(dev, prev, mailbox); - if (err) - goto out; - - mgm->next_gid_index = cpu_to_be32(index << 6); - - err = mlx4_WRITE_ENTRY(dev, prev, mailbox); - if (err) - goto out; - -skip_link: - if (prot == MLX4_PROT_ETH) { - /* manage the steering entry for promisc mode */ - if (new_entry) - new_steering_entry(dev, port, steer, index, qp->qpn); - else - existing_steering_entry(dev, port, steer, - index, qp->qpn); - } - -out: - if (err && link && index != -1) { - if (index < dev->caps.num_mgms) - mlx4_warn(dev, "Got AMGM index %d < %d", - index, dev->caps.num_mgms); - else - mlx4_bitmap_free(&priv->mcg_table.bitmap, - index - dev->caps.num_mgms, MLX4_USE_RR); - } - mutex_unlock(&priv->mcg_table.mutex); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - -int mlx4_qp_detach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], - enum mlx4_protocol prot, enum mlx4_steer_type steer) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_mgm *mgm; - u32 members_count; - int prev, index; - int i, loc = -1; - int err; - u8 port = gid[5]; - bool removed_entry = false; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - mgm = mailbox->buf; - - mutex_lock(&priv->mcg_table.mutex); - - err = find_entry(dev, port, gid, prot, - mailbox, &prev, &index); - if (err) - goto out; - - if (index == -1) { - mlx4_err(dev, "MGID "GID_PRINT_FMT" not found\n", - GID_PRINT_ARGS(gid)); - err = -EINVAL; - goto out; - } - - /* - if this QP is also a promisc QP, it shouldn't be removed only if - at least one none promisc QP is also attached to this MCG - */ - if (prot == MLX4_PROT_ETH && - check_duplicate_entry(dev, port, steer, index, qp->qpn) && - !promisc_steering_entry(dev, port, steer, index, qp->qpn, NULL)) - goto out; - - members_count = be32_to_cpu(mgm->members_count) & 0xffffff; - for (i = 0; i < members_count; ++i) - if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qp->qpn) { - loc = i; - break; - } - - if (loc == -1) { - mlx4_err(dev, "QP %06x not found in MGM\n", qp->qpn); - err = -EINVAL; - goto out; - } - - /* copy the last QP in this MGM over removed QP */ - mgm->qp[loc] = mgm->qp[members_count - 1]; - mgm->qp[members_count - 1] = 0; - mgm->members_count = cpu_to_be32(--members_count | (u32) prot << 30); - - if (prot == MLX4_PROT_ETH) - removed_entry = can_remove_steering_entry(dev, port, steer, - index, qp->qpn); - if (members_count && (prot != MLX4_PROT_ETH || !removed_entry)) { - err = mlx4_WRITE_ENTRY(dev, index, mailbox); - goto out; - } - - /* We are going to delete the entry, members count should be 0 */ - mgm->members_count = cpu_to_be32((u32) prot << 30); - - if (prev == -1) { - /* Remove entry from MGM */ - int amgm_index = be32_to_cpu(mgm->next_gid_index) >> 6; - if (amgm_index) { - err = mlx4_READ_ENTRY(dev, amgm_index, mailbox); - if (err) - goto out; - } else - memset(mgm->gid, 0, 16); - - err = mlx4_WRITE_ENTRY(dev, index, mailbox); - if (err) - goto out; - - if (amgm_index) { - if (amgm_index < dev->caps.num_mgms) - mlx4_warn(dev, "MGM entry %d had AMGM index %d < %d", - index, amgm_index, dev->caps.num_mgms); - else - mlx4_bitmap_free(&priv->mcg_table.bitmap, - amgm_index - dev->caps.num_mgms, MLX4_USE_RR); - } - } else { - /* Remove entry from AMGM */ - int cur_next_index = be32_to_cpu(mgm->next_gid_index) >> 6; - err = mlx4_READ_ENTRY(dev, prev, mailbox); - if (err) - goto out; - - mgm->next_gid_index = cpu_to_be32(cur_next_index << 6); - - err = mlx4_WRITE_ENTRY(dev, prev, mailbox); - if (err) - goto out; - - if (index < dev->caps.num_mgms) - mlx4_warn(dev, "entry %d had next AMGM index %d < %d", - prev, index, dev->caps.num_mgms); - else - mlx4_bitmap_free(&priv->mcg_table.bitmap, - index - dev->caps.num_mgms, MLX4_USE_RR); - } - -out: - mutex_unlock(&priv->mcg_table.mutex); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - -static int mlx4_QP_ATTACH(struct mlx4_dev *dev, struct mlx4_qp *qp, - u8 gid[16], u8 attach, u8 block_loopback, - enum mlx4_protocol prot) -{ - struct mlx4_cmd_mailbox *mailbox; - int err = 0; - int qpn; - - if (!mlx4_is_mfunc(dev)) - return -EBADF; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - memcpy(mailbox->buf, gid, 16); - qpn = qp->qpn; - qpn |= (prot << 28); - if (attach && block_loopback) - qpn |= (1 << 31); - - err = mlx4_cmd(dev, mailbox->dma, qpn, attach, - MLX4_CMD_QP_ATTACH, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_WRAPPED); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - -int mlx4_trans_to_dmfs_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, - u8 gid[16], u8 port, - int block_mcast_loopback, - enum mlx4_protocol prot, u64 *reg_id) -{ - struct mlx4_spec_list spec = { {NULL} }; - __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); - - struct mlx4_net_trans_rule rule = { - .queue_mode = MLX4_NET_TRANS_Q_FIFO, - .exclusive = 0, - .promisc_mode = MLX4_FS_REGULAR, - .priority = MLX4_DOMAIN_NIC, - }; - - rule.allow_loopback = !block_mcast_loopback; - rule.port = port; - rule.qpn = qp->qpn; - INIT_LIST_HEAD(&rule.list); - - switch (prot) { - case MLX4_PROT_ETH: - spec.id = MLX4_NET_TRANS_RULE_ID_ETH; - memcpy(spec.eth.dst_mac, &gid[10], ETH_ALEN); - memcpy(spec.eth.dst_mac_msk, &mac_mask, ETH_ALEN); - break; - - case MLX4_PROT_IB_IPV6: - spec.id = MLX4_NET_TRANS_RULE_ID_IB; - memcpy(spec.ib.dst_gid, gid, 16); - memset(&spec.ib.dst_gid_msk, 0xff, 16); - break; - default: - return -EINVAL; - } - list_add_tail(&spec.list, &rule.list); - - return mlx4_flow_attach(dev, &rule, reg_id); -} - -int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], - u8 port, int block_mcast_loopback, - enum mlx4_protocol prot, u64 *reg_id) -{ - enum mlx4_steer_type steer; - steer = (is_valid_ether_addr(&gid[10])) ? MLX4_UC_STEER : MLX4_MC_STEER; - - switch (dev->caps.steering_mode) { - case MLX4_STEERING_MODE_A0: - if (prot == MLX4_PROT_ETH) - return 0; - - case MLX4_STEERING_MODE_B0: - if (prot == MLX4_PROT_ETH) - gid[7] |= (steer << 1); - - if (mlx4_is_mfunc(dev)) - return mlx4_QP_ATTACH(dev, qp, gid, 1, - block_mcast_loopback, prot); - return mlx4_qp_attach_common(dev, qp, gid, - block_mcast_loopback, prot, - MLX4_MC_STEER); - - case MLX4_STEERING_MODE_DEVICE_MANAGED: - return mlx4_trans_to_dmfs_attach(dev, qp, gid, port, - block_mcast_loopback, - prot, reg_id); - default: - return -EINVAL; - } -} -EXPORT_SYMBOL_GPL(mlx4_multicast_attach); - -int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], - enum mlx4_protocol prot, u64 reg_id) -{ - enum mlx4_steer_type steer; - steer = (is_valid_ether_addr(&gid[10])) ? MLX4_UC_STEER : MLX4_MC_STEER; - - switch (dev->caps.steering_mode) { - case MLX4_STEERING_MODE_A0: - if (prot == MLX4_PROT_ETH) - return 0; - - case MLX4_STEERING_MODE_B0: - if (prot == MLX4_PROT_ETH) - gid[7] |= (steer << 1); - - if (mlx4_is_mfunc(dev)) - return mlx4_QP_ATTACH(dev, qp, gid, 0, 0, prot); - - return mlx4_qp_detach_common(dev, qp, gid, prot, - MLX4_MC_STEER); - - case MLX4_STEERING_MODE_DEVICE_MANAGED: - return mlx4_flow_detach(dev, reg_id); - - default: - return -EINVAL; - } -} -EXPORT_SYMBOL_GPL(mlx4_multicast_detach); - -int mlx4_flow_steer_promisc_add(struct mlx4_dev *dev, u8 port, - u32 qpn, enum mlx4_net_trans_promisc_mode mode) -{ - struct mlx4_net_trans_rule rule; - u64 *regid_p; - - switch (mode) { - case MLX4_FS_ALL_DEFAULT: - regid_p = &dev->regid_promisc_array[port]; - break; - case MLX4_FS_MC_DEFAULT: - regid_p = &dev->regid_allmulti_array[port]; - break; - default: - return -1; - } - - if (*regid_p != 0) - return -1; - - rule.promisc_mode = mode; - rule.port = port; - rule.qpn = qpn; - INIT_LIST_HEAD(&rule.list); - mlx4_err(dev, "going promisc on %x\n", port); - - return mlx4_flow_attach(dev, &rule, regid_p); -} -EXPORT_SYMBOL_GPL(mlx4_flow_steer_promisc_add); - -int mlx4_flow_steer_promisc_remove(struct mlx4_dev *dev, u8 port, - enum mlx4_net_trans_promisc_mode mode) -{ - int ret; - u64 *regid_p; - - switch (mode) { - case MLX4_FS_ALL_DEFAULT: - regid_p = &dev->regid_promisc_array[port]; - break; - case MLX4_FS_MC_DEFAULT: - regid_p = &dev->regid_allmulti_array[port]; - break; - default: - return -1; - } - - if (*regid_p == 0) - return -1; - - ret = mlx4_flow_detach(dev, *regid_p); - if (ret == 0) - *regid_p = 0; - - return ret; -} -EXPORT_SYMBOL_GPL(mlx4_flow_steer_promisc_remove); - -int mlx4_unicast_attach(struct mlx4_dev *dev, - struct mlx4_qp *qp, u8 gid[16], - int block_mcast_loopback, enum mlx4_protocol prot) -{ - if (prot == MLX4_PROT_ETH) - gid[7] |= (MLX4_UC_STEER << 1); - - if (mlx4_is_mfunc(dev)) - return mlx4_QP_ATTACH(dev, qp, gid, 1, - block_mcast_loopback, prot); - - return mlx4_qp_attach_common(dev, qp, gid, block_mcast_loopback, - prot, MLX4_UC_STEER); -} -EXPORT_SYMBOL_GPL(mlx4_unicast_attach); - -int mlx4_unicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, - u8 gid[16], enum mlx4_protocol prot) -{ - if (prot == MLX4_PROT_ETH) - gid[7] |= (MLX4_UC_STEER << 1); - - if (mlx4_is_mfunc(dev)) - return mlx4_QP_ATTACH(dev, qp, gid, 0, 0, prot); - - return mlx4_qp_detach_common(dev, qp, gid, prot, MLX4_UC_STEER); -} -EXPORT_SYMBOL_GPL(mlx4_unicast_detach); - -int mlx4_PROMISC_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - u32 qpn = (u32) vhcr->in_param & 0xffffffff; - u8 port = vhcr->in_param >> 62; - enum mlx4_steer_type steer = vhcr->in_modifier; - - /* Promiscuous unicast is not allowed in mfunc for VFs */ - if ((slave != dev->caps.function) && (steer == MLX4_UC_STEER)) - return 0; - - if (vhcr->op_modifier) - return add_promisc_qp(dev, port, steer, qpn); - else - return remove_promisc_qp(dev, port, steer, qpn); -} - -static int mlx4_PROMISC(struct mlx4_dev *dev, u32 qpn, - enum mlx4_steer_type steer, u8 add, u8 port) -{ - return mlx4_cmd(dev, (u64) qpn | (u64) port << 62, (u32) steer, add, - MLX4_CMD_PROMISC, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_WRAPPED); -} - -int mlx4_multicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port) -{ - if (mlx4_is_mfunc(dev)) - return mlx4_PROMISC(dev, qpn, MLX4_MC_STEER, 1, port); - - return add_promisc_qp(dev, port, MLX4_MC_STEER, qpn); -} -EXPORT_SYMBOL_GPL(mlx4_multicast_promisc_add); - -int mlx4_multicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port) -{ - if (mlx4_is_mfunc(dev)) - return mlx4_PROMISC(dev, qpn, MLX4_MC_STEER, 0, port); - - return remove_promisc_qp(dev, port, MLX4_MC_STEER, qpn); -} -EXPORT_SYMBOL_GPL(mlx4_multicast_promisc_remove); - -int mlx4_unicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port) -{ - if (mlx4_is_mfunc(dev)) - return mlx4_PROMISC(dev, qpn, MLX4_UC_STEER, 1, port); - - return add_promisc_qp(dev, port, MLX4_UC_STEER, qpn); -} -EXPORT_SYMBOL_GPL(mlx4_unicast_promisc_add); - -int mlx4_unicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port) -{ - if (mlx4_is_mfunc(dev)) - return mlx4_PROMISC(dev, qpn, MLX4_UC_STEER, 0, port); - - return remove_promisc_qp(dev, port, MLX4_UC_STEER, qpn); -} -EXPORT_SYMBOL_GPL(mlx4_unicast_promisc_remove); - -int mlx4_init_mcg_table(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int err; - - /* No need for mcg_table when fw managed the mcg table*/ - if (dev->caps.steering_mode == - MLX4_STEERING_MODE_DEVICE_MANAGED) - return 0; - err = mlx4_bitmap_init(&priv->mcg_table.bitmap, dev->caps.num_amgms, - dev->caps.num_amgms - 1, 0, 0); - if (err) - return err; - - mutex_init(&priv->mcg_table.mutex); - - return 0; -} - -void mlx4_cleanup_mcg_table(struct mlx4_dev *dev) -{ - if (dev->caps.steering_mode != - MLX4_STEERING_MODE_DEVICE_MANAGED) - mlx4_bitmap_cleanup(&mlx4_priv(dev)->mcg_table.bitmap); -} Property changes on: stable/11/sys/ofed/drivers/net/mlx4/mcg.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/mr.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/mr.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/mr.c (nonexistent) @@ -1,996 +0,0 @@ -/* - * Copyright (c) 2004 Topspin Communications. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include "mlx4.h" -#include "icm.h" - -static u32 mlx4_buddy_alloc(struct mlx4_buddy *buddy, int order) -{ - int o; - int m; - u32 seg; - - spin_lock(&buddy->lock); - - for (o = order; o <= buddy->max_order; ++o) - if (buddy->num_free[o]) { - m = 1 << (buddy->max_order - o); - seg = find_first_bit(buddy->bits[o], m); - if (seg < m) - goto found; - } - - spin_unlock(&buddy->lock); - return -1; - - found: - clear_bit(seg, buddy->bits[o]); - --buddy->num_free[o]; - - while (o > order) { - --o; - seg <<= 1; - set_bit(seg ^ 1, buddy->bits[o]); - ++buddy->num_free[o]; - } - - spin_unlock(&buddy->lock); - - seg <<= order; - - return seg; -} - -static void mlx4_buddy_free(struct mlx4_buddy *buddy, u32 seg, int order) -{ - seg >>= order; - - spin_lock(&buddy->lock); - - while (test_bit(seg ^ 1, buddy->bits[order])) { - clear_bit(seg ^ 1, buddy->bits[order]); - --buddy->num_free[order]; - seg >>= 1; - ++order; - } - - set_bit(seg, buddy->bits[order]); - ++buddy->num_free[order]; - - spin_unlock(&buddy->lock); -} - -static int mlx4_buddy_init(struct mlx4_buddy *buddy, int max_order) -{ - int i, s; - - buddy->max_order = max_order; - spin_lock_init(&buddy->lock); - - buddy->bits = kcalloc(buddy->max_order + 1, sizeof (long *), - GFP_KERNEL); - buddy->num_free = kcalloc((buddy->max_order + 1), sizeof *buddy->num_free, - GFP_KERNEL); - if (!buddy->bits || !buddy->num_free) - goto err_out; - - for (i = 0; i <= buddy->max_order; ++i) { - s = BITS_TO_LONGS(1 << (buddy->max_order - i)); - buddy->bits[i] = kcalloc(s, sizeof (long), GFP_KERNEL | __GFP_NOWARN); - if (!buddy->bits[i]) - goto err_out_free; - } - - set_bit(0, buddy->bits[buddy->max_order]); - buddy->num_free[buddy->max_order] = 1; - - return 0; - -err_out_free: - for (i = 0; i <= buddy->max_order; ++i) - kfree(buddy->bits[i]); - -err_out: - kfree(buddy->bits); - kfree(buddy->num_free); - - return -ENOMEM; -} - -static void mlx4_buddy_cleanup(struct mlx4_buddy *buddy) -{ - int i; - - for (i = 0; i <= buddy->max_order; ++i) - kfree(buddy->bits[i]); - - kfree(buddy->bits); - kfree(buddy->num_free); -} - -u32 __mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) -{ - struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; - u32 seg; - int seg_order; - u32 offset; - - seg_order = max_t(int, order - log_mtts_per_seg, 0); - - seg = mlx4_buddy_alloc(&mr_table->mtt_buddy, seg_order); - if (seg == -1) - return -1; - - offset = seg * (1 << log_mtts_per_seg); - - if (mlx4_table_get_range(dev, &mr_table->mtt_table, offset, - offset + (1 << order) - 1)) { - mlx4_buddy_free(&mr_table->mtt_buddy, seg, seg_order); - return -1; - } - - return offset; -} - -static u32 mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) -{ - u64 in_param = 0; - u64 out_param; - int err; - - if (mlx4_is_mfunc(dev)) { - set_param_l(&in_param, order); - err = mlx4_cmd_imm(dev, in_param, &out_param, RES_MTT, - RES_OP_RESERVE_AND_MAP, - MLX4_CMD_ALLOC_RES, - MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_WRAPPED); - if (err) - return -1; - return get_param_l(&out_param); - } - return __mlx4_alloc_mtt_range(dev, order); -} - -int mlx4_mtt_init(struct mlx4_dev *dev, int npages, int page_shift, - struct mlx4_mtt *mtt) -{ - int i; - - if (!npages) { - mtt->order = -1; - mtt->page_shift = MLX4_ICM_PAGE_SHIFT; - return 0; - } else - mtt->page_shift = page_shift; - - for (mtt->order = 0, i = 1; i < npages; i <<= 1) - ++mtt->order; - - mtt->offset = mlx4_alloc_mtt_range(dev, mtt->order); - if (mtt->offset == -1) { - mlx4_err(dev, "Failed to allocate mtts for %d pages(order %d)\n", - npages, mtt->order); - return -ENOMEM; - } - - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_mtt_init); - -void __mlx4_free_mtt_range(struct mlx4_dev *dev, u32 offset, int order) -{ - u32 first_seg; - int seg_order; - struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; - - seg_order = max_t(int, order - log_mtts_per_seg, 0); - first_seg = offset / (1 << log_mtts_per_seg); - - mlx4_buddy_free(&mr_table->mtt_buddy, first_seg, seg_order); - mlx4_table_put_range(dev, &mr_table->mtt_table, offset, - offset + (1 << order) - 1); -} - -static void mlx4_free_mtt_range(struct mlx4_dev *dev, u32 offset, int order) -{ - u64 in_param = 0; - int err; - - if (mlx4_is_mfunc(dev)) { - set_param_l(&in_param, offset); - set_param_h(&in_param, order); - err = mlx4_cmd(dev, in_param, RES_MTT, RES_OP_RESERVE_AND_MAP, - MLX4_CMD_FREE_RES, - MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_WRAPPED); - if (err) - mlx4_warn(dev, "Failed to free mtt range at:" - "%d order:%d\n", offset, order); - return; - } - __mlx4_free_mtt_range(dev, offset, order); -} - -void mlx4_mtt_cleanup(struct mlx4_dev *dev, struct mlx4_mtt *mtt) -{ - if (mtt->order < 0) - return; - - mlx4_free_mtt_range(dev, mtt->offset, mtt->order); -} -EXPORT_SYMBOL_GPL(mlx4_mtt_cleanup); - -u64 mlx4_mtt_addr(struct mlx4_dev *dev, struct mlx4_mtt *mtt) -{ - return (u64) mtt->offset * dev->caps.mtt_entry_sz; -} -EXPORT_SYMBOL_GPL(mlx4_mtt_addr); - -static u32 hw_index_to_key(u32 ind) -{ - return (ind >> 24) | (ind << 8); -} - -static u32 key_to_hw_index(u32 key) -{ - return (key << 24) | (key >> 8); -} - -static int mlx4_SW2HW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, - int mpt_index) -{ - return mlx4_cmd(dev, mailbox->dma, mpt_index, - 0, MLX4_CMD_SW2HW_MPT, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_WRAPPED); -} - -static int mlx4_HW2SW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, - int mpt_index) -{ - return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, mpt_index, - !mailbox, MLX4_CMD_HW2SW_MPT, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); -} - -static int mlx4_mr_alloc_reserved(struct mlx4_dev *dev, u32 mridx, u32 pd, - u64 iova, u64 size, u32 access, int npages, - int page_shift, struct mlx4_mr *mr) -{ - mr->iova = iova; - mr->size = size; - mr->pd = pd; - mr->access = access; - mr->enabled = MLX4_MPT_DISABLED; - mr->key = hw_index_to_key(mridx); - - return mlx4_mtt_init(dev, npages, page_shift, &mr->mtt); -} - -static int mlx4_WRITE_MTT(struct mlx4_dev *dev, - struct mlx4_cmd_mailbox *mailbox, - int num_entries) -{ - return mlx4_cmd(dev, mailbox->dma, num_entries, 0, MLX4_CMD_WRITE_MTT, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); -} - -int __mlx4_mpt_reserve(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - return mlx4_bitmap_alloc(&priv->mr_table.mpt_bitmap); -} - -static int mlx4_mpt_reserve(struct mlx4_dev *dev) -{ - u64 out_param; - - if (mlx4_is_mfunc(dev)) { - if (mlx4_cmd_imm(dev, 0, &out_param, RES_MPT, RES_OP_RESERVE, - MLX4_CMD_ALLOC_RES, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) - return -1; - return get_param_l(&out_param); - } - return __mlx4_mpt_reserve(dev); -} - -void __mlx4_mpt_release(struct mlx4_dev *dev, u32 index) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - mlx4_bitmap_free(&priv->mr_table.mpt_bitmap, index, MLX4_NO_RR); -} - -static void mlx4_mpt_release(struct mlx4_dev *dev, u32 index) -{ - u64 in_param = 0; - - if (mlx4_is_mfunc(dev)) { - set_param_l(&in_param, index); - if (mlx4_cmd(dev, in_param, RES_MPT, RES_OP_RESERVE, - MLX4_CMD_FREE_RES, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) - mlx4_warn(dev, "Failed to release mr index:%d\n", - index); - return; - } - __mlx4_mpt_release(dev, index); -} - -int __mlx4_mpt_alloc_icm(struct mlx4_dev *dev, u32 index) -{ - struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; - - return mlx4_table_get(dev, &mr_table->dmpt_table, index); -} - -static int mlx4_mpt_alloc_icm(struct mlx4_dev *dev, u32 index) -{ - u64 param = 0; - - if (mlx4_is_mfunc(dev)) { - set_param_l(¶m, index); - return mlx4_cmd_imm(dev, param, ¶m, RES_MPT, RES_OP_MAP_ICM, - MLX4_CMD_ALLOC_RES, - MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_WRAPPED); - } - return __mlx4_mpt_alloc_icm(dev, index); -} - -void __mlx4_mpt_free_icm(struct mlx4_dev *dev, u32 index) -{ - struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; - - mlx4_table_put(dev, &mr_table->dmpt_table, index); -} - -static void mlx4_mpt_free_icm(struct mlx4_dev *dev, u32 index) -{ - u64 in_param = 0; - - if (mlx4_is_mfunc(dev)) { - set_param_l(&in_param, index); - if (mlx4_cmd(dev, in_param, RES_MPT, RES_OP_MAP_ICM, - MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_WRAPPED)) - mlx4_warn(dev, "Failed to free icm of mr index:%d\n", - index); - return; - } - return __mlx4_mpt_free_icm(dev, index); -} - -int mlx4_mr_alloc(struct mlx4_dev *dev, u32 pd, u64 iova, u64 size, u32 access, - int npages, int page_shift, struct mlx4_mr *mr) -{ - u32 index; - int err; - - index = mlx4_mpt_reserve(dev); - if (index == -1) - return -ENOMEM; - - err = mlx4_mr_alloc_reserved(dev, index, pd, iova, size, - access, npages, page_shift, mr); - if (err) - mlx4_mpt_release(dev, index); - - return err; -} -EXPORT_SYMBOL_GPL(mlx4_mr_alloc); - -static int mlx4_mr_free_reserved(struct mlx4_dev *dev, struct mlx4_mr *mr) -{ - int err; - - if (mr->enabled == MLX4_MPT_EN_HW) { - err = mlx4_HW2SW_MPT(dev, NULL, - key_to_hw_index(mr->key) & - (dev->caps.num_mpts - 1)); - if (err) { - mlx4_warn(dev, "HW2SW_MPT failed (%d).", err); - mlx4_warn(dev, "Most likely the MR has MWs bound to it.\n"); - return err; - } - - mr->enabled = MLX4_MPT_EN_SW; - } - mlx4_mtt_cleanup(dev, &mr->mtt); - - return 0; -} - -int mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr) -{ - int ret; - - ret = mlx4_mr_free_reserved(dev, mr); - if (ret) - return ret; - if (mr->enabled) - mlx4_mpt_free_icm(dev, key_to_hw_index(mr->key)); - mlx4_mpt_release(dev, key_to_hw_index(mr->key)); - - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_mr_free); - -int mlx4_mr_enable(struct mlx4_dev *dev, struct mlx4_mr *mr) -{ - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_mpt_entry *mpt_entry; - int err; - - err = mlx4_mpt_alloc_icm(dev, key_to_hw_index(mr->key)); - if (err) - return err; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) { - err = PTR_ERR(mailbox); - goto err_table; - } - mpt_entry = mailbox->buf; - - memset(mpt_entry, 0, sizeof *mpt_entry); - - mpt_entry->flags = cpu_to_be32(MLX4_MPT_FLAG_MIO | - MLX4_MPT_FLAG_REGION | - mr->access); - - mpt_entry->key = cpu_to_be32(key_to_hw_index(mr->key)); - mpt_entry->pd_flags = cpu_to_be32(mr->pd | MLX4_MPT_PD_FLAG_EN_INV); - mpt_entry->start = cpu_to_be64(mr->iova); - mpt_entry->length = cpu_to_be64(mr->size); - mpt_entry->entity_size = cpu_to_be32(mr->mtt.page_shift); - - if (mr->mtt.order < 0) { - mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_PHYSICAL); - mpt_entry->mtt_addr = 0; - } else { - mpt_entry->mtt_addr = cpu_to_be64(mlx4_mtt_addr(dev, - &mr->mtt)); - } - - if (mr->mtt.order >= 0 && mr->mtt.page_shift == 0) { - /* fast register MR in free state */ - mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_FREE); - mpt_entry->pd_flags |= cpu_to_be32(MLX4_MPT_PD_FLAG_FAST_REG | - MLX4_MPT_PD_FLAG_RAE); - mpt_entry->mtt_sz = cpu_to_be32(1 << mr->mtt.order); - } else { - mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_SW_OWNS); - } - - err = mlx4_SW2HW_MPT(dev, mailbox, - key_to_hw_index(mr->key) & (dev->caps.num_mpts - 1)); - if (err) { - mlx4_warn(dev, "SW2HW_MPT failed (%d)\n", err); - goto err_cmd; - } - mr->enabled = MLX4_MPT_EN_HW; - - mlx4_free_cmd_mailbox(dev, mailbox); - - return 0; - -err_cmd: - mlx4_free_cmd_mailbox(dev, mailbox); - -err_table: - mlx4_mpt_free_icm(dev, key_to_hw_index(mr->key)); - return err; -} -EXPORT_SYMBOL_GPL(mlx4_mr_enable); - -static int mlx4_write_mtt_chunk(struct mlx4_dev *dev, struct mlx4_mtt *mtt, - int start_index, int npages, u64 *page_list) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - __be64 *mtts; - dma_addr_t dma_handle; - int i; - - mtts = mlx4_table_find(&priv->mr_table.mtt_table, mtt->offset + - start_index, &dma_handle); - - if (!mtts) - return -ENOMEM; - - dma_sync_single_for_cpu(&dev->pdev->dev, dma_handle, - npages * sizeof (u64), DMA_TO_DEVICE); - - for (i = 0; i < npages; ++i) - mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT); - - dma_sync_single_for_device(&dev->pdev->dev, dma_handle, - npages * sizeof (u64), DMA_TO_DEVICE); - - return 0; -} - -int __mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, - int start_index, int npages, u64 *page_list) -{ - int err = 0; - int chunk; - int mtts_per_page; - int max_mtts_first_page; - - /* compute how may mtts fit in the first page */ - mtts_per_page = PAGE_SIZE / sizeof(u64); - max_mtts_first_page = mtts_per_page - (mtt->offset + start_index) - % mtts_per_page; - - chunk = min_t(int, max_mtts_first_page, npages); - - while (npages > 0) { - err = mlx4_write_mtt_chunk(dev, mtt, start_index, chunk, page_list); - if (err) - return err; - npages -= chunk; - start_index += chunk; - page_list += chunk; - - chunk = min_t(int, mtts_per_page, npages); - } - return err; -} - -int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, - int start_index, int npages, u64 *page_list) -{ - struct mlx4_cmd_mailbox *mailbox = NULL; - __be64 *inbox = NULL; - int chunk; - int err = 0; - int i; - - if (mtt->order < 0) - return -EINVAL; - - if (mlx4_is_mfunc(dev)) { - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - inbox = mailbox->buf; - - while (npages > 0) { - chunk = min_t(int, MLX4_MAILBOX_SIZE / sizeof(u64) - 2, - npages); - inbox[0] = cpu_to_be64(mtt->offset + start_index); - inbox[1] = 0; - for (i = 0; i < chunk; ++i) - inbox[i + 2] = cpu_to_be64(page_list[i] | - MLX4_MTT_FLAG_PRESENT); - err = mlx4_WRITE_MTT(dev, mailbox, chunk); - if (err) { - mlx4_free_cmd_mailbox(dev, mailbox); - return err; - } - - npages -= chunk; - start_index += chunk; - page_list += chunk; - } - mlx4_free_cmd_mailbox(dev, mailbox); - return err; - } - - return __mlx4_write_mtt(dev, mtt, start_index, npages, page_list); -} -EXPORT_SYMBOL_GPL(mlx4_write_mtt); - -int mlx4_buf_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, - struct mlx4_buf *buf) -{ - u64 *page_list; - int err; - int i; - - page_list = kmalloc(buf->npages * sizeof *page_list, GFP_KERNEL); - if (!page_list) - return -ENOMEM; - - for (i = 0; i < buf->npages; ++i) - if (buf->nbufs == 1) - page_list[i] = buf->direct.map + (i << buf->page_shift); - else - page_list[i] = buf->page_list[i].map; - - err = mlx4_write_mtt(dev, mtt, 0, buf->npages, page_list); - - kfree(page_list); - return err; -} -EXPORT_SYMBOL_GPL(mlx4_buf_write_mtt); - -int mlx4_mw_alloc(struct mlx4_dev *dev, u32 pd, enum mlx4_mw_type type, - struct mlx4_mw *mw) -{ - u32 index; - - index = mlx4_mpt_reserve(dev); - if (index == -1) - return -ENOMEM; - - mw->key = hw_index_to_key(index); - mw->pd = pd; - mw->type = type; - mw->enabled = MLX4_MPT_DISABLED; - - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_mw_alloc); - -int mlx4_mw_enable(struct mlx4_dev *dev, struct mlx4_mw *mw) -{ - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_mpt_entry *mpt_entry; - int err; - - err = mlx4_mpt_alloc_icm(dev, key_to_hw_index(mw->key)); - if (err) - return err; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) { - err = PTR_ERR(mailbox); - goto err_table; - } - mpt_entry = mailbox->buf; - - memset(mpt_entry, 0, sizeof(*mpt_entry)); - - /* Note that the MLX4_MPT_FLAG_REGION bit in mpt_entry->flags is turned - * off, thus creating a memory window and not a memory region. - */ - mpt_entry->key = cpu_to_be32(key_to_hw_index(mw->key)); - mpt_entry->pd_flags = cpu_to_be32(mw->pd); - if (mw->type == MLX4_MW_TYPE_2) { - mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_FREE); - mpt_entry->qpn = cpu_to_be32(MLX4_MPT_QP_FLAG_BOUND_QP); - mpt_entry->pd_flags |= cpu_to_be32(MLX4_MPT_PD_FLAG_EN_INV); - } - - err = mlx4_SW2HW_MPT(dev, mailbox, - key_to_hw_index(mw->key) & - (dev->caps.num_mpts - 1)); - if (err) { - mlx4_warn(dev, "SW2HW_MPT failed (%d)\n", err); - goto err_cmd; - } - mw->enabled = MLX4_MPT_EN_HW; - - mlx4_free_cmd_mailbox(dev, mailbox); - - return 0; - -err_cmd: - mlx4_free_cmd_mailbox(dev, mailbox); - -err_table: - mlx4_mpt_free_icm(dev, key_to_hw_index(mw->key)); - return err; -} -EXPORT_SYMBOL_GPL(mlx4_mw_enable); - -void mlx4_mw_free(struct mlx4_dev *dev, struct mlx4_mw *mw) -{ - int err; - - if (mw->enabled == MLX4_MPT_EN_HW) { - err = mlx4_HW2SW_MPT(dev, NULL, - key_to_hw_index(mw->key) & - (dev->caps.num_mpts - 1)); - if (err) - mlx4_warn(dev, "xxx HW2SW_MPT failed (%d)\n", err); - - mw->enabled = MLX4_MPT_EN_SW; - } - if (mw->enabled) - mlx4_mpt_free_icm(dev, key_to_hw_index(mw->key)); - mlx4_mpt_release(dev, key_to_hw_index(mw->key)); -} -EXPORT_SYMBOL_GPL(mlx4_mw_free); - -int mlx4_init_mr_table(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_mr_table *mr_table = &priv->mr_table; - int err; - - /* Nothing to do for slaves - all MR handling is forwarded - * to the master */ - if (mlx4_is_slave(dev)) - return 0; - - if (!is_power_of_2(dev->caps.num_mpts)) - return -EINVAL; - - err = mlx4_bitmap_init(&mr_table->mpt_bitmap, dev->caps.num_mpts, - ~0, dev->caps.reserved_mrws, 0); - if (err) - return err; - - err = mlx4_buddy_init(&mr_table->mtt_buddy, - ilog2(div_u64(dev->caps.num_mtts, - (1 << log_mtts_per_seg)))); - if (err) - goto err_buddy; - - if (dev->caps.reserved_mtts) { - priv->reserved_mtts = - mlx4_alloc_mtt_range(dev, - fls(dev->caps.reserved_mtts - 1)); - if (priv->reserved_mtts < 0) { - mlx4_warn(dev, "MTT table of order %u is too small.\n", - mr_table->mtt_buddy.max_order); - err = -ENOMEM; - goto err_reserve_mtts; - } - } - - return 0; - -err_reserve_mtts: - mlx4_buddy_cleanup(&mr_table->mtt_buddy); - -err_buddy: - mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); - - return err; -} - -void mlx4_cleanup_mr_table(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_mr_table *mr_table = &priv->mr_table; - - if (mlx4_is_slave(dev)) - return; - if (priv->reserved_mtts >= 0) - mlx4_free_mtt_range(dev, priv->reserved_mtts, - fls(dev->caps.reserved_mtts - 1)); - mlx4_buddy_cleanup(&mr_table->mtt_buddy); - mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); -} - -static inline int mlx4_check_fmr(struct mlx4_fmr *fmr, u64 *page_list, - int npages, u64 iova) -{ - int i, page_mask; - - if (npages > fmr->max_pages) - return -EINVAL; - - page_mask = (1 << fmr->page_shift) - 1; - - /* We are getting page lists, so va must be page aligned. */ - if (iova & page_mask) - return -EINVAL; - - /* Trust the user not to pass misaligned data in page_list */ - if (0) - for (i = 0; i < npages; ++i) { - if (page_list[i] & ~page_mask) - return -EINVAL; - } - - if (fmr->maps >= fmr->max_maps) - return -EINVAL; - - return 0; -} - -int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list, - int npages, u64 iova, u32 *lkey, u32 *rkey) -{ - u32 key; - int i, err; - - err = mlx4_check_fmr(fmr, page_list, npages, iova); - if (err) - return err; - - ++fmr->maps; - - key = key_to_hw_index(fmr->mr.key); - key += dev->caps.num_mpts; - *lkey = *rkey = fmr->mr.key = hw_index_to_key(key); - - *(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW; - - /* Make sure MPT status is visible before writing MTT entries */ - wmb(); - - dma_sync_single_for_cpu(&dev->pdev->dev, fmr->dma_handle, - npages * sizeof(u64), DMA_TO_DEVICE); - - for (i = 0; i < npages; ++i) - fmr->mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT); - - dma_sync_single_for_device(&dev->pdev->dev, fmr->dma_handle, - npages * sizeof(u64), DMA_TO_DEVICE); - - fmr->mpt->key = cpu_to_be32(key); - fmr->mpt->lkey = cpu_to_be32(key); - fmr->mpt->length = cpu_to_be64(npages * (1ull << fmr->page_shift)); - fmr->mpt->start = cpu_to_be64(iova); - - /* Make MTT entries are visible before setting MPT status */ - wmb(); - - *(u8 *) fmr->mpt = MLX4_MPT_STATUS_HW; - - /* Make sure MPT status is visible before consumer can use FMR */ - wmb(); - - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_map_phys_fmr); - -int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages, - int max_maps, u8 page_shift, struct mlx4_fmr *fmr) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int err = -ENOMEM, ret; - - if (max_maps > dev->caps.max_fmr_maps) - return -EINVAL; - - if (page_shift < (ffs(dev->caps.page_size_cap) - 1) || page_shift >= 32) - return -EINVAL; - - /* All MTTs must fit in the same page */ - if (max_pages * sizeof *fmr->mtts > PAGE_SIZE) - return -EINVAL; - - fmr->page_shift = page_shift; - fmr->max_pages = max_pages; - fmr->max_maps = max_maps; - fmr->maps = 0; - - err = mlx4_mr_alloc(dev, pd, 0, 0, access, max_pages, - page_shift, &fmr->mr); - if (err) - return err; - - fmr->mtts = mlx4_table_find(&priv->mr_table.mtt_table, - fmr->mr.mtt.offset, - &fmr->dma_handle); - - if (!fmr->mtts) { - err = -ENOMEM; - goto err_free; - } - - return 0; - -err_free: - ret = mlx4_mr_free(dev, &fmr->mr); - if (ret) - mlx4_err(dev, "Error deregistering MR. The system may have become unstable."); - return err; -} -EXPORT_SYMBOL_GPL(mlx4_fmr_alloc); - -int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int err; - - err = mlx4_mr_enable(dev, &fmr->mr); - if (err) - return err; - - fmr->mpt = mlx4_table_find(&priv->mr_table.dmpt_table, - key_to_hw_index(fmr->mr.key), NULL); - if (!fmr->mpt) - return -ENOMEM; - - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_fmr_enable); - -void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr, - u32 *lkey, u32 *rkey) -{ - u32 key; - - if (!fmr->maps) - return; - - key = key_to_hw_index(fmr->mr.key) & (dev->caps.num_mpts - 1); - - *(u8 *)fmr->mpt = MLX4_MPT_STATUS_SW; - - /* Make sure MPT status is visible before changing MPT fields */ - wmb(); - - fmr->mr.key = hw_index_to_key(key); - - fmr->mpt->key = cpu_to_be32(key); - fmr->mpt->lkey = cpu_to_be32(key); - fmr->mpt->length = 0; - fmr->mpt->start = 0; - - /* Make sure MPT data is visible before changing MPT status */ - wmb(); - - *(u8 *)fmr->mpt = MLX4_MPT_STATUS_HW; - - /* Make sure MPT satus is visible */ - wmb(); - - fmr->maps = 0; -} -EXPORT_SYMBOL_GPL(mlx4_fmr_unmap); - -int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr) -{ - int ret; - - if (fmr->maps) - return -EBUSY; - - ret = mlx4_mr_free(dev, &fmr->mr); - if (ret) - return ret; - fmr->mr.enabled = MLX4_MPT_DISABLED; - - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_fmr_free); - -int mlx4_SYNC_TPT(struct mlx4_dev *dev) -{ - return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_SYNC_TPT, 1000, - MLX4_CMD_NATIVE); -} -EXPORT_SYMBOL_GPL(mlx4_SYNC_TPT); Property changes on: stable/11/sys/ofed/drivers/net/mlx4/mr.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/en_port.h =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/en_port.h (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/en_port.h (nonexistent) @@ -1,563 +0,0 @@ -/* - * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#ifndef _MLX4_EN_PORT_H_ -#define _MLX4_EN_PORT_H_ - - -#define SET_PORT_GEN_ALL_VALID 0x7 -#define SET_PORT_PROMISC_SHIFT 31 -#define SET_PORT_MC_PROMISC_SHIFT 30 - -#define MLX4_EN_NUM_TC 8 - -#define VLAN_FLTR_SIZE 128 -struct mlx4_set_vlan_fltr_mbox { - __be32 entry[VLAN_FLTR_SIZE]; -}; - - -enum { - MLX4_MCAST_CONFIG = 0, - MLX4_MCAST_DISABLE = 1, - MLX4_MCAST_ENABLE = 2, -}; - -enum { - MLX4_EN_10G_SPEED_XAUI = 0x00, - MLX4_EN_10G_SPEED_XFI = 0x01, - MLX4_EN_1G_SPEED = 0x02, - MLX4_EN_20G_SPEED = 0x08, - MLX4_EN_40G_SPEED = 0x40, - MLX4_EN_56G_SPEED = 0x20, - MLX4_EN_OTHER_SPEED = 0x0f, -}; - -struct mlx4_en_query_port_context { - u8 link_up; -#define MLX4_EN_LINK_UP_MASK 0x80 - u8 autoneg; -#define MLX4_EN_AUTONEG_MASK 0x80 - __be16 mtu; - u8 reserved2; - u8 link_speed; -#define MLX4_EN_SPEED_MASK 0x6b - u16 reserved3[5]; - __be64 mac; - u8 transceiver; -}; - - -struct mlx4_en_stat_out_mbox { - /* Received frames with a length of 64 octets */ - __be64 R64_prio_0; - __be64 R64_prio_1; - __be64 R64_prio_2; - __be64 R64_prio_3; - __be64 R64_prio_4; - __be64 R64_prio_5; - __be64 R64_prio_6; - __be64 R64_prio_7; - __be64 R64_novlan; - /* Received frames with a length of 127 octets */ - __be64 R127_prio_0; - __be64 R127_prio_1; - __be64 R127_prio_2; - __be64 R127_prio_3; - __be64 R127_prio_4; - __be64 R127_prio_5; - __be64 R127_prio_6; - __be64 R127_prio_7; - __be64 R127_novlan; - /* Received frames with a length of 255 octets */ - __be64 R255_prio_0; - __be64 R255_prio_1; - __be64 R255_prio_2; - __be64 R255_prio_3; - __be64 R255_prio_4; - __be64 R255_prio_5; - __be64 R255_prio_6; - __be64 R255_prio_7; - __be64 R255_novlan; - /* Received frames with a length of 511 octets */ - __be64 R511_prio_0; - __be64 R511_prio_1; - __be64 R511_prio_2; - __be64 R511_prio_3; - __be64 R511_prio_4; - __be64 R511_prio_5; - __be64 R511_prio_6; - __be64 R511_prio_7; - __be64 R511_novlan; - /* Received frames with a length of 1023 octets */ - __be64 R1023_prio_0; - __be64 R1023_prio_1; - __be64 R1023_prio_2; - __be64 R1023_prio_3; - __be64 R1023_prio_4; - __be64 R1023_prio_5; - __be64 R1023_prio_6; - __be64 R1023_prio_7; - __be64 R1023_novlan; - /* Received frames with a length of 1518 octets */ - __be64 R1518_prio_0; - __be64 R1518_prio_1; - __be64 R1518_prio_2; - __be64 R1518_prio_3; - __be64 R1518_prio_4; - __be64 R1518_prio_5; - __be64 R1518_prio_6; - __be64 R1518_prio_7; - __be64 R1518_novlan; - /* Received frames with a length of 1522 octets */ - __be64 R1522_prio_0; - __be64 R1522_prio_1; - __be64 R1522_prio_2; - __be64 R1522_prio_3; - __be64 R1522_prio_4; - __be64 R1522_prio_5; - __be64 R1522_prio_6; - __be64 R1522_prio_7; - __be64 R1522_novlan; - /* Received frames with a length of 1548 octets */ - __be64 R1548_prio_0; - __be64 R1548_prio_1; - __be64 R1548_prio_2; - __be64 R1548_prio_3; - __be64 R1548_prio_4; - __be64 R1548_prio_5; - __be64 R1548_prio_6; - __be64 R1548_prio_7; - __be64 R1548_novlan; - /* Received frames with a length of 1548 < octets < MTU */ - __be64 R2MTU_prio_0; - __be64 R2MTU_prio_1; - __be64 R2MTU_prio_2; - __be64 R2MTU_prio_3; - __be64 R2MTU_prio_4; - __be64 R2MTU_prio_5; - __be64 R2MTU_prio_6; - __be64 R2MTU_prio_7; - __be64 R2MTU_novlan; - /* Received frames with a length of MTU< octets and good CRC */ - __be64 RGIANT_prio_0; - __be64 RGIANT_prio_1; - __be64 RGIANT_prio_2; - __be64 RGIANT_prio_3; - __be64 RGIANT_prio_4; - __be64 RGIANT_prio_5; - __be64 RGIANT_prio_6; - __be64 RGIANT_prio_7; - __be64 RGIANT_novlan; - /* Received broadcast frames with good CRC */ - __be64 RBCAST_prio_0; - __be64 RBCAST_prio_1; - __be64 RBCAST_prio_2; - __be64 RBCAST_prio_3; - __be64 RBCAST_prio_4; - __be64 RBCAST_prio_5; - __be64 RBCAST_prio_6; - __be64 RBCAST_prio_7; - __be64 RBCAST_novlan; - /* Received multicast frames with good CRC */ - __be64 MCAST_prio_0; - __be64 MCAST_prio_1; - __be64 MCAST_prio_2; - __be64 MCAST_prio_3; - __be64 MCAST_prio_4; - __be64 MCAST_prio_5; - __be64 MCAST_prio_6; - __be64 MCAST_prio_7; - __be64 MCAST_novlan; - /* Received unicast not short or GIANT frames with good CRC */ - __be64 RTOTG_prio_0; - __be64 RTOTG_prio_1; - __be64 RTOTG_prio_2; - __be64 RTOTG_prio_3; - __be64 RTOTG_prio_4; - __be64 RTOTG_prio_5; - __be64 RTOTG_prio_6; - __be64 RTOTG_prio_7; - __be64 RTOTG_novlan; - - /* Count of total octets of received frames, includes framing characters */ - __be64 RTTLOCT_prio_0; - /* Count of total octets of received frames, not including framing - characters */ - __be64 RTTLOCT_NOFRM_prio_0; - /* Count of Total number of octets received - (only for frames without errors) */ - __be64 ROCT_prio_0; - - __be64 RTTLOCT_prio_1; - __be64 RTTLOCT_NOFRM_prio_1; - __be64 ROCT_prio_1; - - __be64 RTTLOCT_prio_2; - __be64 RTTLOCT_NOFRM_prio_2; - __be64 ROCT_prio_2; - - __be64 RTTLOCT_prio_3; - __be64 RTTLOCT_NOFRM_prio_3; - __be64 ROCT_prio_3; - - __be64 RTTLOCT_prio_4; - __be64 RTTLOCT_NOFRM_prio_4; - __be64 ROCT_prio_4; - - __be64 RTTLOCT_prio_5; - __be64 RTTLOCT_NOFRM_prio_5; - __be64 ROCT_prio_5; - - __be64 RTTLOCT_prio_6; - __be64 RTTLOCT_NOFRM_prio_6; - __be64 ROCT_prio_6; - - __be64 RTTLOCT_prio_7; - __be64 RTTLOCT_NOFRM_prio_7; - __be64 ROCT_prio_7; - - __be64 RTTLOCT_novlan; - __be64 RTTLOCT_NOFRM_novlan; - __be64 ROCT_novlan; - - /* Count of Total received frames including bad frames */ - __be64 RTOT_prio_0; - /* Count of Total number of received frames with 802.1Q encapsulation */ - __be64 R1Q_prio_0; - __be64 reserved1; - - __be64 RTOT_prio_1; - __be64 R1Q_prio_1; - __be64 reserved2; - - __be64 RTOT_prio_2; - __be64 R1Q_prio_2; - __be64 reserved3; - - __be64 RTOT_prio_3; - __be64 R1Q_prio_3; - __be64 reserved4; - - __be64 RTOT_prio_4; - __be64 R1Q_prio_4; - __be64 reserved5; - - __be64 RTOT_prio_5; - __be64 R1Q_prio_5; - __be64 reserved6; - - __be64 RTOT_prio_6; - __be64 R1Q_prio_6; - __be64 reserved7; - - __be64 RTOT_prio_7; - __be64 R1Q_prio_7; - __be64 reserved8; - - __be64 RTOT_novlan; - __be64 R1Q_novlan; - __be64 reserved9; - - /* Total number of Successfully Received Control Frames */ - __be64 RCNTL; - __be64 reserved10; - __be64 reserved11; - __be64 reserved12; - /* Count of received frames with a length/type field value between 46 - (42 for VLANtagged frames) and 1500 (also 1500 for VLAN-tagged frames), - inclusive */ - __be64 RInRangeLengthErr; - /* Count of received frames with length/type field between 1501 and 1535 - decimal, inclusive */ - __be64 ROutRangeLengthErr; - /* Count of received frames that are longer than max allowed size for - 802.3 frames (1518/1522) */ - __be64 RFrmTooLong; - /* Count frames received with PCS error */ - __be64 PCS; - - /* Transmit frames with a length of 64 octets */ - __be64 T64_prio_0; - __be64 T64_prio_1; - __be64 T64_prio_2; - __be64 T64_prio_3; - __be64 T64_prio_4; - __be64 T64_prio_5; - __be64 T64_prio_6; - __be64 T64_prio_7; - __be64 T64_novlan; - __be64 T64_loopbk; - /* Transmit frames with a length of 65 to 127 octets. */ - __be64 T127_prio_0; - __be64 T127_prio_1; - __be64 T127_prio_2; - __be64 T127_prio_3; - __be64 T127_prio_4; - __be64 T127_prio_5; - __be64 T127_prio_6; - __be64 T127_prio_7; - __be64 T127_novlan; - __be64 T127_loopbk; - /* Transmit frames with a length of 128 to 255 octets */ - __be64 T255_prio_0; - __be64 T255_prio_1; - __be64 T255_prio_2; - __be64 T255_prio_3; - __be64 T255_prio_4; - __be64 T255_prio_5; - __be64 T255_prio_6; - __be64 T255_prio_7; - __be64 T255_novlan; - __be64 T255_loopbk; - /* Transmit frames with a length of 256 to 511 octets */ - __be64 T511_prio_0; - __be64 T511_prio_1; - __be64 T511_prio_2; - __be64 T511_prio_3; - __be64 T511_prio_4; - __be64 T511_prio_5; - __be64 T511_prio_6; - __be64 T511_prio_7; - __be64 T511_novlan; - __be64 T511_loopbk; - /* Transmit frames with a length of 512 to 1023 octets */ - __be64 T1023_prio_0; - __be64 T1023_prio_1; - __be64 T1023_prio_2; - __be64 T1023_prio_3; - __be64 T1023_prio_4; - __be64 T1023_prio_5; - __be64 T1023_prio_6; - __be64 T1023_prio_7; - __be64 T1023_novlan; - __be64 T1023_loopbk; - /* Transmit frames with a length of 1024 to 1518 octets */ - __be64 T1518_prio_0; - __be64 T1518_prio_1; - __be64 T1518_prio_2; - __be64 T1518_prio_3; - __be64 T1518_prio_4; - __be64 T1518_prio_5; - __be64 T1518_prio_6; - __be64 T1518_prio_7; - __be64 T1518_novlan; - __be64 T1518_loopbk; - /* Counts transmit frames with a length of 1519 to 1522 bytes */ - __be64 T1522_prio_0; - __be64 T1522_prio_1; - __be64 T1522_prio_2; - __be64 T1522_prio_3; - __be64 T1522_prio_4; - __be64 T1522_prio_5; - __be64 T1522_prio_6; - __be64 T1522_prio_7; - __be64 T1522_novlan; - __be64 T1522_loopbk; - /* Transmit frames with a length of 1523 to 1548 octets */ - __be64 T1548_prio_0; - __be64 T1548_prio_1; - __be64 T1548_prio_2; - __be64 T1548_prio_3; - __be64 T1548_prio_4; - __be64 T1548_prio_5; - __be64 T1548_prio_6; - __be64 T1548_prio_7; - __be64 T1548_novlan; - __be64 T1548_loopbk; - /* Counts transmit frames with a length of 1549 to MTU bytes */ - __be64 T2MTU_prio_0; - __be64 T2MTU_prio_1; - __be64 T2MTU_prio_2; - __be64 T2MTU_prio_3; - __be64 T2MTU_prio_4; - __be64 T2MTU_prio_5; - __be64 T2MTU_prio_6; - __be64 T2MTU_prio_7; - __be64 T2MTU_novlan; - __be64 T2MTU_loopbk; - /* Transmit frames with a length greater than MTU octets and a good CRC. */ - __be64 TGIANT_prio_0; - __be64 TGIANT_prio_1; - __be64 TGIANT_prio_2; - __be64 TGIANT_prio_3; - __be64 TGIANT_prio_4; - __be64 TGIANT_prio_5; - __be64 TGIANT_prio_6; - __be64 TGIANT_prio_7; - __be64 TGIANT_novlan; - __be64 TGIANT_loopbk; - /* Transmit broadcast frames with a good CRC */ - __be64 TBCAST_prio_0; - __be64 TBCAST_prio_1; - __be64 TBCAST_prio_2; - __be64 TBCAST_prio_3; - __be64 TBCAST_prio_4; - __be64 TBCAST_prio_5; - __be64 TBCAST_prio_6; - __be64 TBCAST_prio_7; - __be64 TBCAST_novlan; - __be64 TBCAST_loopbk; - /* Transmit multicast frames with a good CRC */ - __be64 TMCAST_prio_0; - __be64 TMCAST_prio_1; - __be64 TMCAST_prio_2; - __be64 TMCAST_prio_3; - __be64 TMCAST_prio_4; - __be64 TMCAST_prio_5; - __be64 TMCAST_prio_6; - __be64 TMCAST_prio_7; - __be64 TMCAST_novlan; - __be64 TMCAST_loopbk; - /* Transmit good frames that are neither broadcast nor multicast */ - __be64 TTOTG_prio_0; - __be64 TTOTG_prio_1; - __be64 TTOTG_prio_2; - __be64 TTOTG_prio_3; - __be64 TTOTG_prio_4; - __be64 TTOTG_prio_5; - __be64 TTOTG_prio_6; - __be64 TTOTG_prio_7; - __be64 TTOTG_novlan; - __be64 TTOTG_loopbk; - - /* total octets of transmitted frames, including framing characters */ - __be64 TTTLOCT_prio_0; - /* total octets of transmitted frames, not including framing characters */ - __be64 TTTLOCT_NOFRM_prio_0; - /* ifOutOctets */ - __be64 TOCT_prio_0; - - __be64 TTTLOCT_prio_1; - __be64 TTTLOCT_NOFRM_prio_1; - __be64 TOCT_prio_1; - - __be64 TTTLOCT_prio_2; - __be64 TTTLOCT_NOFRM_prio_2; - __be64 TOCT_prio_2; - - __be64 TTTLOCT_prio_3; - __be64 TTTLOCT_NOFRM_prio_3; - __be64 TOCT_prio_3; - - __be64 TTTLOCT_prio_4; - __be64 TTTLOCT_NOFRM_prio_4; - __be64 TOCT_prio_4; - - __be64 TTTLOCT_prio_5; - __be64 TTTLOCT_NOFRM_prio_5; - __be64 TOCT_prio_5; - - __be64 TTTLOCT_prio_6; - __be64 TTTLOCT_NOFRM_prio_6; - __be64 TOCT_prio_6; - - __be64 TTTLOCT_prio_7; - __be64 TTTLOCT_NOFRM_prio_7; - __be64 TOCT_prio_7; - - __be64 TTTLOCT_novlan; - __be64 TTTLOCT_NOFRM_novlan; - __be64 TOCT_novlan; - - __be64 TTTLOCT_loopbk; - __be64 TTTLOCT_NOFRM_loopbk; - __be64 TOCT_loopbk; - - /* Total frames transmitted with a good CRC that are not aborted */ - __be64 TTOT_prio_0; - /* Total number of frames transmitted with 802.1Q encapsulation */ - __be64 T1Q_prio_0; - __be64 reserved13; - - __be64 TTOT_prio_1; - __be64 T1Q_prio_1; - __be64 reserved14; - - __be64 TTOT_prio_2; - __be64 T1Q_prio_2; - __be64 reserved15; - - __be64 TTOT_prio_3; - __be64 T1Q_prio_3; - __be64 reserved16; - - __be64 TTOT_prio_4; - __be64 T1Q_prio_4; - __be64 reserved17; - - __be64 TTOT_prio_5; - __be64 T1Q_prio_5; - __be64 reserved18; - - __be64 TTOT_prio_6; - __be64 T1Q_prio_6; - __be64 reserved19; - - __be64 TTOT_prio_7; - __be64 T1Q_prio_7; - __be64 reserved20; - - __be64 TTOT_novlan; - __be64 T1Q_novlan; - __be64 reserved21; - - __be64 TTOT_loopbk; - __be64 T1Q_loopbk; - __be64 reserved22; - - /* Received frames with a length greater than MTU octets and a bad CRC */ - __be32 RJBBR; - /* Received frames with a bad CRC that are not runts, jabbers, - or alignment errors */ - __be32 RCRC; - /* Received frames with SFD with a length of less than 64 octets and a - bad CRC */ - __be32 RRUNT; - /* Received frames with a length less than 64 octets and a good CRC */ - __be32 RSHORT; - /* Total Number of Received Packets Dropped */ - __be32 RDROP; - /* Drop due to overflow */ - __be32 RdropOvflw; - /* Drop due to overflow */ - __be32 RdropLength; - /* Total of good frames. Does not include frames received with - frame-too-long, FCS, or length errors */ - __be32 RTOTFRMS; - /* Total dropped Xmited packets */ - __be32 TDROP; -}; - - -#endif Property changes on: stable/11/sys/ofed/drivers/net/mlx4/en_port.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/pd.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/pd.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/pd.c (nonexistent) @@ -1,302 +0,0 @@ -/* - * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2005, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include - -#include - -#include "mlx4.h" -#include "icm.h" - -enum { - MLX4_NUM_RESERVED_UARS = 8 -}; - -int mlx4_pd_alloc(struct mlx4_dev *dev, u32 *pdn) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - *pdn = mlx4_bitmap_alloc(&priv->pd_bitmap); - if (*pdn == -1) - return -ENOMEM; - - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_pd_alloc); - -void mlx4_pd_free(struct mlx4_dev *dev, u32 pdn) -{ - mlx4_bitmap_free(&mlx4_priv(dev)->pd_bitmap, pdn, MLX4_USE_RR); -} -EXPORT_SYMBOL_GPL(mlx4_pd_free); - -int __mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - *xrcdn = mlx4_bitmap_alloc(&priv->xrcd_bitmap); - if (*xrcdn == -1) - return -ENOMEM; - - return 0; -} - -int mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn) -{ - u64 out_param; - int err; - - if (mlx4_is_mfunc(dev)) { - err = mlx4_cmd_imm(dev, 0, &out_param, - RES_XRCD, RES_OP_RESERVE, - MLX4_CMD_ALLOC_RES, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); - if (err) - return err; - - *xrcdn = get_param_l(&out_param); - return 0; - } - return __mlx4_xrcd_alloc(dev, xrcdn); -} -EXPORT_SYMBOL_GPL(mlx4_xrcd_alloc); - -void __mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn) -{ - mlx4_bitmap_free(&mlx4_priv(dev)->xrcd_bitmap, xrcdn, MLX4_USE_RR); -} - -void mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn) -{ - u64 in_param = 0; - int err; - - if (mlx4_is_mfunc(dev)) { - set_param_l(&in_param, xrcdn); - err = mlx4_cmd(dev, in_param, RES_XRCD, - RES_OP_RESERVE, MLX4_CMD_FREE_RES, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); - if (err) - mlx4_warn(dev, "Failed to release xrcdn %d\n", xrcdn); - } else - __mlx4_xrcd_free(dev, xrcdn); -} -EXPORT_SYMBOL_GPL(mlx4_xrcd_free); - -int mlx4_init_pd_table(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - return mlx4_bitmap_init(&priv->pd_bitmap, dev->caps.num_pds, - (1 << NOT_MASKED_PD_BITS) - 1, - dev->caps.reserved_pds, 0); -} - -void mlx4_cleanup_pd_table(struct mlx4_dev *dev) -{ - mlx4_bitmap_cleanup(&mlx4_priv(dev)->pd_bitmap); -} - -int mlx4_init_xrcd_table(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - return mlx4_bitmap_init(&priv->xrcd_bitmap, (1 << 16), - (1 << 16) - 1, dev->caps.reserved_xrcds + 1, 0); -} - -void mlx4_cleanup_xrcd_table(struct mlx4_dev *dev) -{ - mlx4_bitmap_cleanup(&mlx4_priv(dev)->xrcd_bitmap); -} - -int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar) -{ - int offset; - - uar->index = mlx4_bitmap_alloc(&mlx4_priv(dev)->uar_table.bitmap); - if (uar->index == -1) - return -ENOMEM; - - if (mlx4_is_slave(dev)) - offset = uar->index % ((int) pci_resource_len(dev->pdev, 2) / - dev->caps.uar_page_size); - else - offset = uar->index; - uar->pfn = (pci_resource_start(dev->pdev, 2) >> PAGE_SHIFT) + offset; - uar->map = NULL; - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_uar_alloc); - -void mlx4_uar_free(struct mlx4_dev *dev, struct mlx4_uar *uar) -{ - mlx4_bitmap_free(&mlx4_priv(dev)->uar_table.bitmap, uar->index, MLX4_USE_RR); -} -EXPORT_SYMBOL_GPL(mlx4_uar_free); - -#ifndef CONFIG_PPC -int mlx4_bf_alloc(struct mlx4_dev *dev, struct mlx4_bf *bf, int node) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_uar *uar; - int err = 0; - int idx; - - if (!priv->bf_mapping) - return -ENOMEM; - - mutex_lock(&priv->bf_mutex); - if (!list_empty(&priv->bf_list)) - uar = list_entry(priv->bf_list.next, struct mlx4_uar, bf_list); - else { - if (mlx4_bitmap_avail(&priv->uar_table.bitmap) < MLX4_NUM_RESERVED_UARS) { - err = -ENOMEM; - goto out; - } - uar = kmalloc_node(sizeof *uar, GFP_KERNEL, node); - if (!uar) { - uar = kmalloc(sizeof *uar, GFP_KERNEL); - if (!uar) { - err = -ENOMEM; - goto out; - } - } - err = mlx4_uar_alloc(dev, uar); - if (err) - goto free_kmalloc; - - uar->map = ioremap(uar->pfn << PAGE_SHIFT, PAGE_SIZE); - if (!uar->map) { - err = -ENOMEM; - goto free_uar; - } - - uar->bf_map = io_mapping_map_wc(priv->bf_mapping, uar->index << PAGE_SHIFT, PAGE_SIZE); - if (!uar->bf_map) { - err = -ENOMEM; - goto unamp_uar; - } - uar->free_bf_bmap = 0; - list_add(&uar->bf_list, &priv->bf_list); - } - - bf->uar = uar; - idx = ffz(uar->free_bf_bmap); - uar->free_bf_bmap |= 1 << idx; - bf->uar = uar; - bf->offset = 0; - bf->buf_size = dev->caps.bf_reg_size / 2; - bf->reg = uar->bf_map + idx * dev->caps.bf_reg_size; - if (uar->free_bf_bmap == (1 << dev->caps.bf_regs_per_page) - 1) - list_del_init(&uar->bf_list); - - goto out; - -unamp_uar: - bf->uar = NULL; - iounmap(uar->map); - -free_uar: - mlx4_uar_free(dev, uar); - -free_kmalloc: - kfree(uar); - -out: - mutex_unlock(&priv->bf_mutex); - return err; -} -EXPORT_SYMBOL_GPL(mlx4_bf_alloc); - -void mlx4_bf_free(struct mlx4_dev *dev, struct mlx4_bf *bf) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int idx; - - if (!bf->uar || !bf->uar->bf_map) - return; - - mutex_lock(&priv->bf_mutex); - idx = (bf->reg - bf->uar->bf_map) / dev->caps.bf_reg_size; - bf->uar->free_bf_bmap &= ~(1 << idx); - if (!bf->uar->free_bf_bmap) { - if (!list_empty(&bf->uar->bf_list)) - list_del(&bf->uar->bf_list); - - io_mapping_unmap(bf->uar->bf_map); - iounmap(bf->uar->map); - mlx4_uar_free(dev, bf->uar); - kfree(bf->uar); - } else if (list_empty(&bf->uar->bf_list)) - list_add(&bf->uar->bf_list, &priv->bf_list); - - mutex_unlock(&priv->bf_mutex); -} -EXPORT_SYMBOL_GPL(mlx4_bf_free); - -#else -int mlx4_bf_alloc(struct mlx4_dev *dev, struct mlx4_bf *bf, int node) -{ - memset(bf, 0, sizeof *bf); - return -ENOSYS; -} -EXPORT_SYMBOL_GPL(mlx4_bf_alloc); - -void mlx4_bf_free(struct mlx4_dev *dev, struct mlx4_bf *bf) -{ - return; -} -EXPORT_SYMBOL_GPL(mlx4_bf_free); -#endif - -int mlx4_init_uar_table(struct mlx4_dev *dev) -{ - if (dev->caps.num_uars <= 128) { - mlx4_err(dev, "Only %d UAR pages (need more than 128)\n", - dev->caps.num_uars); - mlx4_err(dev, "Increase firmware log2_uar_bar_megabytes?\n"); - return -ENODEV; - } - - return mlx4_bitmap_init(&mlx4_priv(dev)->uar_table.bitmap, - dev->caps.num_uars, dev->caps.num_uars - 1, - dev->caps.reserved_uars, 0); -} - -void mlx4_cleanup_uar_table(struct mlx4_dev *dev) -{ - mlx4_bitmap_cleanup(&mlx4_priv(dev)->uar_table.bitmap); -} Property changes on: stable/11/sys/ofed/drivers/net/mlx4/pd.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/mlx4_en.h =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/mlx4_en.h (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/mlx4_en.h (nonexistent) @@ -1,914 +0,0 @@ -/* - * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#ifndef _MLX4_EN_H_ -#define _MLX4_EN_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef CONFIG_MLX4_EN_DCB -#include -#endif - -#include -#include -#include -#include -#include -#include - -#include - -#include "en_port.h" -#include "mlx4_stats.h" - -#define DRV_NAME "mlx4_en" - -#define MLX4_EN_MSG_LEVEL (NETIF_MSG_LINK | NETIF_MSG_IFDOWN) - -/* - * Device constants - */ - - -#define MLX4_EN_PAGE_SHIFT 12 -#define MLX4_EN_PAGE_SIZE (1 << MLX4_EN_PAGE_SHIFT) -#define MLX4_NET_IP_ALIGN 2 /* bytes */ -#define DEF_RX_RINGS 16 -#define MAX_RX_RINGS 128 -#define MIN_RX_RINGS 4 -#define TXBB_SIZE 64 -#define HEADROOM (2048 / TXBB_SIZE + 1) -#define STAMP_STRIDE 64 -#define STAMP_DWORDS (STAMP_STRIDE / 4) -#define STAMP_SHIFT 31 -#define STAMP_VAL 0x7fffffff -#define STATS_DELAY (HZ / 4) -#define SERVICE_TASK_DELAY (HZ / 4) -#define MAX_NUM_OF_FS_RULES 256 - -#define MLX4_EN_FILTER_HASH_SHIFT 4 -#define MLX4_EN_FILTER_EXPIRY_QUOTA 60 - -#ifdef CONFIG_NET_RX_BUSY_POLL -#define LL_EXTENDED_STATS -#endif - -/* vlan valid range */ -#define VLAN_MIN_VALUE 1 -#define VLAN_MAX_VALUE 4094 - -/* - * OS related constants and tunables - */ - -#define MLX4_EN_WATCHDOG_TIMEOUT (15 * HZ) - -#define MLX4_EN_ALLOC_SIZE PAGE_ALIGN(PAGE_SIZE) -#define MLX4_EN_ALLOC_ORDER get_order(MLX4_EN_ALLOC_SIZE) - -enum mlx4_en_alloc_type { - MLX4_EN_ALLOC_NEW = 0, - MLX4_EN_ALLOC_REPLACEMENT = 1, -}; - -/* Maximum ring sizes */ -#define MLX4_EN_DEF_TX_QUEUE_SIZE 4096 - -/* Minimum packet number till arming the CQ */ -#define MLX4_EN_MIN_RX_ARM 2048 -#define MLX4_EN_MIN_TX_ARM 2048 - -/* Maximum ring sizes */ -#define MLX4_EN_MAX_TX_SIZE 8192 -#define MLX4_EN_MAX_RX_SIZE 8192 - -/* Minimum ring sizes */ -#define MLX4_EN_MIN_RX_SIZE (4096 / TXBB_SIZE) -#define MLX4_EN_MIN_TX_SIZE (4096 / TXBB_SIZE) - -#define MLX4_EN_SMALL_PKT_SIZE 64 - -#define MLX4_EN_MAX_TX_RING_P_UP 32 -#define MLX4_EN_NUM_UP 1 - -#define MAX_TX_RINGS (MLX4_EN_MAX_TX_RING_P_UP * \ - MLX4_EN_NUM_UP) - -#define MLX4_EN_DEF_TX_RING_SIZE 1024 -#define MLX4_EN_DEF_RX_RING_SIZE 1024 - -/* Target number of bytes to coalesce with interrupt moderation */ -#define MLX4_EN_RX_COAL_TARGET 0x20000 -#define MLX4_EN_RX_COAL_TIME 0x10 - -#define MLX4_EN_TX_COAL_PKTS 64 -#define MLX4_EN_TX_COAL_TIME 64 - -#define MLX4_EN_RX_RATE_LOW 400000 -#define MLX4_EN_RX_COAL_TIME_LOW 0 -#define MLX4_EN_RX_RATE_HIGH 450000 -#define MLX4_EN_RX_COAL_TIME_HIGH 128 -#define MLX4_EN_RX_SIZE_THRESH 1024 -#define MLX4_EN_RX_RATE_THRESH (1000000 / MLX4_EN_RX_COAL_TIME_HIGH) -#define MLX4_EN_SAMPLE_INTERVAL 0 -#define MLX4_EN_AVG_PKT_SMALL 256 - -#define MLX4_EN_AUTO_CONF 0xffff - -#define MLX4_EN_DEF_RX_PAUSE 1 -#define MLX4_EN_DEF_TX_PAUSE 1 - -/* Interval between successive polls in the Tx routine when polling is used - instead of interrupts (in per-core Tx rings) - should be power of 2 */ -#define MLX4_EN_TX_POLL_MODER 16 -#define MLX4_EN_TX_POLL_TIMEOUT (HZ / 4) - -#define MLX4_EN_64_ALIGN (64 - NET_SKB_PAD) -#define SMALL_PACKET_SIZE (256 - NET_IP_ALIGN) -#define HEADER_COPY_SIZE (128) -#define MLX4_LOOPBACK_TEST_PAYLOAD (HEADER_COPY_SIZE - ETHER_HDR_LEN) - -#define MLX4_EN_MIN_MTU 46 -#define ETH_BCAST 0xffffffffffffULL - -#define MLX4_EN_LOOPBACK_RETRIES 5 -#define MLX4_EN_LOOPBACK_TIMEOUT 100 - -#ifdef MLX4_EN_PERF_STAT -/* Number of samples to 'average' */ -#define AVG_SIZE 128 -#define AVG_FACTOR 1024 - -#define INC_PERF_COUNTER(cnt) (++(cnt)) -#define ADD_PERF_COUNTER(cnt, add) ((cnt) += (add)) -#define AVG_PERF_COUNTER(cnt, sample) \ - ((cnt) = ((cnt) * (AVG_SIZE - 1) + (sample) * AVG_FACTOR) / AVG_SIZE) -#define GET_PERF_COUNTER(cnt) (cnt) -#define GET_AVG_PERF_COUNTER(cnt) ((cnt) / AVG_FACTOR) - -#else - -#define INC_PERF_COUNTER(cnt) do {} while (0) -#define ADD_PERF_COUNTER(cnt, add) do {} while (0) -#define AVG_PERF_COUNTER(cnt, sample) do {} while (0) -#define GET_PERF_COUNTER(cnt) (0) -#define GET_AVG_PERF_COUNTER(cnt) (0) -#endif /* MLX4_EN_PERF_STAT */ - -/* - * Configurables - */ - -enum cq_type { - RX = 0, - TX = 1, -}; - - -/* - * Useful macros - */ -#define ROUNDUP_LOG2(x) ilog2(roundup_pow_of_two(x)) -#define XNOR(x, y) (!(x) == !(y)) -#define ILLEGAL_MAC(addr) (addr == 0xffffffffffffULL || addr == 0x0) - -struct mlx4_en_tx_info { - bus_dmamap_t dma_map; - struct mbuf *mb; - u32 nr_txbb; - u32 nr_bytes; -}; - - -#define MLX4_EN_BIT_DESC_OWN 0x80000000 -#define CTRL_SIZE sizeof(struct mlx4_wqe_ctrl_seg) -#define MLX4_EN_MEMTYPE_PAD 0x100 -#define DS_SIZE sizeof(struct mlx4_wqe_data_seg) - - -struct mlx4_en_tx_desc { - struct mlx4_wqe_ctrl_seg ctrl; - union { - struct mlx4_wqe_data_seg data; /* at least one data segment */ - struct mlx4_wqe_lso_seg lso; - struct mlx4_wqe_inline_seg inl; - }; -}; - -#define MLX4_EN_USE_SRQ 0x01000000 - -#define MLX4_EN_RX_BUDGET 64 - -#define MLX4_EN_TX_MAX_DESC_SIZE 512 /* bytes */ -#define MLX4_EN_TX_MAX_MBUF_SIZE 65536 /* bytes */ -#define MLX4_EN_TX_MAX_PAYLOAD_SIZE 65536 /* bytes */ -#define MLX4_EN_TX_MAX_MBUF_FRAGS \ - ((MLX4_EN_TX_MAX_DESC_SIZE - 128) / DS_SIZE_ALIGNMENT) /* units */ -#define MLX4_EN_TX_WQE_MAX_WQEBBS \ - (MLX4_EN_TX_MAX_DESC_SIZE / TXBB_SIZE) /* units */ - -#define MLX4_EN_CX3_LOW_ID 0x1000 -#define MLX4_EN_CX3_HIGH_ID 0x1005 - -struct mlx4_en_tx_ring { - spinlock_t tx_lock; - bus_dma_tag_t dma_tag; - struct mlx4_hwq_resources wqres; - u32 size ; /* number of TXBBs */ - u32 size_mask; - u16 stride; - u16 cqn; /* index of port CQ associated with this ring */ - u32 prod; - u32 cons; - u32 buf_size; - u32 doorbell_qpn; - u8 *buf; - u16 poll_cnt; - int blocked; - struct mlx4_en_tx_info *tx_info; - u8 queue_index; - cpuset_t affinity_mask; - struct buf_ring *br; - u32 last_nr_txbb; - struct mlx4_qp qp; - struct mlx4_qp_context context; - int qpn; - enum mlx4_qp_state qp_state; - struct mlx4_srq dummy; - unsigned long bytes; - unsigned long packets; - unsigned long tx_csum; - unsigned long queue_stopped; - unsigned long oversized_packets; - unsigned long wake_queue; - unsigned long tso_packets; - unsigned long defrag_attempts; - struct mlx4_bf bf; - bool bf_enabled; - int hwtstamp_tx_type; - spinlock_t comp_lock; - int inline_thold; - u64 watchdog_time; -}; - -struct mlx4_en_rx_desc { - /* actual number of entries depends on rx ring stride */ - struct mlx4_wqe_data_seg data[0]; -}; - -struct mlx4_en_rx_mbuf { - bus_dmamap_t dma_map; - struct mbuf *mbuf; -}; - -struct mlx4_en_rx_spare { - bus_dmamap_t dma_map; - struct mbuf *mbuf; - u64 paddr_be; -}; - -struct mlx4_en_rx_ring { - struct mlx4_hwq_resources wqres; - bus_dma_tag_t dma_tag; - struct mlx4_en_rx_spare spare; - u32 size ; /* number of Rx descs*/ - u32 actual_size; - u32 size_mask; - u16 stride; - u16 log_stride; - u16 cqn; /* index of port CQ associated with this ring */ - u32 prod; - u32 cons; - u32 buf_size; - u8 fcs_del; - u32 rx_mb_size; - int qpn; - u8 *buf; - struct mlx4_en_rx_mbuf *mbuf; - unsigned long errors; - unsigned long bytes; - unsigned long packets; -#ifdef LL_EXTENDED_STATS - unsigned long yields; - unsigned long misses; - unsigned long cleaned; -#endif - unsigned long csum_ok; - unsigned long csum_none; - int hwtstamp_rx_filter; - int numa_node; - struct lro_ctrl lro; -}; - -static inline int mlx4_en_can_lro(__be16 status) -{ - const __be16 status_all = cpu_to_be16( - MLX4_CQE_STATUS_IPV4 | - MLX4_CQE_STATUS_IPV4F | - MLX4_CQE_STATUS_IPV6 | - MLX4_CQE_STATUS_IPV4OPT | - MLX4_CQE_STATUS_TCP | - MLX4_CQE_STATUS_UDP | - MLX4_CQE_STATUS_IPOK); - const __be16 status_ipv4_ipok_tcp = cpu_to_be16( - MLX4_CQE_STATUS_IPV4 | - MLX4_CQE_STATUS_IPOK | - MLX4_CQE_STATUS_TCP); - const __be16 status_ipv6_ipok_tcp = cpu_to_be16( - MLX4_CQE_STATUS_IPV6 | - MLX4_CQE_STATUS_IPOK | - MLX4_CQE_STATUS_TCP); - - status &= status_all; - return (status == status_ipv4_ipok_tcp || - status == status_ipv6_ipok_tcp); -} - -struct mlx4_en_cq { - struct mlx4_cq mcq; - struct mlx4_hwq_resources wqres; - int ring; - spinlock_t lock; - struct net_device *dev; - /* Per-core Tx cq processing support */ - struct timer_list timer; - int size; - int buf_size; - unsigned vector; - enum cq_type is_tx; - u16 moder_time; - u16 moder_cnt; - struct mlx4_cqe *buf; - struct task cq_task; - struct taskqueue *tq; -#define MLX4_EN_OPCODE_ERROR 0x1e - u32 tot_rx; - u32 tot_tx; - u32 curr_poll_rx_cpu_id; - -#ifdef CONFIG_NET_RX_BUSY_POLL - unsigned int state; -#define MLX4_EN_CQ_STATEIDLE 0 -#define MLX4_EN_CQ_STATENAPI 1 /* NAPI owns this CQ */ -#define MLX4_EN_CQ_STATEPOLL 2 /* poll owns this CQ */ -#define MLX4_CQ_LOCKED (MLX4_EN_CQ_STATENAPI | MLX4_EN_CQ_STATEPOLL) -#define MLX4_EN_CQ_STATENAPI_YIELD 4 /* NAPI yielded this CQ */ -#define MLX4_EN_CQ_STATEPOLL_YIELD 8 /* poll yielded this CQ */ -#define CQ_YIELD (MLX4_EN_CQ_STATENAPI_YIELD | MLX4_EN_CQ_STATEPOLL_YIELD) -#define CQ_USER_PEND (MLX4_EN_CQ_STATEPOLL | MLX4_EN_CQ_STATEPOLL_YIELD) - spinlock_t poll_lock; /* protects from LLS/napi conflicts */ -#endif /* CONFIG_NET_RX_BUSY_POLL */ -}; - -struct mlx4_en_port_profile { - u32 flags; - u32 tx_ring_num; - u32 rx_ring_num; - u32 tx_ring_size; - u32 rx_ring_size; - u8 rx_pause; - u8 rx_ppp; - u8 tx_pause; - u8 tx_ppp; - int rss_rings; -}; - -struct mlx4_en_profile { - int rss_xor; - int udp_rss; - u8 rss_mask; - u32 active_ports; - u32 small_pkt_int; - u8 no_reset; - u8 num_tx_rings_p_up; - struct mlx4_en_port_profile prof[MLX4_MAX_PORTS + 1]; -}; - -struct mlx4_en_dev { - struct mlx4_dev *dev; - struct pci_dev *pdev; - struct mutex state_lock; - struct net_device *pndev[MLX4_MAX_PORTS + 1]; - u32 port_cnt; - bool device_up; - struct mlx4_en_profile profile; - u32 LSO_support; - struct workqueue_struct *workqueue; - struct device *dma_device; - void __iomem *uar_map; - struct mlx4_uar priv_uar; - struct mlx4_mr mr; - u32 priv_pdn; - spinlock_t uar_lock; - u8 mac_removed[MLX4_MAX_PORTS + 1]; - unsigned long last_overflow_check; - unsigned long overflow_period; -}; - - -struct mlx4_en_rss_map { - int base_qpn; - struct mlx4_qp qps[MAX_RX_RINGS]; - enum mlx4_qp_state state[MAX_RX_RINGS]; - struct mlx4_qp indir_qp; - enum mlx4_qp_state indir_state; -}; - -struct mlx4_en_port_state { - int link_state; - int link_speed; - int transciver; - int autoneg; -}; - -enum mlx4_en_mclist_act { - MCLIST_NONE, - MCLIST_REM, - MCLIST_ADD, -}; - -struct mlx4_en_mc_list { - struct list_head list; - enum mlx4_en_mclist_act action; - u8 addr[ETH_ALEN]; - u64 reg_id; -}; - -#ifdef CONFIG_MLX4_EN_DCB -/* Minimal TC BW - setting to 0 will block traffic */ -#define MLX4_EN_BW_MIN 1 -#define MLX4_EN_BW_MAX 100 /* Utilize 100% of the line */ - -#define MLX4_EN_TC_ETS 7 - -#endif - - -enum { - MLX4_EN_FLAG_PROMISC = (1 << 0), - MLX4_EN_FLAG_MC_PROMISC = (1 << 1), - /* whether we need to enable hardware loopback by putting dmac - * in Tx WQE - */ - MLX4_EN_FLAG_ENABLE_HW_LOOPBACK = (1 << 2), - /* whether we need to drop packets that hardware loopback-ed */ - MLX4_EN_FLAG_RX_FILTER_NEEDED = (1 << 3), - MLX4_EN_FLAG_FORCE_PROMISC = (1 << 4), -#ifdef CONFIG_MLX4_EN_DCB - MLX4_EN_FLAG_DCB_ENABLED = (1 << 5) -#endif -}; - -#define MLX4_EN_MAC_HASH_SIZE (1 << BITS_PER_BYTE) -#define MLX4_EN_MAC_HASH_IDX 5 - -struct en_port { - struct kobject kobj; - struct mlx4_dev *dev; - u8 port_num; - u8 vport_num; -}; - -struct mlx4_en_priv { - struct mlx4_en_dev *mdev; - struct mlx4_en_port_profile *prof; - struct net_device *dev; - unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; - struct mlx4_en_port_state port_state; - spinlock_t stats_lock; - /* To allow rules removal while port is going down */ - struct list_head ethtool_list; - - unsigned long last_moder_packets[MAX_RX_RINGS]; - unsigned long last_moder_tx_packets; - unsigned long last_moder_bytes[MAX_RX_RINGS]; - unsigned long last_moder_jiffies; - int last_moder_time[MAX_RX_RINGS]; - u16 rx_usecs; - u16 rx_frames; - u16 tx_usecs; - u16 tx_frames; - u32 pkt_rate_low; - u32 rx_usecs_low; - u32 pkt_rate_high; - u32 rx_usecs_high; - u32 sample_interval; - u32 adaptive_rx_coal; - u32 msg_enable; - u32 loopback_ok; - u32 validate_loopback; - - struct mlx4_hwq_resources res; - int link_state; - int last_link_state; - bool port_up; - int port; - int registered; - int allocated; - int stride; - unsigned char current_mac[ETH_ALEN + 2]; - u64 mac; - int mac_index; - unsigned max_mtu; - int base_qpn; - int cqe_factor; - - struct mlx4_en_rss_map rss_map; - u32 flags; - u8 num_tx_rings_p_up; - u32 tx_ring_num; - u32 rx_ring_num; - u32 rx_mb_size; - - struct mlx4_en_tx_ring **tx_ring; - struct mlx4_en_rx_ring *rx_ring[MAX_RX_RINGS]; - struct mlx4_en_cq **tx_cq; - struct mlx4_en_cq *rx_cq[MAX_RX_RINGS]; - struct mlx4_qp drop_qp; - struct work_struct rx_mode_task; - struct work_struct watchdog_task; - struct work_struct linkstate_task; - struct delayed_work stats_task; - struct delayed_work service_task; - struct mlx4_en_perf_stats pstats; - struct mlx4_en_pkt_stats pkstats; - struct mlx4_en_pkt_stats pkstats_last; - struct mlx4_en_flow_stats flowstats[MLX4_NUM_PRIORITIES]; - struct mlx4_en_port_stats port_stats; - struct mlx4_en_vport_stats vport_stats; - struct mlx4_en_vf_stats vf_stats; - struct list_head mc_list; - struct list_head curr_list; - u64 broadcast_id; - struct mlx4_en_stat_out_mbox hw_stats; - int vids[128]; - bool wol; - struct device *ddev; - struct dentry *dev_root; - u32 counter_index; - eventhandler_tag vlan_attach; - eventhandler_tag vlan_detach; - struct callout watchdog_timer; - struct ifmedia media; - volatile int blocked; - struct sysctl_oid *conf_sysctl; - struct sysctl_oid *stat_sysctl; - struct sysctl_ctx_list conf_ctx; - struct sysctl_ctx_list stat_ctx; -#define MLX4_EN_MAC_HASH_IDX 5 - struct hlist_head mac_hash[MLX4_EN_MAC_HASH_SIZE]; - -#ifdef CONFIG_MLX4_EN_DCB - struct ieee_ets ets; - u16 maxrate[IEEE_8021QAZ_MAX_TCS]; - u8 dcbx_cap; -#endif -#ifdef CONFIG_RFS_ACCEL - spinlock_t filters_lock; - int last_filter_id; - struct list_head filters; - struct hlist_head filter_hash[1 << MLX4_EN_FILTER_HASH_SHIFT]; -#endif - struct en_port *vf_ports[MLX4_MAX_NUM_VF]; - unsigned long last_ifq_jiffies; - u64 if_counters_rx_errors; - u64 if_counters_rx_no_buffer; -}; - -enum mlx4_en_wol { - MLX4_EN_WOL_MAGIC = (1ULL << 61), - MLX4_EN_WOL_ENABLED = (1ULL << 62), -}; - -struct mlx4_mac_entry { - struct hlist_node hlist; - unsigned char mac[ETH_ALEN + 2]; - u64 reg_id; -}; - -#ifdef CONFIG_NET_RX_BUSY_POLL -static inline void mlx4_en_cq_init_lock(struct mlx4_en_cq *cq) -{ - spin_lock_init(&cq->poll_lock); - cq->state = MLX4_EN_CQ_STATEIDLE; -} - -/* called from the device poll rutine to get ownership of a cq */ -static inline bool mlx4_en_cq_lock_napi(struct mlx4_en_cq *cq) -{ - int rc = true; - spin_lock(&cq->poll_lock); - if (cq->state & MLX4_CQ_LOCKED) { - WARN_ON(cq->state & MLX4_EN_CQ_STATENAPI); - cq->state |= MLX4_EN_CQ_STATENAPI_YIELD; - rc = false; - } else - /* we don't care if someone yielded */ - cq->state = MLX4_EN_CQ_STATENAPI; - spin_unlock(&cq->poll_lock); - return rc; -} - -/* returns true is someone tried to get the cq while napi had it */ -static inline bool mlx4_en_cq_unlock_napi(struct mlx4_en_cq *cq) -{ - int rc = false; - spin_lock(&cq->poll_lock); - WARN_ON(cq->state & (MLX4_EN_CQ_STATEPOLL | - MLX4_EN_CQ_STATENAPI_YIELD)); - - if (cq->state & MLX4_EN_CQ_STATEPOLL_YIELD) - rc = true; - cq->state = MLX4_EN_CQ_STATEIDLE; - spin_unlock(&cq->poll_lock); - return rc; -} - -/* called from mlx4_en_low_latency_poll() */ -static inline bool mlx4_en_cq_lock_poll(struct mlx4_en_cq *cq) -{ - int rc = true; - spin_lock_bh(&cq->poll_lock); - if ((cq->state & MLX4_CQ_LOCKED)) { - struct net_device *dev = cq->dev; - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_rx_ring *rx_ring = priv->rx_ring[cq->ring]; - - cq->state |= MLX4_EN_CQ_STATEPOLL_YIELD; - rc = false; -#ifdef LL_EXTENDED_STATS - rx_ring->yields++; -#endif - } else - /* preserve yield marks */ - cq->state |= MLX4_EN_CQ_STATEPOLL; - spin_unlock_bh(&cq->poll_lock); - return rc; -} - -/* returns true if someone tried to get the cq while it was locked */ -static inline bool mlx4_en_cq_unlock_poll(struct mlx4_en_cq *cq) -{ - int rc = false; - spin_lock_bh(&cq->poll_lock); - WARN_ON(cq->state & (MLX4_EN_CQ_STATENAPI)); - - if (cq->state & MLX4_EN_CQ_STATEPOLL_YIELD) - rc = true; - cq->state = MLX4_EN_CQ_STATEIDLE; - spin_unlock_bh(&cq->poll_lock); - return rc; -} - -/* true if a socket is polling, even if it did not get the lock */ -static inline bool mlx4_en_cq_ll_polling(struct mlx4_en_cq *cq) -{ - WARN_ON(!(cq->state & MLX4_CQ_LOCKED)); - return cq->state & CQ_USER_PEND; -} -#else -static inline void mlx4_en_cq_init_lock(struct mlx4_en_cq *cq) -{ -} - -static inline bool mlx4_en_cq_lock_napi(struct mlx4_en_cq *cq) -{ - return true; -} - -static inline bool mlx4_en_cq_unlock_napi(struct mlx4_en_cq *cq) -{ - return false; -} - -static inline bool mlx4_en_cq_lock_poll(struct mlx4_en_cq *cq) -{ - return false; -} - -static inline bool mlx4_en_cq_unlock_poll(struct mlx4_en_cq *cq) -{ - return false; -} - -static inline bool mlx4_en_cq_ll_polling(struct mlx4_en_cq *cq) -{ - return false; -} -#endif /* CONFIG_NET_RX_BUSY_POLL */ - -#define MLX4_EN_WOL_DO_MODIFY (1ULL << 63) - -void mlx4_en_destroy_netdev(struct net_device *dev); -int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, - struct mlx4_en_port_profile *prof); - -int mlx4_en_start_port(struct net_device *dev); -void mlx4_en_stop_port(struct net_device *dev); - -void mlx4_en_free_resources(struct mlx4_en_priv *priv); -int mlx4_en_alloc_resources(struct mlx4_en_priv *priv); - -int mlx4_en_pre_config(struct mlx4_en_priv *priv); -int mlx4_en_create_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq, - int entries, int ring, enum cq_type mode, int node); -void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq); -int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq, - int cq_idx); -void mlx4_en_deactivate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); -int mlx4_en_set_cq_moder(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); -int mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); - -void mlx4_en_tx_irq(struct mlx4_cq *mcq); -u16 mlx4_en_select_queue(struct net_device *dev, struct mbuf *mb); - -int mlx4_en_transmit(struct ifnet *dev, struct mbuf *m); -int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring **pring, - u32 size, u16 stride, int node, int queue_idx); -void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring **pring); -int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring *ring, - int cq, int user_prio); -void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring *ring); -void mlx4_en_qflush(struct ifnet *dev); - -int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring **pring, - u32 size, int node); -void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring **pring, - u32 size, u16 stride); -void mlx4_en_tx_que(void *context, int pending); -void mlx4_en_rx_que(void *context, int pending); -int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv); -void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring); -int mlx4_en_process_rx_cq(struct net_device *dev, - struct mlx4_en_cq *cq, - int budget); -void mlx4_en_poll_tx_cq(unsigned long data); -void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride, - int is_tx, int rss, int qpn, int cqn, int user_prio, - struct mlx4_qp_context *context); -void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event); -int mlx4_en_map_buffer(struct mlx4_buf *buf); -void mlx4_en_unmap_buffer(struct mlx4_buf *buf); -void mlx4_en_calc_rx_buf(struct net_device *dev); - -int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv); -void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv); -int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv); -void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv); -int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring); -void mlx4_en_rx_irq(struct mlx4_cq *mcq); - -int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, u64 mac, u64 clear, u8 mode); -int mlx4_SET_VLAN_FLTR(struct mlx4_dev *dev, struct mlx4_en_priv *priv); - -int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset); -int mlx4_en_QUERY_PORT(struct mlx4_en_dev *mdev, u8 port); -int mlx4_en_get_vport_stats(struct mlx4_en_dev *mdev, u8 port); -void mlx4_en_create_debug_files(struct mlx4_en_priv *priv); -void mlx4_en_delete_debug_files(struct mlx4_en_priv *priv); -int mlx4_en_register_debugfs(void); -void mlx4_en_unregister_debugfs(void); - -#ifdef CONFIG_MLX4_EN_DCB -extern const struct dcbnl_rtnl_ops mlx4_en_dcbnl_ops; -extern const struct dcbnl_rtnl_ops mlx4_en_dcbnl_pfc_ops; -#endif - -int mlx4_en_setup_tc(struct net_device *dev, u8 up); - -#ifdef CONFIG_RFS_ACCEL -void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *rx_ring); -#endif - -#define MLX4_EN_NUM_SELF_TEST 5 -void mlx4_en_ex_selftest(struct net_device *dev, u32 *flags, u64 *buf); -void mlx4_en_ptp_overflow_check(struct mlx4_en_dev *mdev); - -/* - * Functions for time stamping - */ -#define SKBTX_HW_TSTAMP (1 << 0) -#define SKBTX_IN_PROGRESS (1 << 2) - -u64 mlx4_en_get_cqe_ts(struct mlx4_cqe *cqe); - -/* Functions for caching and restoring statistics */ -int mlx4_en_get_sset_count(struct net_device *dev, int sset); -void mlx4_en_restore_ethtool_stats(struct mlx4_en_priv *priv, - u64 *data); - -/* - * Globals - */ -extern const struct ethtool_ops mlx4_en_ethtool_ops; - -/* - * Defines for link speed - needed by selftest - */ -#define MLX4_EN_LINK_SPEED_1G 1000 -#define MLX4_EN_LINK_SPEED_10G 10000 -#define MLX4_EN_LINK_SPEED_40G 40000 - -enum { - NETIF_MSG_DRV = 0x0001, - NETIF_MSG_PROBE = 0x0002, - NETIF_MSG_LINK = 0x0004, - NETIF_MSG_TIMER = 0x0008, - NETIF_MSG_IFDOWN = 0x0010, - NETIF_MSG_IFUP = 0x0020, - NETIF_MSG_RX_ERR = 0x0040, - NETIF_MSG_TX_ERR = 0x0080, - NETIF_MSG_TX_QUEUED = 0x0100, - NETIF_MSG_INTR = 0x0200, - NETIF_MSG_TX_DONE = 0x0400, - NETIF_MSG_RX_STATUS = 0x0800, - NETIF_MSG_PKTDATA = 0x1000, - NETIF_MSG_HW = 0x2000, - NETIF_MSG_WOL = 0x4000, -}; - - -/* - * printk / logging functions - */ - -#define en_print(level, priv, format, arg...) \ - { \ - if ((priv)->registered) \ - printk(level "%s: %s: " format, DRV_NAME, \ - (priv->dev)->if_xname, ## arg); \ - else \ - printk(level "%s: %s: Port %d: " format, \ - DRV_NAME, dev_name(&priv->mdev->pdev->dev), \ - (priv)->port, ## arg); \ - } - - -#define en_dbg(mlevel, priv, format, arg...) \ -do { \ - if (NETIF_MSG_##mlevel & priv->msg_enable) \ - en_print(KERN_DEBUG, priv, format, ##arg); \ -} while (0) -#define en_warn(priv, format, arg...) \ - en_print(KERN_WARNING, priv, format, ##arg) -#define en_err(priv, format, arg...) \ - en_print(KERN_ERR, priv, format, ##arg) -#define en_info(priv, format, arg...) \ - en_print(KERN_INFO, priv, format, ## arg) - -#define mlx4_err(mdev, format, arg...) \ - pr_err("%s %s: " format, DRV_NAME, \ - dev_name(&mdev->pdev->dev), ##arg) -#define mlx4_info(mdev, format, arg...) \ - pr_info("%s %s: " format, DRV_NAME, \ - dev_name(&mdev->pdev->dev), ##arg) -#define mlx4_warn(mdev, format, arg...) \ - pr_warning("%s %s: " format, DRV_NAME, \ - dev_name(&mdev->pdev->dev), ##arg) - -#endif Property changes on: stable/11/sys/ofed/drivers/net/mlx4/mlx4_en.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/qp.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/qp.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/qp.c (nonexistent) @@ -1,618 +0,0 @@ -/* - * Copyright (c) 2004 Topspin Communications. All rights reserved. - * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * Copyright (c) 2004 Voltaire, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include - -#include -#include - -#include "mlx4.h" -#include "icm.h" - -/* - * QP to support BF should have bits 6,7 cleared - */ -#define MLX4_BF_QP_SKIP_MASK 0xc0 -#define MLX4_MAX_BF_QP_RANGE 0x40 - -void mlx4_qp_event(struct mlx4_dev *dev, u32 qpn, int event_type) -{ - struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table; - struct mlx4_qp *qp; - - spin_lock(&qp_table->lock); - - qp = __mlx4_qp_lookup(dev, qpn); - if (qp) - atomic_inc(&qp->refcount); - - spin_unlock(&qp_table->lock); - - if (!qp) { - mlx4_dbg(dev, "Async event for none existent QP %08x\n", qpn); - return; - } - - qp->event(qp, event_type); - - if (atomic_dec_and_test(&qp->refcount)) - complete(&qp->free); -} - -/* used for INIT/CLOSE port logic */ -static int is_master_qp0(struct mlx4_dev *dev, struct mlx4_qp *qp, int *real_qp0, int *proxy_qp0) -{ - /* this procedure is called after we already know we are on the master */ - /* qp0 is either the proxy qp0, or the real qp0 */ - u32 pf_proxy_offset = dev->phys_caps.base_proxy_sqpn + 8 * mlx4_master_func_num(dev); - *proxy_qp0 = qp->qpn >= pf_proxy_offset && qp->qpn <= pf_proxy_offset + 1; - - *real_qp0 = qp->qpn >= dev->phys_caps.base_sqpn && - qp->qpn <= dev->phys_caps.base_sqpn + 1; - - return *real_qp0 || *proxy_qp0; -} - -static int __mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt, - enum mlx4_qp_state cur_state, enum mlx4_qp_state new_state, - struct mlx4_qp_context *context, - enum mlx4_qp_optpar optpar, - int sqd_event, struct mlx4_qp *qp, int native) -{ - static const u16 op[MLX4_QP_NUM_STATE][MLX4_QP_NUM_STATE] = { - [MLX4_QP_STATE_RST] = { - [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, - [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, - [MLX4_QP_STATE_INIT] = MLX4_CMD_RST2INIT_QP, - }, - [MLX4_QP_STATE_INIT] = { - [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, - [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, - [MLX4_QP_STATE_INIT] = MLX4_CMD_INIT2INIT_QP, - [MLX4_QP_STATE_RTR] = MLX4_CMD_INIT2RTR_QP, - }, - [MLX4_QP_STATE_RTR] = { - [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, - [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, - [MLX4_QP_STATE_RTS] = MLX4_CMD_RTR2RTS_QP, - }, - [MLX4_QP_STATE_RTS] = { - [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, - [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, - [MLX4_QP_STATE_RTS] = MLX4_CMD_RTS2RTS_QP, - [MLX4_QP_STATE_SQD] = MLX4_CMD_RTS2SQD_QP, - }, - [MLX4_QP_STATE_SQD] = { - [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, - [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, - [MLX4_QP_STATE_RTS] = MLX4_CMD_SQD2RTS_QP, - [MLX4_QP_STATE_SQD] = MLX4_CMD_SQD2SQD_QP, - }, - [MLX4_QP_STATE_SQER] = { - [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, - [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, - [MLX4_QP_STATE_RTS] = MLX4_CMD_SQERR2RTS_QP, - }, - [MLX4_QP_STATE_ERR] = { - [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, - [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, - } - }; - - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_cmd_mailbox *mailbox; - int ret = 0; - int real_qp0 = 0; - int proxy_qp0 = 0; - u8 port; - - if (cur_state >= MLX4_QP_NUM_STATE || new_state >= MLX4_QP_NUM_STATE || - !op[cur_state][new_state]) - return -EINVAL; - - if (op[cur_state][new_state] == MLX4_CMD_2RST_QP) { - ret = mlx4_cmd(dev, 0, qp->qpn, 2, - MLX4_CMD_2RST_QP, MLX4_CMD_TIME_CLASS_A, native); - if (mlx4_is_master(dev) && cur_state != MLX4_QP_STATE_ERR && - cur_state != MLX4_QP_STATE_RST && - is_master_qp0(dev, qp, &real_qp0, &proxy_qp0)) { - port = (qp->qpn & 1) + 1; - if (proxy_qp0) - priv->mfunc.master.qp0_state[port].proxy_qp0_active = 0; - else - priv->mfunc.master.qp0_state[port].qp0_active = 0; - } - return ret; - } - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - if (cur_state == MLX4_QP_STATE_RST && new_state == MLX4_QP_STATE_INIT) { - u64 mtt_addr = mlx4_mtt_addr(dev, mtt); - context->mtt_base_addr_h = mtt_addr >> 32; - context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); - context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT; - } - - *(__be32 *) mailbox->buf = cpu_to_be32(optpar); - memcpy(mailbox->buf + 8, context, sizeof *context); - - ((struct mlx4_qp_context *) (mailbox->buf + 8))->local_qpn = - cpu_to_be32(qp->qpn); - - ret = mlx4_cmd(dev, mailbox->dma, - qp->qpn | (!!sqd_event << 31), - new_state == MLX4_QP_STATE_RST ? 2 : 0, - op[cur_state][new_state], MLX4_CMD_TIME_CLASS_C, native); - - if (mlx4_is_master(dev) && is_master_qp0(dev, qp, &real_qp0, &proxy_qp0)) { - port = (qp->qpn & 1) + 1; - if (cur_state != MLX4_QP_STATE_ERR && - cur_state != MLX4_QP_STATE_RST && - new_state == MLX4_QP_STATE_ERR) { - if (proxy_qp0) - priv->mfunc.master.qp0_state[port].proxy_qp0_active = 0; - else - priv->mfunc.master.qp0_state[port].qp0_active = 0; - } else if (new_state == MLX4_QP_STATE_RTR) { - if (proxy_qp0) - priv->mfunc.master.qp0_state[port].proxy_qp0_active = 1; - else - priv->mfunc.master.qp0_state[port].qp0_active = 1; - } - } - - mlx4_free_cmd_mailbox(dev, mailbox); - return ret; -} - -int mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt, - enum mlx4_qp_state cur_state, enum mlx4_qp_state new_state, - struct mlx4_qp_context *context, - enum mlx4_qp_optpar optpar, - int sqd_event, struct mlx4_qp *qp) -{ - return __mlx4_qp_modify(dev, mtt, cur_state, new_state, context, - optpar, sqd_event, qp, 0); -} -EXPORT_SYMBOL_GPL(mlx4_qp_modify); - -int __mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, - int *base, u8 flags) -{ - int bf_qp = !!(flags & (u8) MLX4_RESERVE_BF_QP); - - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_qp_table *qp_table = &priv->qp_table; - - /* Only IPoIB uses a large cnt. In this case, just allocate - * as usual, ignoring bf skipping, since IPoIB does not run over RoCE - */ - if (cnt > MLX4_MAX_BF_QP_RANGE && bf_qp) - bf_qp = 0; - - *base = mlx4_bitmap_alloc_range(&qp_table->bitmap, cnt, align, - bf_qp ? MLX4_BF_QP_SKIP_MASK : 0); - if (*base == -1) - return -ENOMEM; - - return 0; -} - -int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, - int *base, u8 flags) -{ - u64 in_param = 0; - u64 out_param; - int err; - - /* Turn off all unsupported QP allocation flags */ - flags &= dev->caps.alloc_res_qp_mask; - - if (mlx4_is_mfunc(dev)) { - set_param_l(&in_param, (((u32) flags) << 24) | (u32) cnt); - set_param_h(&in_param, align); - err = mlx4_cmd_imm(dev, in_param, &out_param, - RES_QP, RES_OP_RESERVE, - MLX4_CMD_ALLOC_RES, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); - if (err) - return err; - - *base = get_param_l(&out_param); - return 0; - } - return __mlx4_qp_reserve_range(dev, cnt, align, base, flags); -} -EXPORT_SYMBOL_GPL(mlx4_qp_reserve_range); - -void __mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_qp_table *qp_table = &priv->qp_table; - - if (mlx4_is_qp_reserved(dev, (u32) base_qpn)) - return; - mlx4_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt, MLX4_USE_RR); -} - -void mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt) -{ - u64 in_param = 0; - int err; - - if (mlx4_is_mfunc(dev)) { - set_param_l(&in_param, base_qpn); - set_param_h(&in_param, cnt); - err = mlx4_cmd(dev, in_param, RES_QP, RES_OP_RESERVE, - MLX4_CMD_FREE_RES, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); - if (err) { - mlx4_warn(dev, "Failed to release qp range" - " base:%d cnt:%d\n", base_qpn, cnt); - } - } else - __mlx4_qp_release_range(dev, base_qpn, cnt); -} -EXPORT_SYMBOL_GPL(mlx4_qp_release_range); - -int __mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_qp_table *qp_table = &priv->qp_table; - int err; - - err = mlx4_table_get(dev, &qp_table->qp_table, qpn); - if (err) - goto err_out; - - err = mlx4_table_get(dev, &qp_table->auxc_table, qpn); - if (err) - goto err_put_qp; - - err = mlx4_table_get(dev, &qp_table->altc_table, qpn); - if (err) - goto err_put_auxc; - - err = mlx4_table_get(dev, &qp_table->rdmarc_table, qpn); - if (err) - goto err_put_altc; - - err = mlx4_table_get(dev, &qp_table->cmpt_table, qpn); - if (err) - goto err_put_rdmarc; - - return 0; - -err_put_rdmarc: - mlx4_table_put(dev, &qp_table->rdmarc_table, qpn); - -err_put_altc: - mlx4_table_put(dev, &qp_table->altc_table, qpn); - -err_put_auxc: - mlx4_table_put(dev, &qp_table->auxc_table, qpn); - -err_put_qp: - mlx4_table_put(dev, &qp_table->qp_table, qpn); - -err_out: - return err; -} - -static int mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn) -{ - u64 param = 0; - - if (mlx4_is_mfunc(dev)) { - set_param_l(¶m, qpn); - return mlx4_cmd_imm(dev, param, ¶m, RES_QP, RES_OP_MAP_ICM, - MLX4_CMD_ALLOC_RES, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_WRAPPED); - } - return __mlx4_qp_alloc_icm(dev, qpn); -} - -void __mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_qp_table *qp_table = &priv->qp_table; - - mlx4_table_put(dev, &qp_table->cmpt_table, qpn); - mlx4_table_put(dev, &qp_table->rdmarc_table, qpn); - mlx4_table_put(dev, &qp_table->altc_table, qpn); - mlx4_table_put(dev, &qp_table->auxc_table, qpn); - mlx4_table_put(dev, &qp_table->qp_table, qpn); -} - -static void mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn) -{ - u64 in_param = 0; - - if (mlx4_is_mfunc(dev)) { - set_param_l(&in_param, qpn); - if (mlx4_cmd(dev, in_param, RES_QP, RES_OP_MAP_ICM, - MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_WRAPPED)) - mlx4_warn(dev, "Failed to free icm of qp:%d\n", qpn); - } else - __mlx4_qp_free_icm(dev, qpn); -} - -int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_qp_table *qp_table = &priv->qp_table; - int err; - - if (!qpn) - return -EINVAL; - - qp->qpn = qpn; - - err = mlx4_qp_alloc_icm(dev, qpn); - if (err) - return err; - - spin_lock_irq(&qp_table->lock); - err = radix_tree_insert(&dev->qp_table_tree, qp->qpn & - (dev->caps.num_qps - 1), qp); - spin_unlock_irq(&qp_table->lock); - if (err) - goto err_icm; - - atomic_set(&qp->refcount, 1); - init_completion(&qp->free); - - return 0; - -err_icm: - mlx4_qp_free_icm(dev, qpn); - return err; -} - -EXPORT_SYMBOL_GPL(mlx4_qp_alloc); - -void mlx4_qp_remove(struct mlx4_dev *dev, struct mlx4_qp *qp) -{ - struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table; - unsigned long flags; - - spin_lock_irqsave(&qp_table->lock, flags); - radix_tree_delete(&dev->qp_table_tree, qp->qpn & (dev->caps.num_qps - 1)); - spin_unlock_irqrestore(&qp_table->lock, flags); -} -EXPORT_SYMBOL_GPL(mlx4_qp_remove); - -void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp) -{ - if (atomic_dec_and_test(&qp->refcount)) - complete(&qp->free); - wait_for_completion(&qp->free); - - mlx4_qp_free_icm(dev, qp->qpn); -} -EXPORT_SYMBOL_GPL(mlx4_qp_free); - -static int mlx4_CONF_SPECIAL_QP(struct mlx4_dev *dev, u32 base_qpn) -{ - return mlx4_cmd(dev, 0, base_qpn, 0, MLX4_CMD_CONF_SPECIAL_QP, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); -} - -int mlx4_init_qp_table(struct mlx4_dev *dev) -{ - struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table; - int err; - int reserved_from_top = 0; - int reserved_from_bot; - int k; - - spin_lock_init(&qp_table->lock); - INIT_RADIX_TREE(&dev->qp_table_tree, GFP_ATOMIC); - if (mlx4_is_slave(dev)) - return 0; - - /* - * We reserve 2 extra QPs per port for the special QPs. The - * block of special QPs must be aligned to a multiple of 8, so - * round up. - * - * We also reserve the MSB of the 24-bit QP number to indicate - * that a QP is an XRC QP. - */ - dev->phys_caps.base_sqpn = - ALIGN(dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], 8); - - { - int sort[MLX4_NUM_QP_REGION]; - int i, j, tmp; - int last_base = dev->caps.num_qps; - - for (i = 1; i < MLX4_NUM_QP_REGION; ++i) - sort[i] = i; - - for (i = MLX4_NUM_QP_REGION; i > 0; --i) { - for (j = 2; j < i; ++j) { - if (dev->caps.reserved_qps_cnt[sort[j]] > - dev->caps.reserved_qps_cnt[sort[j - 1]]) { - tmp = sort[j]; - sort[j] = sort[j - 1]; - sort[j - 1] = tmp; - } - } - } - - for (i = 1; i < MLX4_NUM_QP_REGION; ++i) { - last_base -= dev->caps.reserved_qps_cnt[sort[i]]; - dev->caps.reserved_qps_base[sort[i]] = last_base; - reserved_from_top += - dev->caps.reserved_qps_cnt[sort[i]]; - } - - } - - /* Reserve 8 real SQPs in both native and SRIOV modes. - * In addition, in SRIOV mode, reserve 8 proxy SQPs per function - * (for all PFs and VFs), and 8 corresponding tunnel QPs. - * Each proxy SQP works opposite its own tunnel QP. - * - * The QPs are arranged as follows: - * a. 8 real SQPs - * b. All the proxy SQPs (8 per function) - * c. All the tunnel QPs (8 per function) - */ - reserved_from_bot = mlx4_num_reserved_sqps(dev); - if (reserved_from_bot + reserved_from_top > dev->caps.num_qps) { - mlx4_err(dev, "Number of reserved QPs is higher than number " - "of QPs, increase the value of log_num_qp\n"); - return -EINVAL; - } - - err = mlx4_bitmap_init(&qp_table->bitmap, dev->caps.num_qps, - (1 << 23) - 1, reserved_from_bot, - reserved_from_top); - if (err) - return err; - - if (mlx4_is_mfunc(dev)) { - /* for PPF use */ - dev->phys_caps.base_proxy_sqpn = dev->phys_caps.base_sqpn + 8; - dev->phys_caps.base_tunnel_sqpn = dev->phys_caps.base_sqpn + 8 + 8 * MLX4_MFUNC_MAX; - - /* In mfunc, calculate proxy and tunnel qp offsets for the PF here, - * since the PF does not call mlx4_slave_caps */ - dev->caps.qp0_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); - dev->caps.qp0_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); - dev->caps.qp1_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); - dev->caps.qp1_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); - - if (!dev->caps.qp0_tunnel || !dev->caps.qp0_proxy || - !dev->caps.qp1_tunnel || !dev->caps.qp1_proxy) { - err = -ENOMEM; - goto err_mem; - } - - for (k = 0; k < dev->caps.num_ports; k++) { - dev->caps.qp0_proxy[k] = dev->phys_caps.base_proxy_sqpn + - 8 * mlx4_master_func_num(dev) + k; - dev->caps.qp0_tunnel[k] = dev->caps.qp0_proxy[k] + 8 * MLX4_MFUNC_MAX; - dev->caps.qp1_proxy[k] = dev->phys_caps.base_proxy_sqpn + - 8 * mlx4_master_func_num(dev) + MLX4_MAX_PORTS + k; - dev->caps.qp1_tunnel[k] = dev->caps.qp1_proxy[k] + 8 * MLX4_MFUNC_MAX; - } - } - - - err = mlx4_CONF_SPECIAL_QP(dev, dev->phys_caps.base_sqpn); - if (err) - goto err_mem; - return 0; - -err_mem: - kfree(dev->caps.qp0_tunnel); - kfree(dev->caps.qp0_proxy); - kfree(dev->caps.qp1_tunnel); - kfree(dev->caps.qp1_proxy); - dev->caps.qp0_tunnel = dev->caps.qp0_proxy = - dev->caps.qp1_tunnel = dev->caps.qp1_proxy = NULL; - return err; -} - -void mlx4_cleanup_qp_table(struct mlx4_dev *dev) -{ - if (mlx4_is_slave(dev)) - return; - - mlx4_CONF_SPECIAL_QP(dev, 0); - mlx4_bitmap_cleanup(&mlx4_priv(dev)->qp_table.bitmap); -} - -int mlx4_qp_query(struct mlx4_dev *dev, struct mlx4_qp *qp, - struct mlx4_qp_context *context) -{ - struct mlx4_cmd_mailbox *mailbox; - int err; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - err = mlx4_cmd_box(dev, 0, mailbox->dma, qp->qpn, 0, - MLX4_CMD_QUERY_QP, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_WRAPPED); - if (!err) - memcpy(context, mailbox->buf + 8, sizeof *context); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} -EXPORT_SYMBOL_GPL(mlx4_qp_query); - -int mlx4_qp_to_ready(struct mlx4_dev *dev, struct mlx4_mtt *mtt, - struct mlx4_qp_context *context, - struct mlx4_qp *qp, enum mlx4_qp_state *qp_state) -{ - int err; - int i; - enum mlx4_qp_state states[] = { - MLX4_QP_STATE_RST, - MLX4_QP_STATE_INIT, - MLX4_QP_STATE_RTR, - MLX4_QP_STATE_RTS - }; - - for (i = 0; i < ARRAY_SIZE(states) - 1; i++) { - context->flags &= cpu_to_be32(~(0xf << 28)); - context->flags |= cpu_to_be32(states[i + 1] << 28); - err = mlx4_qp_modify(dev, mtt, states[i], states[i + 1], - context, 0, 0, qp); - if (err) { - mlx4_err(dev, "Failed to bring QP to state: " - "%d with error: %d\n", - states[i + 1], err); - return err; - } - - *qp_state = states[i + 1]; - } - - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_qp_to_ready); Property changes on: stable/11/sys/ofed/drivers/net/mlx4/qp.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/mlx4_stats.h =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/mlx4_stats.h (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/mlx4_stats.h (nonexistent) @@ -1,177 +0,0 @@ -/* - * Copyright (c) 2014 Mellanox Technologies Ltd. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef _MLX4_STATS_ -#define _MLX4_STATS_ - - -#ifdef MLX4_EN_PERF_STAT -#define NUM_PERF_STATS NUM_PERF_COUNTERS -#else -#define NUM_PERF_STATS 0 -#endif - -#define NUM_PRIORITIES 9 -#define NUM_PRIORITY_STATS 2 - -struct mlx4_en_pkt_stats { - unsigned long rx_packets; - unsigned long rx_bytes; - unsigned long rx_multicast_packets; - unsigned long rx_broadcast_packets; - unsigned long rx_errors; - unsigned long rx_dropped; - unsigned long rx_length_errors; - unsigned long rx_over_errors; - unsigned long rx_crc_errors; - unsigned long rx_jabbers; - unsigned long rx_in_range_length_error; - unsigned long rx_out_range_length_error; - unsigned long rx_lt_64_bytes_packets; - unsigned long rx_127_bytes_packets; - unsigned long rx_255_bytes_packets; - unsigned long rx_511_bytes_packets; - unsigned long rx_1023_bytes_packets; - unsigned long rx_1518_bytes_packets; - unsigned long rx_1522_bytes_packets; - unsigned long rx_1548_bytes_packets; - unsigned long rx_gt_1548_bytes_packets; - unsigned long tx_packets; - unsigned long tx_bytes; - unsigned long tx_multicast_packets; - unsigned long tx_broadcast_packets; - unsigned long tx_errors; - unsigned long tx_dropped; - unsigned long tx_lt_64_bytes_packets; - unsigned long tx_127_bytes_packets; - unsigned long tx_255_bytes_packets; - unsigned long tx_511_bytes_packets; - unsigned long tx_1023_bytes_packets; - unsigned long tx_1518_bytes_packets; - unsigned long tx_1522_bytes_packets; - unsigned long tx_1548_bytes_packets; - unsigned long tx_gt_1548_bytes_packets; - unsigned long rx_prio[NUM_PRIORITIES][NUM_PRIORITY_STATS]; - unsigned long tx_prio[NUM_PRIORITIES][NUM_PRIORITY_STATS]; -}; - -struct mlx4_en_vf_stats { - unsigned long rx_packets; - unsigned long rx_bytes; - unsigned long rx_multicast_packets; - unsigned long rx_broadcast_packets; - unsigned long rx_errors; - unsigned long rx_dropped; - unsigned long tx_packets; - unsigned long tx_bytes; - unsigned long tx_multicast_packets; - unsigned long tx_broadcast_packets; - unsigned long tx_errors; -}; - -struct mlx4_en_vport_stats { - unsigned long rx_unicast_packets; - unsigned long rx_unicast_bytes; - unsigned long rx_multicast_packets; - unsigned long rx_multicast_bytes; - unsigned long rx_broadcast_packets; - unsigned long rx_broadcast_bytes; - unsigned long rx_dropped; - unsigned long rx_errors; - unsigned long tx_unicast_packets; - unsigned long tx_unicast_bytes; - unsigned long tx_multicast_packets; - unsigned long tx_multicast_bytes; - unsigned long tx_broadcast_packets; - unsigned long tx_broadcast_bytes; - unsigned long tx_errors; -}; - -struct mlx4_en_port_stats { - unsigned long tso_packets; - unsigned long queue_stopped; - unsigned long wake_queue; - unsigned long tx_timeout; - unsigned long oversized_packets; - unsigned long rx_alloc_failed; - unsigned long rx_chksum_good; - unsigned long rx_chksum_none; - unsigned long tx_chksum_offload; - unsigned long defrag_attempts; -}; - -struct mlx4_en_perf_stats { - u32 tx_poll; - u64 tx_pktsz_avg; - u32 inflight_avg; - u16 tx_coal_avg; - u16 rx_coal_avg; - u32 napi_quota; -}; - -struct mlx4_en_flow_stats { - u64 rx_pause; - u64 rx_pause_duration; - u64 rx_pause_transition; - u64 tx_pause; - u64 tx_pause_duration; - u64 tx_pause_transition; -}; -#define MLX4_NUM_PRIORITIES 8 - - -struct mlx4_en_stat_out_flow_control_mbox { - /* Total number of PAUSE frames received from the far-end port */ - __be64 rx_pause; - /* Total number of microseconds that far-end port requested to pause - * transmission of packets - */ - __be64 rx_pause_duration; - /* Number of received transmission from XOFF state to XON state */ - __be64 rx_pause_transition; - /* Total number of PAUSE frames sent from the far-end port */ - __be64 tx_pause; - /* Total time in microseconds that transmission of packets has been - * paused - */ - __be64 tx_pause_duration; - /* Number of transmitter transitions from XOFF state to XON state */ - __be64 tx_pause_transition; - /* Reserverd */ - __be64 reserved[2]; -}; - -int mlx4_get_vport_ethtool_stats(struct mlx4_dev *dev, int port, - struct mlx4_en_vport_stats *vport_stats, - int reset); - -#endif Property changes on: stable/11/sys/ofed/drivers/net/mlx4/mlx4_stats.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/resource_tracker.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/resource_tracker.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/resource_tracker.c (nonexistent) @@ -1,4689 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. - * All rights reserved. - * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "mlx4.h" -#include "fw.h" - -#define MLX4_MAC_VALID (1ull << 63) - -struct mac_res { - struct list_head list; - u64 mac; - int ref_count; - u8 smac_index; - u8 port; -}; - -struct vlan_res { - struct list_head list; - u16 vlan; - int ref_count; - int vlan_index; - u8 port; -}; - -struct res_common { - struct list_head list; - struct rb_node node; - u64 res_id; - int owner; - int state; - int from_state; - int to_state; - int removing; -}; - -enum { - RES_ANY_BUSY = 1 -}; - -struct res_gid { - struct list_head list; - u8 gid[16]; - enum mlx4_protocol prot; - enum mlx4_steer_type steer; - u64 reg_id; -}; - -enum res_qp_states { - RES_QP_BUSY = RES_ANY_BUSY, - - /* QP number was allocated */ - RES_QP_RESERVED, - - /* ICM memory for QP context was mapped */ - RES_QP_MAPPED, - - /* QP is in hw ownership */ - RES_QP_HW -}; - -struct res_qp { - struct res_common com; - struct res_mtt *mtt; - struct res_cq *rcq; - struct res_cq *scq; - struct res_srq *srq; - struct list_head mcg_list; - spinlock_t mcg_spl; - int local_qpn; - atomic_t ref_count; - u32 qpc_flags; - /* saved qp params before VST enforcement in order to restore on VGT */ - u8 sched_queue; - __be32 param3; - u8 vlan_control; - u8 fvl_rx; - u8 pri_path_fl; - u8 vlan_index; - u8 feup; -}; - -enum res_mtt_states { - RES_MTT_BUSY = RES_ANY_BUSY, - RES_MTT_ALLOCATED, -}; - -static inline const char *mtt_states_str(enum res_mtt_states state) -{ - switch (state) { - case RES_MTT_BUSY: return "RES_MTT_BUSY"; - case RES_MTT_ALLOCATED: return "RES_MTT_ALLOCATED"; - default: return "Unknown"; - } -} - -struct res_mtt { - struct res_common com; - int order; - atomic_t ref_count; -}; - -enum res_mpt_states { - RES_MPT_BUSY = RES_ANY_BUSY, - RES_MPT_RESERVED, - RES_MPT_MAPPED, - RES_MPT_HW, -}; - -struct res_mpt { - struct res_common com; - struct res_mtt *mtt; - int key; -}; - -enum res_eq_states { - RES_EQ_BUSY = RES_ANY_BUSY, - RES_EQ_RESERVED, - RES_EQ_HW, -}; - -struct res_eq { - struct res_common com; - struct res_mtt *mtt; -}; - -enum res_cq_states { - RES_CQ_BUSY = RES_ANY_BUSY, - RES_CQ_ALLOCATED, - RES_CQ_HW, -}; - -struct res_cq { - struct res_common com; - struct res_mtt *mtt; - atomic_t ref_count; -}; - -enum res_srq_states { - RES_SRQ_BUSY = RES_ANY_BUSY, - RES_SRQ_ALLOCATED, - RES_SRQ_HW, -}; - -struct res_srq { - struct res_common com; - struct res_mtt *mtt; - struct res_cq *cq; - atomic_t ref_count; -}; - -enum res_counter_states { - RES_COUNTER_BUSY = RES_ANY_BUSY, - RES_COUNTER_ALLOCATED, -}; - -struct res_counter { - struct res_common com; - int port; -}; - -enum res_xrcdn_states { - RES_XRCD_BUSY = RES_ANY_BUSY, - RES_XRCD_ALLOCATED, -}; - -struct res_xrcdn { - struct res_common com; - int port; -}; - -enum res_fs_rule_states { - RES_FS_RULE_BUSY = RES_ANY_BUSY, - RES_FS_RULE_ALLOCATED, -}; - -struct res_fs_rule { - struct res_common com; - int qpn; -}; - -static int mlx4_is_eth(struct mlx4_dev *dev, int port) -{ - return dev->caps.port_mask[port] == MLX4_PORT_TYPE_IB ? 0 : 1; -} - -static void *res_tracker_lookup(struct rb_root *root, u64 res_id) -{ - struct rb_node *node = root->rb_node; - - while (node) { - struct res_common *res = container_of(node, struct res_common, - node); - - if (res_id < res->res_id) - node = node->rb_left; - else if (res_id > res->res_id) - node = node->rb_right; - else - return res; - } - return NULL; -} - -static int res_tracker_insert(struct rb_root *root, struct res_common *res) -{ - struct rb_node **new = &(root->rb_node), *parent = NULL; - - /* Figure out where to put new node */ - while (*new) { - struct res_common *this = container_of(*new, struct res_common, - node); - - parent = *new; - if (res->res_id < this->res_id) - new = &((*new)->rb_left); - else if (res->res_id > this->res_id) - new = &((*new)->rb_right); - else - return -EEXIST; - } - - /* Add new node and rebalance tree. */ - rb_link_node(&res->node, parent, new); - rb_insert_color(&res->node, root); - - return 0; -} - -enum qp_transition { - QP_TRANS_INIT2RTR, - QP_TRANS_RTR2RTS, - QP_TRANS_RTS2RTS, - QP_TRANS_SQERR2RTS, - QP_TRANS_SQD2SQD, - QP_TRANS_SQD2RTS -}; - -/* For Debug uses */ -static const char *ResourceType(enum mlx4_resource rt) -{ - switch (rt) { - case RES_QP: return "RES_QP"; - case RES_CQ: return "RES_CQ"; - case RES_SRQ: return "RES_SRQ"; - case RES_MPT: return "RES_MPT"; - case RES_MTT: return "RES_MTT"; - case RES_MAC: return "RES_MAC"; - case RES_VLAN: return "RES_VLAN"; - case RES_EQ: return "RES_EQ"; - case RES_COUNTER: return "RES_COUNTER"; - case RES_FS_RULE: return "RES_FS_RULE"; - case RES_XRCD: return "RES_XRCD"; - default: return "Unknown resource type !!!"; - }; -} - -static void rem_slave_vlans(struct mlx4_dev *dev, int slave); -static inline int mlx4_grant_resource(struct mlx4_dev *dev, int slave, - enum mlx4_resource res_type, int count, - int port) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct resource_allocator *res_alloc = - &priv->mfunc.master.res_tracker.res_alloc[res_type]; - int err = -EINVAL; - int allocated, free, reserved, guaranteed, from_free; - - spin_lock(&res_alloc->alloc_lock); - allocated = (port > 0) ? - res_alloc->allocated[(port - 1) * (dev->num_vfs + 1) + slave] : - res_alloc->allocated[slave]; - free = (port > 0) ? res_alloc->res_port_free[port - 1] : - res_alloc->res_free; - reserved = (port > 0) ? res_alloc->res_port_rsvd[port - 1] : - res_alloc->res_reserved; - guaranteed = res_alloc->guaranteed[slave]; - - if (allocated + count > res_alloc->quota[slave]) - goto out; - - if (allocated + count <= guaranteed) { - err = 0; - } else { - /* portion may need to be obtained from free area */ - if (guaranteed - allocated > 0) - from_free = count - (guaranteed - allocated); - else - from_free = count; - - if (free - from_free > reserved) - err = 0; - } - - if (!err) { - /* grant the request */ - if (port > 0) { - res_alloc->allocated[(port - 1) * (dev->num_vfs + 1) + slave] += count; - res_alloc->res_port_free[port - 1] -= count; - } else { - res_alloc->allocated[slave] += count; - res_alloc->res_free -= count; - } - } - -out: - spin_unlock(&res_alloc->alloc_lock); - return err; - -} - -static inline void mlx4_release_resource(struct mlx4_dev *dev, int slave, - enum mlx4_resource res_type, int count, - int port) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct resource_allocator *res_alloc = - &priv->mfunc.master.res_tracker.res_alloc[res_type]; - - spin_lock(&res_alloc->alloc_lock); - if (port > 0) { - res_alloc->allocated[(port - 1) * (dev->num_vfs + 1) + slave] -= count; - res_alloc->res_port_free[port - 1] += count; - } else { - res_alloc->allocated[slave] -= count; - res_alloc->res_free += count; - } - - spin_unlock(&res_alloc->alloc_lock); - return; -} - -static inline void initialize_res_quotas(struct mlx4_dev *dev, - struct resource_allocator *res_alloc, - enum mlx4_resource res_type, - int vf, int num_instances) -{ - res_alloc->guaranteed[vf] = num_instances / (2 * (dev->num_vfs + 1)); - res_alloc->quota[vf] = (num_instances / 2) + res_alloc->guaranteed[vf]; - if (vf == mlx4_master_func_num(dev)) { - res_alloc->res_free = num_instances; - if (res_type == RES_MTT) { - /* reserved mtts will be taken out of the PF allocation */ - res_alloc->res_free += dev->caps.reserved_mtts; - res_alloc->guaranteed[vf] += dev->caps.reserved_mtts; - res_alloc->quota[vf] += dev->caps.reserved_mtts; - } - } -} - -void mlx4_init_quotas(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int pf; - - /* quotas for VFs are initialized in mlx4_slave_cap */ - if (mlx4_is_slave(dev)) - return; - - if (!mlx4_is_mfunc(dev)) { - dev->quotas.qp = dev->caps.num_qps - dev->caps.reserved_qps - - mlx4_num_reserved_sqps(dev); - dev->quotas.cq = dev->caps.num_cqs - dev->caps.reserved_cqs; - dev->quotas.srq = dev->caps.num_srqs - dev->caps.reserved_srqs; - dev->quotas.mtt = dev->caps.num_mtts - dev->caps.reserved_mtts; - dev->quotas.mpt = dev->caps.num_mpts - dev->caps.reserved_mrws; - return; - } - - pf = mlx4_master_func_num(dev); - dev->quotas.qp = - priv->mfunc.master.res_tracker.res_alloc[RES_QP].quota[pf]; - dev->quotas.cq = - priv->mfunc.master.res_tracker.res_alloc[RES_CQ].quota[pf]; - dev->quotas.srq = - priv->mfunc.master.res_tracker.res_alloc[RES_SRQ].quota[pf]; - dev->quotas.mtt = - priv->mfunc.master.res_tracker.res_alloc[RES_MTT].quota[pf]; - dev->quotas.mpt = - priv->mfunc.master.res_tracker.res_alloc[RES_MPT].quota[pf]; -} -int mlx4_init_resource_tracker(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int i, j; - int t; - - priv->mfunc.master.res_tracker.slave_list = - kzalloc(dev->num_slaves * sizeof(struct slave_list), - GFP_KERNEL); - if (!priv->mfunc.master.res_tracker.slave_list) - return -ENOMEM; - - for (i = 0 ; i < dev->num_slaves; i++) { - for (t = 0; t < MLX4_NUM_OF_RESOURCE_TYPE; ++t) - INIT_LIST_HEAD(&priv->mfunc.master.res_tracker. - slave_list[i].res_list[t]); - mutex_init(&priv->mfunc.master.res_tracker.slave_list[i].mutex); - } - - mlx4_dbg(dev, "Started init_resource_tracker: %ld slaves\n", - dev->num_slaves); - for (i = 0 ; i < MLX4_NUM_OF_RESOURCE_TYPE; i++) - priv->mfunc.master.res_tracker.res_tree[i] = RB_ROOT; - - for (i = 0; i < MLX4_NUM_OF_RESOURCE_TYPE; i++) { - struct resource_allocator *res_alloc = - &priv->mfunc.master.res_tracker.res_alloc[i]; - res_alloc->quota = kmalloc((dev->num_vfs + 1) * sizeof(int), GFP_KERNEL); - res_alloc->guaranteed = kmalloc((dev->num_vfs + 1) * sizeof(int), GFP_KERNEL); - if (i == RES_MAC || i == RES_VLAN) - res_alloc->allocated = kzalloc(MLX4_MAX_PORTS * - (dev->num_vfs + 1) * sizeof(int), - GFP_KERNEL); - else - res_alloc->allocated = kzalloc((dev->num_vfs + 1) * sizeof(int), GFP_KERNEL); - - if (!res_alloc->quota || !res_alloc->guaranteed || - !res_alloc->allocated) - goto no_mem_err; - - spin_lock_init(&res_alloc->alloc_lock); - for (t = 0; t < dev->num_vfs + 1; t++) { - switch (i) { - case RES_QP: - initialize_res_quotas(dev, res_alloc, RES_QP, - t, dev->caps.num_qps - - dev->caps.reserved_qps - - mlx4_num_reserved_sqps(dev)); - break; - case RES_CQ: - initialize_res_quotas(dev, res_alloc, RES_CQ, - t, dev->caps.num_cqs - - dev->caps.reserved_cqs); - break; - case RES_SRQ: - initialize_res_quotas(dev, res_alloc, RES_SRQ, - t, dev->caps.num_srqs - - dev->caps.reserved_srqs); - break; - case RES_MPT: - initialize_res_quotas(dev, res_alloc, RES_MPT, - t, dev->caps.num_mpts - - dev->caps.reserved_mrws); - break; - case RES_MTT: - initialize_res_quotas(dev, res_alloc, RES_MTT, - t, dev->caps.num_mtts - - dev->caps.reserved_mtts); - break; - case RES_MAC: - if (t == mlx4_master_func_num(dev)) { - res_alloc->quota[t] = - MLX4_MAX_MAC_NUM - 2 * dev->num_vfs; - res_alloc->guaranteed[t] = res_alloc->quota[t]; - for (j = 0; j < MLX4_MAX_PORTS; j++) - res_alloc->res_port_free[j] = MLX4_MAX_MAC_NUM; - } else { - res_alloc->quota[t] = 2; - res_alloc->guaranteed[t] = 2; - } - break; - case RES_VLAN: - if (t == mlx4_master_func_num(dev)) { - res_alloc->quota[t] = MLX4_MAX_VLAN_NUM; - res_alloc->guaranteed[t] = MLX4_MAX_VLAN_NUM / 2; - for (j = 0; j < MLX4_MAX_PORTS; j++) - res_alloc->res_port_free[j] = - res_alloc->quota[t]; - } else { - res_alloc->quota[t] = MLX4_MAX_VLAN_NUM / 2; - res_alloc->guaranteed[t] = 0; - } - break; - case RES_COUNTER: - res_alloc->quota[t] = dev->caps.max_counters; - res_alloc->guaranteed[t] = 0; - if (t == mlx4_master_func_num(dev)) - res_alloc->res_free = res_alloc->quota[t]; - break; - default: - break; - } - if (i == RES_MAC || i == RES_VLAN) { - for (j = 0; j < MLX4_MAX_PORTS; j++) - res_alloc->res_port_rsvd[j] += - res_alloc->guaranteed[t]; - } else { - res_alloc->res_reserved += res_alloc->guaranteed[t]; - } - } - } - spin_lock_init(&priv->mfunc.master.res_tracker.lock); - return 0; - -no_mem_err: - for (i = 0; i < MLX4_NUM_OF_RESOURCE_TYPE; i++) { - kfree(priv->mfunc.master.res_tracker.res_alloc[i].allocated); - priv->mfunc.master.res_tracker.res_alloc[i].allocated = NULL; - kfree(priv->mfunc.master.res_tracker.res_alloc[i].guaranteed); - priv->mfunc.master.res_tracker.res_alloc[i].guaranteed = NULL; - kfree(priv->mfunc.master.res_tracker.res_alloc[i].quota); - priv->mfunc.master.res_tracker.res_alloc[i].quota = NULL; - } - return -ENOMEM; -} - -void mlx4_free_resource_tracker(struct mlx4_dev *dev, - enum mlx4_res_tracker_free_type type) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int i; - - if (priv->mfunc.master.res_tracker.slave_list) { - if (type != RES_TR_FREE_STRUCTS_ONLY) { - for (i = 0; i < dev->num_slaves; i++) { - if (type == RES_TR_FREE_ALL || - dev->caps.function != i) - mlx4_delete_all_resources_for_slave(dev, i); - } - /* free master's vlans */ - i = dev->caps.function; - mutex_lock(&priv->mfunc.master.res_tracker.slave_list[i].mutex); - rem_slave_vlans(dev, i); - mutex_unlock(&priv->mfunc.master.res_tracker.slave_list[i].mutex); - } - - if (type != RES_TR_FREE_SLAVES_ONLY) { - for (i = 0; i < MLX4_NUM_OF_RESOURCE_TYPE; i++) { - kfree(priv->mfunc.master.res_tracker.res_alloc[i].allocated); - priv->mfunc.master.res_tracker.res_alloc[i].allocated = NULL; - kfree(priv->mfunc.master.res_tracker.res_alloc[i].guaranteed); - priv->mfunc.master.res_tracker.res_alloc[i].guaranteed = NULL; - kfree(priv->mfunc.master.res_tracker.res_alloc[i].quota); - priv->mfunc.master.res_tracker.res_alloc[i].quota = NULL; - } - kfree(priv->mfunc.master.res_tracker.slave_list); - priv->mfunc.master.res_tracker.slave_list = NULL; - } - } -} - -static void update_pkey_index(struct mlx4_dev *dev, int slave, - struct mlx4_cmd_mailbox *inbox) -{ - u8 sched = *(u8 *)(inbox->buf + 64); - u8 orig_index = *(u8 *)(inbox->buf + 35); - u8 new_index; - struct mlx4_priv *priv = mlx4_priv(dev); - int port; - - port = (sched >> 6 & 1) + 1; - - new_index = priv->virt2phys_pkey[slave][port - 1][orig_index]; - *(u8 *)(inbox->buf + 35) = new_index; -} - -static void update_gid(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *inbox, - u8 slave) -{ - struct mlx4_qp_context *qp_ctx = inbox->buf + 8; - enum mlx4_qp_optpar optpar = be32_to_cpu(*(__be32 *) inbox->buf); - u32 ts = (be32_to_cpu(qp_ctx->flags) >> 16) & 0xff; - int port; - - if (MLX4_QP_ST_UD == ts) { - port = (qp_ctx->pri_path.sched_queue >> 6 & 1) + 1; - if (mlx4_is_eth(dev, port)) - qp_ctx->pri_path.mgid_index = mlx4_get_base_gid_ix(dev, slave) | 0x80; - else - qp_ctx->pri_path.mgid_index = 0x80 | slave; - - } else if (MLX4_QP_ST_RC == ts || MLX4_QP_ST_UC == ts) { - if (optpar & MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH) { - port = (qp_ctx->pri_path.sched_queue >> 6 & 1) + 1; - if (mlx4_is_eth(dev, port)) { - qp_ctx->pri_path.mgid_index += mlx4_get_base_gid_ix(dev, slave); - qp_ctx->pri_path.mgid_index &= 0x7f; - } else { - qp_ctx->pri_path.mgid_index = slave & 0x7F; - } - } - if (optpar & MLX4_QP_OPTPAR_ALT_ADDR_PATH) { - port = (qp_ctx->alt_path.sched_queue >> 6 & 1) + 1; - if (mlx4_is_eth(dev, port)) { - qp_ctx->alt_path.mgid_index += mlx4_get_base_gid_ix(dev, slave); - qp_ctx->alt_path.mgid_index &= 0x7f; - } else { - qp_ctx->alt_path.mgid_index = slave & 0x7F; - } - } - } -} - -static int check_counter_index_validity(struct mlx4_dev *dev, int slave, int port, int idx) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct counter_index *counter, *tmp_counter; - - if (slave == 0) { - list_for_each_entry_safe(counter, tmp_counter, - &priv->counters_table.global_port_list[port - 1], - list) { - if (counter->index == idx) - return 0; - } - return -EINVAL; - } else { - list_for_each_entry_safe(counter, tmp_counter, - &priv->counters_table.vf_list[slave - 1][port - 1], - list) { - if (counter->index == idx) - return 0; - } - return -EINVAL; - } -} - -static int update_vport_qp_param(struct mlx4_dev *dev, - struct mlx4_cmd_mailbox *inbox, - u8 slave, u32 qpn) -{ - struct mlx4_qp_context *qpc = inbox->buf + 8; - struct mlx4_vport_oper_state *vp_oper; - struct mlx4_priv *priv; - u32 qp_type; - int port; - - port = (qpc->pri_path.sched_queue & 0x40) ? 2 : 1; - priv = mlx4_priv(dev); - vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; - qp_type = (be32_to_cpu(qpc->flags) >> 16) & 0xff; - - if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH && - qpc->pri_path.counter_index != MLX4_SINK_COUNTER_INDEX) { - if (check_counter_index_validity(dev, slave, port, - qpc->pri_path.counter_index)) - return -EINVAL; - } - - mlx4_dbg(dev, "%s: QP counter_index %d for slave %d port %d\n", - __func__, qpc->pri_path.counter_index, slave, port); - - if ((dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_LB_SRC_CHK) && - dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH && - !mlx4_is_qp_reserved(dev, qpn) && - qp_type == MLX4_QP_ST_MLX && - qpc->pri_path.counter_index != 0xFF) { - /* disable multicast loopback to qp with same counter */ - qpc->pri_path.fl |= MLX4_FL_ETH_SRC_CHECK_MC_LB; - qpc->pri_path.vlan_control |= - MLX4_VLAN_CTRL_ETH_SRC_CHECK_IF_COUNTER; - } - - if (MLX4_VGT != vp_oper->state.default_vlan) { - /* the reserved QPs (special, proxy, tunnel) - * do not operate over vlans - */ - if (mlx4_is_qp_reserved(dev, qpn)) - return 0; - - /* force strip vlan by clear vsd */ - qpc->param3 &= ~cpu_to_be32(MLX4_STRIP_VLAN); - /* preserve IF_COUNTER flag */ - qpc->pri_path.vlan_control &= - MLX4_VLAN_CTRL_ETH_SRC_CHECK_IF_COUNTER; - if (MLX4_QP_ST_RC != qp_type) { - if (0 != vp_oper->state.default_vlan) { - qpc->pri_path.vlan_control |= - MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED | - MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED | - MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED; - } else { /* priority tagged */ - qpc->pri_path.vlan_control |= - MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED | - MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED; - } - } - qpc->pri_path.fvl_rx |= MLX4_FVL_RX_FORCE_ETH_VLAN; - qpc->pri_path.vlan_index = vp_oper->vlan_idx; - qpc->pri_path.fl |= MLX4_FL_CV | MLX4_FL_ETH_HIDE_CQE_VLAN; - qpc->pri_path.feup |= MLX4_FEUP_FORCE_ETH_UP | MLX4_FVL_FORCE_ETH_VLAN; - qpc->pri_path.sched_queue &= 0xC7; - qpc->pri_path.sched_queue |= (vp_oper->state.default_qos) << 3; - } - if (vp_oper->state.spoofchk) { - qpc->pri_path.feup |= MLX4_FSM_FORCE_ETH_SRC_MAC; - qpc->pri_path.grh_mylmc = (0x80 & qpc->pri_path.grh_mylmc) + vp_oper->mac_idx; - } - return 0; -} - -static int mpt_mask(struct mlx4_dev *dev) -{ - return dev->caps.num_mpts - 1; -} - -static void *find_res(struct mlx4_dev *dev, u64 res_id, - enum mlx4_resource type) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - return res_tracker_lookup(&priv->mfunc.master.res_tracker.res_tree[type], - res_id); -} - -static int get_res(struct mlx4_dev *dev, int slave, u64 res_id, - enum mlx4_resource type, - void *res) -{ - struct res_common *r; - int err = 0; - - spin_lock_irq(mlx4_tlock(dev)); - r = find_res(dev, res_id, type); - if (!r) { - err = -ENONET; - goto exit; - } - - if (r->state == RES_ANY_BUSY) { - err = -EBUSY; - goto exit; - } - - if (r->owner != slave) { - err = -EPERM; - goto exit; - } - - r->from_state = r->state; - r->state = RES_ANY_BUSY; - - if (res) - *((struct res_common **)res) = r; - -exit: - spin_unlock_irq(mlx4_tlock(dev)); - return err; -} - -int mlx4_get_slave_from_resource_id(struct mlx4_dev *dev, - enum mlx4_resource type, - u64 res_id, int *slave) -{ - - struct res_common *r; - int err = -ENOENT; - int id = res_id; - - if (type == RES_QP) - id &= 0x7fffff; - spin_lock(mlx4_tlock(dev)); - - r = find_res(dev, id, type); - if (r) { - *slave = r->owner; - err = 0; - } - spin_unlock(mlx4_tlock(dev)); - - return err; -} - -static void put_res(struct mlx4_dev *dev, int slave, u64 res_id, - enum mlx4_resource type) -{ - struct res_common *r; - - spin_lock_irq(mlx4_tlock(dev)); - r = find_res(dev, res_id, type); - if (r) - r->state = r->from_state; - spin_unlock_irq(mlx4_tlock(dev)); -} - -static struct res_common *alloc_qp_tr(int id) -{ - struct res_qp *ret; - - ret = kzalloc(sizeof *ret, GFP_KERNEL); - if (!ret) - return NULL; - - ret->com.res_id = id; - ret->com.state = RES_QP_RESERVED; - ret->local_qpn = id; - INIT_LIST_HEAD(&ret->mcg_list); - spin_lock_init(&ret->mcg_spl); - atomic_set(&ret->ref_count, 0); - - return &ret->com; -} - -static struct res_common *alloc_mtt_tr(int id, int order) -{ - struct res_mtt *ret; - - ret = kzalloc(sizeof *ret, GFP_KERNEL); - if (!ret) - return NULL; - - ret->com.res_id = id; - ret->order = order; - ret->com.state = RES_MTT_ALLOCATED; - atomic_set(&ret->ref_count, 0); - - return &ret->com; -} - -static struct res_common *alloc_mpt_tr(int id, int key) -{ - struct res_mpt *ret; - - ret = kzalloc(sizeof *ret, GFP_KERNEL); - if (!ret) - return NULL; - - ret->com.res_id = id; - ret->com.state = RES_MPT_RESERVED; - ret->key = key; - - return &ret->com; -} - -static struct res_common *alloc_eq_tr(int id) -{ - struct res_eq *ret; - - ret = kzalloc(sizeof *ret, GFP_KERNEL); - if (!ret) - return NULL; - - ret->com.res_id = id; - ret->com.state = RES_EQ_RESERVED; - - return &ret->com; -} - -static struct res_common *alloc_cq_tr(int id) -{ - struct res_cq *ret; - - ret = kzalloc(sizeof *ret, GFP_KERNEL); - if (!ret) - return NULL; - - ret->com.res_id = id; - ret->com.state = RES_CQ_ALLOCATED; - atomic_set(&ret->ref_count, 0); - - return &ret->com; -} - -static struct res_common *alloc_srq_tr(int id) -{ - struct res_srq *ret; - - ret = kzalloc(sizeof *ret, GFP_KERNEL); - if (!ret) - return NULL; - - ret->com.res_id = id; - ret->com.state = RES_SRQ_ALLOCATED; - atomic_set(&ret->ref_count, 0); - - return &ret->com; -} - -static struct res_common *alloc_counter_tr(int id) -{ - struct res_counter *ret; - - ret = kzalloc(sizeof *ret, GFP_KERNEL); - if (!ret) - return NULL; - - ret->com.res_id = id; - ret->com.state = RES_COUNTER_ALLOCATED; - - return &ret->com; -} - -static struct res_common *alloc_xrcdn_tr(int id) -{ - struct res_xrcdn *ret; - - ret = kzalloc(sizeof *ret, GFP_KERNEL); - if (!ret) - return NULL; - - ret->com.res_id = id; - ret->com.state = RES_XRCD_ALLOCATED; - - return &ret->com; -} - -static struct res_common *alloc_fs_rule_tr(u64 id, int qpn) -{ - struct res_fs_rule *ret; - - ret = kzalloc(sizeof *ret, GFP_KERNEL); - if (!ret) - return NULL; - - ret->com.res_id = id; - ret->com.state = RES_FS_RULE_ALLOCATED; - ret->qpn = qpn; - return &ret->com; -} - -static struct res_common *alloc_tr(u64 id, enum mlx4_resource type, int slave, - int extra) -{ - struct res_common *ret; - - switch (type) { - case RES_QP: - ret = alloc_qp_tr(id); - break; - case RES_MPT: - ret = alloc_mpt_tr(id, extra); - break; - case RES_MTT: - ret = alloc_mtt_tr(id, extra); - break; - case RES_EQ: - ret = alloc_eq_tr(id); - break; - case RES_CQ: - ret = alloc_cq_tr(id); - break; - case RES_SRQ: - ret = alloc_srq_tr(id); - break; - case RES_MAC: - printk(KERN_ERR "implementation missing\n"); - return NULL; - case RES_COUNTER: - ret = alloc_counter_tr(id); - break; - case RES_XRCD: - ret = alloc_xrcdn_tr(id); - break; - case RES_FS_RULE: - ret = alloc_fs_rule_tr(id, extra); - break; - default: - return NULL; - } - if (ret) - ret->owner = slave; - - return ret; -} - -static int add_res_range(struct mlx4_dev *dev, int slave, u64 base, int count, - enum mlx4_resource type, int extra) -{ - int i; - int err; - struct mlx4_priv *priv = mlx4_priv(dev); - struct res_common **res_arr; - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct rb_root *root = &tracker->res_tree[type]; - - res_arr = kzalloc(count * sizeof *res_arr, GFP_KERNEL); - if (!res_arr) - return -ENOMEM; - - for (i = 0; i < count; ++i) { - res_arr[i] = alloc_tr(base + i, type, slave, extra); - if (!res_arr[i]) { - for (--i; i >= 0; --i) - kfree(res_arr[i]); - - kfree(res_arr); - return -ENOMEM; - } - } - - spin_lock_irq(mlx4_tlock(dev)); - for (i = 0; i < count; ++i) { - if (find_res(dev, base + i, type)) { - err = -EEXIST; - goto undo; - } - err = res_tracker_insert(root, res_arr[i]); - if (err) - goto undo; - list_add_tail(&res_arr[i]->list, - &tracker->slave_list[slave].res_list[type]); - } - spin_unlock_irq(mlx4_tlock(dev)); - kfree(res_arr); - - return 0; - -undo: - for (--i; i >= 0; --i) { - rb_erase(&res_arr[i]->node, root); - list_del_init(&res_arr[i]->list); - } - - spin_unlock_irq(mlx4_tlock(dev)); - - for (i = 0; i < count; ++i) - kfree(res_arr[i]); - - kfree(res_arr); - - return err; -} - -static int remove_qp_ok(struct res_qp *res) -{ - if (res->com.state == RES_QP_BUSY || atomic_read(&res->ref_count) || - !list_empty(&res->mcg_list)) { - pr_err("resource tracker: fail to remove qp, state %d, ref_count %d\n", - res->com.state, atomic_read(&res->ref_count)); - return -EBUSY; - } else if (res->com.state != RES_QP_RESERVED) { - return -EPERM; - } - - return 0; -} - -static int remove_mtt_ok(struct res_mtt *res, int order) -{ - if (res->com.state == RES_MTT_BUSY || - atomic_read(&res->ref_count)) { - printk(KERN_DEBUG "%s-%d: state %s, ref_count %d\n", - __func__, __LINE__, - mtt_states_str(res->com.state), - atomic_read(&res->ref_count)); - return -EBUSY; - } else if (res->com.state != RES_MTT_ALLOCATED) - return -EPERM; - else if (res->order != order) - return -EINVAL; - - return 0; -} - -static int remove_mpt_ok(struct res_mpt *res) -{ - if (res->com.state == RES_MPT_BUSY) - return -EBUSY; - else if (res->com.state != RES_MPT_RESERVED) - return -EPERM; - - return 0; -} - -static int remove_eq_ok(struct res_eq *res) -{ - if (res->com.state == RES_MPT_BUSY) - return -EBUSY; - else if (res->com.state != RES_MPT_RESERVED) - return -EPERM; - - return 0; -} - -static int remove_counter_ok(struct res_counter *res) -{ - if (res->com.state == RES_COUNTER_BUSY) - return -EBUSY; - else if (res->com.state != RES_COUNTER_ALLOCATED) - return -EPERM; - - return 0; -} - -static int remove_xrcdn_ok(struct res_xrcdn *res) -{ - if (res->com.state == RES_XRCD_BUSY) - return -EBUSY; - else if (res->com.state != RES_XRCD_ALLOCATED) - return -EPERM; - - return 0; -} - -static int remove_fs_rule_ok(struct res_fs_rule *res) -{ - if (res->com.state == RES_FS_RULE_BUSY) - return -EBUSY; - else if (res->com.state != RES_FS_RULE_ALLOCATED) - return -EPERM; - - return 0; -} - -static int remove_cq_ok(struct res_cq *res) -{ - if (res->com.state == RES_CQ_BUSY) - return -EBUSY; - else if (res->com.state != RES_CQ_ALLOCATED) - return -EPERM; - - return 0; -} - -static int remove_srq_ok(struct res_srq *res) -{ - if (res->com.state == RES_SRQ_BUSY) - return -EBUSY; - else if (res->com.state != RES_SRQ_ALLOCATED) - return -EPERM; - - return 0; -} - -static int remove_ok(struct res_common *res, enum mlx4_resource type, int extra) -{ - switch (type) { - case RES_QP: - return remove_qp_ok((struct res_qp *)res); - case RES_CQ: - return remove_cq_ok((struct res_cq *)res); - case RES_SRQ: - return remove_srq_ok((struct res_srq *)res); - case RES_MPT: - return remove_mpt_ok((struct res_mpt *)res); - case RES_MTT: - return remove_mtt_ok((struct res_mtt *)res, extra); - case RES_MAC: - return -ENOSYS; - case RES_EQ: - return remove_eq_ok((struct res_eq *)res); - case RES_COUNTER: - return remove_counter_ok((struct res_counter *)res); - case RES_XRCD: - return remove_xrcdn_ok((struct res_xrcdn *)res); - case RES_FS_RULE: - return remove_fs_rule_ok((struct res_fs_rule *)res); - default: - return -EINVAL; - } -} - -static int rem_res_range(struct mlx4_dev *dev, int slave, u64 base, int count, - enum mlx4_resource type, int extra) -{ - u64 i; - int err; - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct res_common *r; - - spin_lock_irq(mlx4_tlock(dev)); - for (i = base; i < base + count; ++i) { - r = res_tracker_lookup(&tracker->res_tree[type], i); - if (!r) { - err = -ENOENT; - goto out; - } - if (r->owner != slave) { - err = -EPERM; - goto out; - } - err = remove_ok(r, type, extra); - if (err) - goto out; - } - - for (i = base; i < base + count; ++i) { - r = res_tracker_lookup(&tracker->res_tree[type], i); - rb_erase(&r->node, &tracker->res_tree[type]); - list_del(&r->list); - kfree(r); - } - err = 0; - -out: - spin_unlock_irq(mlx4_tlock(dev)); - - return err; -} - -static int qp_res_start_move_to(struct mlx4_dev *dev, int slave, int qpn, - enum res_qp_states state, struct res_qp **qp, - int alloc) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct res_qp *r; - int err = 0; - - spin_lock_irq(mlx4_tlock(dev)); - r = res_tracker_lookup(&tracker->res_tree[RES_QP], qpn); - if (!r) - err = -ENOENT; - else if (r->com.owner != slave) - err = -EPERM; - else { - switch (state) { - case RES_QP_BUSY: - mlx4_dbg(dev, "%s: failed RES_QP, 0x%llx\n", - __func__, (unsigned long long)r->com.res_id); - err = -EBUSY; - break; - - case RES_QP_RESERVED: - if (r->com.state == RES_QP_MAPPED && !alloc) - break; - - mlx4_dbg(dev, "failed RES_QP, 0x%llx\n", (unsigned long long)r->com.res_id); - err = -EINVAL; - break; - - case RES_QP_MAPPED: - if ((r->com.state == RES_QP_RESERVED && alloc) || - r->com.state == RES_QP_HW) - break; - else { - mlx4_dbg(dev, "failed RES_QP, 0x%llx\n", - (unsigned long long)r->com.res_id); - err = -EINVAL; - } - - break; - - case RES_QP_HW: - if (r->com.state != RES_QP_MAPPED) - err = -EINVAL; - break; - default: - err = -EINVAL; - } - - if (!err) { - r->com.from_state = r->com.state; - r->com.to_state = state; - r->com.state = RES_QP_BUSY; - if (qp) - *qp = r; - } - } - - spin_unlock_irq(mlx4_tlock(dev)); - - return err; -} - -static int mr_res_start_move_to(struct mlx4_dev *dev, int slave, int index, - enum res_mpt_states state, struct res_mpt **mpt) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct res_mpt *r; - int err = 0; - - spin_lock_irq(mlx4_tlock(dev)); - r = res_tracker_lookup(&tracker->res_tree[RES_MPT], index); - if (!r) - err = -ENOENT; - else if (r->com.owner != slave) - err = -EPERM; - else { - switch (state) { - case RES_MPT_BUSY: - err = -EINVAL; - break; - - case RES_MPT_RESERVED: - if (r->com.state != RES_MPT_MAPPED) - err = -EINVAL; - break; - - case RES_MPT_MAPPED: - if (r->com.state != RES_MPT_RESERVED && - r->com.state != RES_MPT_HW) - err = -EINVAL; - break; - - case RES_MPT_HW: - if (r->com.state != RES_MPT_MAPPED) - err = -EINVAL; - break; - default: - err = -EINVAL; - } - - if (!err) { - r->com.from_state = r->com.state; - r->com.to_state = state; - r->com.state = RES_MPT_BUSY; - if (mpt) - *mpt = r; - } - } - - spin_unlock_irq(mlx4_tlock(dev)); - - return err; -} - -static int eq_res_start_move_to(struct mlx4_dev *dev, int slave, int index, - enum res_eq_states state, struct res_eq **eq) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct res_eq *r; - int err = 0; - - spin_lock_irq(mlx4_tlock(dev)); - r = res_tracker_lookup(&tracker->res_tree[RES_EQ], index); - if (!r) - err = -ENOENT; - else if (r->com.owner != slave) - err = -EPERM; - else { - switch (state) { - case RES_EQ_BUSY: - err = -EINVAL; - break; - - case RES_EQ_RESERVED: - if (r->com.state != RES_EQ_HW) - err = -EINVAL; - break; - - case RES_EQ_HW: - if (r->com.state != RES_EQ_RESERVED) - err = -EINVAL; - break; - - default: - err = -EINVAL; - } - - if (!err) { - r->com.from_state = r->com.state; - r->com.to_state = state; - r->com.state = RES_EQ_BUSY; - if (eq) - *eq = r; - } - } - - spin_unlock_irq(mlx4_tlock(dev)); - - return err; -} - -static int cq_res_start_move_to(struct mlx4_dev *dev, int slave, int cqn, - enum res_cq_states state, struct res_cq **cq) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct res_cq *r; - int err; - - spin_lock_irq(mlx4_tlock(dev)); - r = res_tracker_lookup(&tracker->res_tree[RES_CQ], cqn); - if (!r) - err = -ENOENT; - else if (r->com.owner != slave) - err = -EPERM; - else { - switch (state) { - case RES_CQ_BUSY: - err = -EBUSY; - break; - - case RES_CQ_ALLOCATED: - if (r->com.state != RES_CQ_HW) - err = -EINVAL; - else if (atomic_read(&r->ref_count)) - err = -EBUSY; - else - err = 0; - break; - - case RES_CQ_HW: - if (r->com.state != RES_CQ_ALLOCATED) - err = -EINVAL; - else - err = 0; - break; - - default: - err = -EINVAL; - } - - if (!err) { - r->com.from_state = r->com.state; - r->com.to_state = state; - r->com.state = RES_CQ_BUSY; - if (cq) - *cq = r; - } - } - - spin_unlock_irq(mlx4_tlock(dev)); - - return err; -} - -static int srq_res_start_move_to(struct mlx4_dev *dev, int slave, int index, - enum res_srq_states state, struct res_srq **srq) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct res_srq *r; - int err = 0; - - spin_lock_irq(mlx4_tlock(dev)); - r = res_tracker_lookup(&tracker->res_tree[RES_SRQ], index); - if (!r) - err = -ENOENT; - else if (r->com.owner != slave) - err = -EPERM; - else { - switch (state) { - case RES_SRQ_BUSY: - err = -EINVAL; - break; - - case RES_SRQ_ALLOCATED: - if (r->com.state != RES_SRQ_HW) - err = -EINVAL; - else if (atomic_read(&r->ref_count)) - err = -EBUSY; - break; - - case RES_SRQ_HW: - if (r->com.state != RES_SRQ_ALLOCATED) - err = -EINVAL; - break; - - default: - err = -EINVAL; - } - - if (!err) { - r->com.from_state = r->com.state; - r->com.to_state = state; - r->com.state = RES_SRQ_BUSY; - if (srq) - *srq = r; - } - } - - spin_unlock_irq(mlx4_tlock(dev)); - - return err; -} - -static void res_abort_move(struct mlx4_dev *dev, int slave, - enum mlx4_resource type, int id) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct res_common *r; - - spin_lock_irq(mlx4_tlock(dev)); - r = res_tracker_lookup(&tracker->res_tree[type], id); - if (r && (r->owner == slave)) - r->state = r->from_state; - spin_unlock_irq(mlx4_tlock(dev)); -} - -static void res_end_move(struct mlx4_dev *dev, int slave, - enum mlx4_resource type, int id) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct res_common *r; - - spin_lock_irq(mlx4_tlock(dev)); - r = res_tracker_lookup(&tracker->res_tree[type], id); - if (r && (r->owner == slave)) - r->state = r->to_state; - spin_unlock_irq(mlx4_tlock(dev)); -} - -static int valid_reserved(struct mlx4_dev *dev, int slave, int qpn) -{ - return mlx4_is_qp_reserved(dev, qpn) && - (mlx4_is_master(dev) || mlx4_is_guest_proxy(dev, slave, qpn)); -} - -static int fw_reserved(struct mlx4_dev *dev, int qpn) -{ - return qpn < dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW]; -} - -static int qp_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param, u64 *out_param) -{ - int err; - int count; - int align; - int base; - int qpn; - u8 flags; - - switch (op) { - case RES_OP_RESERVE: - count = get_param_l(&in_param) & 0xffffff; - /* Turn off all unsupported QP allocation flags that the - * slave tries to set. - */ - flags = (get_param_l(&in_param) >> 24) & dev->caps.alloc_res_qp_mask; - align = get_param_h(&in_param); - err = mlx4_grant_resource(dev, slave, RES_QP, count, 0); - if (err) - return err; - - err = __mlx4_qp_reserve_range(dev, count, align, &base, flags); - if (err) { - mlx4_release_resource(dev, slave, RES_QP, count, 0); - return err; - } - - err = add_res_range(dev, slave, base, count, RES_QP, 0); - if (err) { - mlx4_release_resource(dev, slave, RES_QP, count, 0); - __mlx4_qp_release_range(dev, base, count); - return err; - } - set_param_l(out_param, base); - break; - case RES_OP_MAP_ICM: - qpn = get_param_l(&in_param) & 0x7fffff; - if (valid_reserved(dev, slave, qpn)) { - err = add_res_range(dev, slave, qpn, 1, RES_QP, 0); - if (err) - return err; - } - - err = qp_res_start_move_to(dev, slave, qpn, RES_QP_MAPPED, - NULL, 1); - if (err) - return err; - - if (!fw_reserved(dev, qpn)) { - err = __mlx4_qp_alloc_icm(dev, qpn); - if (err) { - res_abort_move(dev, slave, RES_QP, qpn); - return err; - } - } - - res_end_move(dev, slave, RES_QP, qpn); - break; - - default: - err = -EINVAL; - break; - } - return err; -} - -static int mtt_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param, u64 *out_param) -{ - int err = -EINVAL; - int base; - int order; - - if (op != RES_OP_RESERVE_AND_MAP) - return err; - - order = get_param_l(&in_param); - - err = mlx4_grant_resource(dev, slave, RES_MTT, 1 << order, 0); - if (err) - return err; - - base = __mlx4_alloc_mtt_range(dev, order); - if (base == -1) { - mlx4_release_resource(dev, slave, RES_MTT, 1 << order, 0); - return -ENOMEM; - } - - err = add_res_range(dev, slave, base, 1, RES_MTT, order); - if (err) { - mlx4_release_resource(dev, slave, RES_MTT, 1 << order, 0); - __mlx4_free_mtt_range(dev, base, order); - } else - set_param_l(out_param, base); - - return err; -} - -static int mpt_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param, u64 *out_param) -{ - int err = -EINVAL; - int index; - int id; - struct res_mpt *mpt; - - switch (op) { - case RES_OP_RESERVE: - err = mlx4_grant_resource(dev, slave, RES_MPT, 1, 0); - if (err) - break; - - index = __mlx4_mpt_reserve(dev); - if (index == -1) { - mlx4_release_resource(dev, slave, RES_MPT, 1, 0); - break; - } - id = index & mpt_mask(dev); - - err = add_res_range(dev, slave, id, 1, RES_MPT, index); - if (err) { - mlx4_release_resource(dev, slave, RES_MPT, 1, 0); - __mlx4_mpt_release(dev, index); - break; - } - set_param_l(out_param, index); - break; - case RES_OP_MAP_ICM: - index = get_param_l(&in_param); - id = index & mpt_mask(dev); - err = mr_res_start_move_to(dev, slave, id, - RES_MPT_MAPPED, &mpt); - if (err) - return err; - - err = __mlx4_mpt_alloc_icm(dev, mpt->key); - if (err) { - res_abort_move(dev, slave, RES_MPT, id); - return err; - } - - res_end_move(dev, slave, RES_MPT, id); - break; - } - return err; -} - -static int cq_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param, u64 *out_param) -{ - int cqn; - int err; - - switch (op) { - case RES_OP_RESERVE_AND_MAP: - err = mlx4_grant_resource(dev, slave, RES_CQ, 1, 0); - if (err) - break; - - err = __mlx4_cq_alloc_icm(dev, &cqn); - if (err) { - mlx4_release_resource(dev, slave, RES_CQ, 1, 0); - break; - } - - err = add_res_range(dev, slave, cqn, 1, RES_CQ, 0); - if (err) { - mlx4_release_resource(dev, slave, RES_CQ, 1, 0); - __mlx4_cq_free_icm(dev, cqn); - break; - } - - set_param_l(out_param, cqn); - break; - - default: - err = -EINVAL; - } - - return err; -} - -static int srq_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param, u64 *out_param) -{ - int srqn; - int err; - - switch (op) { - case RES_OP_RESERVE_AND_MAP: - err = mlx4_grant_resource(dev, slave, RES_SRQ, 1, 0); - if (err) - break; - - err = __mlx4_srq_alloc_icm(dev, &srqn); - if (err) { - mlx4_release_resource(dev, slave, RES_SRQ, 1, 0); - break; - } - - err = add_res_range(dev, slave, srqn, 1, RES_SRQ, 0); - if (err) { - mlx4_release_resource(dev, slave, RES_SRQ, 1, 0); - __mlx4_srq_free_icm(dev, srqn); - break; - } - - set_param_l(out_param, srqn); - break; - - default: - err = -EINVAL; - } - - return err; -} - -static int mac_find_smac_ix_in_slave(struct mlx4_dev *dev, int slave, int port, - u8 smac_index, u64 *mac) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct list_head *mac_list = - &tracker->slave_list[slave].res_list[RES_MAC]; - struct mac_res *res, *tmp; - - list_for_each_entry_safe(res, tmp, mac_list, list) { - if (res->smac_index == smac_index && res->port == (u8) port) { - *mac = res->mac; - return 0; - } - } - return -ENOENT; -} - -static int mac_add_to_slave(struct mlx4_dev *dev, int slave, u64 mac, int port, u8 smac_index) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct list_head *mac_list = - &tracker->slave_list[slave].res_list[RES_MAC]; - struct mac_res *res, *tmp; - - list_for_each_entry_safe(res, tmp, mac_list, list) { - if (res->mac == mac && res->port == (u8) port) { - /* mac found. update ref count */ - ++res->ref_count; - return 0; - } - } - - if (mlx4_grant_resource(dev, slave, RES_MAC, 1, port)) - return -EINVAL; - res = kzalloc(sizeof *res, GFP_KERNEL); - if (!res) { - mlx4_release_resource(dev, slave, RES_MAC, 1, port); - return -ENOMEM; - } - res->mac = mac; - res->port = (u8) port; - res->smac_index = smac_index; - res->ref_count = 1; - list_add_tail(&res->list, - &tracker->slave_list[slave].res_list[RES_MAC]); - return 0; -} - - -static void mac_del_from_slave(struct mlx4_dev *dev, int slave, u64 mac, - int port) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct list_head *mac_list = - &tracker->slave_list[slave].res_list[RES_MAC]; - struct mac_res *res, *tmp; - - list_for_each_entry_safe(res, tmp, mac_list, list) { - if (res->mac == mac && res->port == (u8) port) { - if (!--res->ref_count) { - list_del(&res->list); - mlx4_release_resource(dev, slave, RES_MAC, 1, port); - kfree(res); - } - break; - } - } -} - -static void rem_slave_macs(struct mlx4_dev *dev, int slave) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct list_head *mac_list = - &tracker->slave_list[slave].res_list[RES_MAC]; - struct mac_res *res, *tmp; - int i; - - list_for_each_entry_safe(res, tmp, mac_list, list) { - list_del(&res->list); - /* dereference the mac the num times the slave referenced it */ - for (i = 0; i < res->ref_count; i++) - __mlx4_unregister_mac(dev, res->port, res->mac); - mlx4_release_resource(dev, slave, RES_MAC, 1, res->port); - kfree(res); - } -} - -static int mac_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param, u64 *out_param, int in_port) -{ - int err = -EINVAL; - int port; - u64 mac; - u8 smac_index = 0; - - if (op != RES_OP_RESERVE_AND_MAP) - return err; - - port = !in_port ? get_param_l(out_param) : in_port; - mac = in_param; - - err = __mlx4_register_mac(dev, port, mac); - if (err >= 0) { - smac_index = err; - set_param_l(out_param, err); - err = 0; - } - - if (!err) { - err = mac_add_to_slave(dev, slave, mac, port, smac_index); - if (err) - __mlx4_unregister_mac(dev, port, mac); - } - return err; -} - -static int vlan_add_to_slave(struct mlx4_dev *dev, int slave, u16 vlan, - int port, int vlan_index) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct list_head *vlan_list = - &tracker->slave_list[slave].res_list[RES_VLAN]; - struct vlan_res *res, *tmp; - - list_for_each_entry_safe(res, tmp, vlan_list, list) { - if (res->vlan == vlan && res->port == (u8) port) { - /* vlan found. update ref count */ - ++res->ref_count; - return 0; - } - } - - if (mlx4_grant_resource(dev, slave, RES_VLAN, 1, port)) - return -EINVAL; - res = kzalloc(sizeof(*res), GFP_KERNEL); - if (!res) { - mlx4_release_resource(dev, slave, RES_VLAN, 1, port); - return -ENOMEM; - } - res->vlan = vlan; - res->port = (u8) port; - res->vlan_index = vlan_index; - res->ref_count = 1; - list_add_tail(&res->list, - &tracker->slave_list[slave].res_list[RES_VLAN]); - return 0; -} - - -static void vlan_del_from_slave(struct mlx4_dev *dev, int slave, u16 vlan, - int port) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct list_head *vlan_list = - &tracker->slave_list[slave].res_list[RES_VLAN]; - struct vlan_res *res, *tmp; - - list_for_each_entry_safe(res, tmp, vlan_list, list) { - if (res->vlan == vlan && res->port == (u8) port) { - if (!--res->ref_count) { - list_del(&res->list); - mlx4_release_resource(dev, slave, RES_VLAN, - 1, port); - kfree(res); - } - break; - } - } -} - -static void rem_slave_vlans(struct mlx4_dev *dev, int slave) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct list_head *vlan_list = - &tracker->slave_list[slave].res_list[RES_VLAN]; - struct vlan_res *res, *tmp; - int i; - - list_for_each_entry_safe(res, tmp, vlan_list, list) { - list_del(&res->list); - /* dereference the vlan the num times the slave referenced it */ - for (i = 0; i < res->ref_count; i++) - __mlx4_unregister_vlan(dev, res->port, res->vlan); - mlx4_release_resource(dev, slave, RES_VLAN, 1, res->port); - kfree(res); - } -} - -static int vlan_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param, u64 *out_param, int in_port) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state; - int err = -EINVAL; - u16 vlan; - int vlan_index; - int port; - - port = !in_port ? get_param_l(out_param) : in_port; - - if (!port) - return err; - - if (op != RES_OP_RESERVE_AND_MAP) - return err; - - /* upstream kernels had NOP for reg/unreg vlan. Continue this. */ - if (!in_port && port > 0 && port <= dev->caps.num_ports) { - slave_state[slave].old_vlan_api = true; - return 0; - } - - vlan = (u16) in_param; - - err = __mlx4_register_vlan(dev, port, vlan, &vlan_index); - if (!err) { - set_param_l(out_param, (u32) vlan_index); - err = vlan_add_to_slave(dev, slave, vlan, port, vlan_index); - if (err) - __mlx4_unregister_vlan(dev, port, vlan); - } - return err; -} - -static int counter_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param, u64 *out_param, int port) -{ - u32 index; - int err; - - if (op != RES_OP_RESERVE) - return -EINVAL; - - err = __mlx4_counter_alloc(dev, slave, port, &index); - if (!err) - set_param_l(out_param, index); - - return err; -} - -static int xrcdn_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param, u64 *out_param) -{ - u32 xrcdn; - int err; - - if (op != RES_OP_RESERVE) - return -EINVAL; - - err = __mlx4_xrcd_alloc(dev, &xrcdn); - if (err) - return err; - - err = add_res_range(dev, slave, xrcdn, 1, RES_XRCD, 0); - if (err) - __mlx4_xrcd_free(dev, xrcdn); - else - set_param_l(out_param, xrcdn); - - return err; -} - -int mlx4_ALLOC_RES_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - int alop = vhcr->op_modifier; - - switch (vhcr->in_modifier & 0xFF) { - case RES_QP: - err = qp_alloc_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param, &vhcr->out_param); - break; - - case RES_MTT: - err = mtt_alloc_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param, &vhcr->out_param); - break; - - case RES_MPT: - err = mpt_alloc_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param, &vhcr->out_param); - break; - - case RES_CQ: - err = cq_alloc_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param, &vhcr->out_param); - break; - - case RES_SRQ: - err = srq_alloc_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param, &vhcr->out_param); - break; - - case RES_MAC: - err = mac_alloc_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param, &vhcr->out_param, - (vhcr->in_modifier >> 8) & 0xFF); - break; - - case RES_VLAN: - err = vlan_alloc_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param, &vhcr->out_param, - (vhcr->in_modifier >> 8) & 0xFF); - break; - - case RES_COUNTER: - err = counter_alloc_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param, &vhcr->out_param, - (vhcr->in_modifier >> 8) & 0xFF); - break; - - case RES_XRCD: - err = xrcdn_alloc_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param, &vhcr->out_param); - break; - - default: - err = -EINVAL; - break; - } - - return err; -} - -static int qp_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param) -{ - int err; - int count; - int base; - int qpn; - - switch (op) { - case RES_OP_RESERVE: - base = get_param_l(&in_param) & 0x7fffff; - count = get_param_h(&in_param); - err = rem_res_range(dev, slave, base, count, RES_QP, 0); - if (err) - break; - mlx4_release_resource(dev, slave, RES_QP, count, 0); - __mlx4_qp_release_range(dev, base, count); - break; - case RES_OP_MAP_ICM: - qpn = get_param_l(&in_param) & 0x7fffff; - err = qp_res_start_move_to(dev, slave, qpn, RES_QP_RESERVED, - NULL, 0); - if (err) - return err; - - if (!fw_reserved(dev, qpn)) - __mlx4_qp_free_icm(dev, qpn); - - res_end_move(dev, slave, RES_QP, qpn); - - if (valid_reserved(dev, slave, qpn)) - err = rem_res_range(dev, slave, qpn, 1, RES_QP, 0); - break; - default: - err = -EINVAL; - break; - } - return err; -} - -static int mtt_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param, u64 *out_param) -{ - int err = -EINVAL; - int base; - int order; - - if (op != RES_OP_RESERVE_AND_MAP) - return err; - - base = get_param_l(&in_param); - order = get_param_h(&in_param); - err = rem_res_range(dev, slave, base, 1, RES_MTT, order); - if (!err) { - mlx4_release_resource(dev, slave, RES_MTT, 1 << order, 0); - __mlx4_free_mtt_range(dev, base, order); - } - return err; -} - -static int mpt_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param) -{ - int err = -EINVAL; - int index; - int id; - struct res_mpt *mpt; - - switch (op) { - case RES_OP_RESERVE: - index = get_param_l(&in_param); - id = index & mpt_mask(dev); - err = get_res(dev, slave, id, RES_MPT, &mpt); - if (err) - break; - index = mpt->key; - put_res(dev, slave, id, RES_MPT); - - err = rem_res_range(dev, slave, id, 1, RES_MPT, 0); - if (err) - break; - mlx4_release_resource(dev, slave, RES_MPT, 1, 0); - __mlx4_mpt_release(dev, index); - break; - case RES_OP_MAP_ICM: - index = get_param_l(&in_param); - id = index & mpt_mask(dev); - err = mr_res_start_move_to(dev, slave, id, - RES_MPT_RESERVED, &mpt); - if (err) - return err; - - __mlx4_mpt_free_icm(dev, mpt->key); - res_end_move(dev, slave, RES_MPT, id); - return err; - break; - default: - err = -EINVAL; - break; - } - return err; -} - -static int cq_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param, u64 *out_param) -{ - int cqn; - int err; - - switch (op) { - case RES_OP_RESERVE_AND_MAP: - cqn = get_param_l(&in_param); - err = rem_res_range(dev, slave, cqn, 1, RES_CQ, 0); - if (err) - break; - - mlx4_release_resource(dev, slave, RES_CQ, 1, 0); - __mlx4_cq_free_icm(dev, cqn); - break; - - default: - err = -EINVAL; - break; - } - - return err; -} - -static int srq_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param, u64 *out_param) -{ - int srqn; - int err; - - switch (op) { - case RES_OP_RESERVE_AND_MAP: - srqn = get_param_l(&in_param); - err = rem_res_range(dev, slave, srqn, 1, RES_SRQ, 0); - if (err) - break; - - mlx4_release_resource(dev, slave, RES_SRQ, 1, 0); - __mlx4_srq_free_icm(dev, srqn); - break; - - default: - err = -EINVAL; - break; - } - - return err; -} - -static int mac_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param, u64 *out_param, int in_port) -{ - int port; - int err = 0; - - switch (op) { - case RES_OP_RESERVE_AND_MAP: - port = !in_port ? get_param_l(out_param) : in_port; - mac_del_from_slave(dev, slave, in_param, port); - __mlx4_unregister_mac(dev, port, in_param); - break; - default: - err = -EINVAL; - break; - } - - return err; - -} - -static int vlan_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param, u64 *out_param, int port) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state; - int err = 0; - - switch (op) { - case RES_OP_RESERVE_AND_MAP: - if (slave_state[slave].old_vlan_api == true) - return 0; - if (!port) - return -EINVAL; - vlan_del_from_slave(dev, slave, in_param, port); - __mlx4_unregister_vlan(dev, port, in_param); - break; - default: - err = -EINVAL; - break; - } - - return err; -} - -static int counter_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param, u64 *out_param, int port) -{ - int index; - - if (op != RES_OP_RESERVE) - return -EINVAL; - - index = get_param_l(&in_param); - - __mlx4_counter_free(dev, slave, port, index); - - return 0; -} - -static int xrcdn_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, - u64 in_param, u64 *out_param) -{ - int xrcdn; - int err; - - if (op != RES_OP_RESERVE) - return -EINVAL; - - xrcdn = get_param_l(&in_param); - err = rem_res_range(dev, slave, xrcdn, 1, RES_XRCD, 0); - if (err) - return err; - - __mlx4_xrcd_free(dev, xrcdn); - - return err; -} - -int mlx4_FREE_RES_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err = -EINVAL; - int alop = vhcr->op_modifier; - - switch (vhcr->in_modifier & 0xFF) { - case RES_QP: - err = qp_free_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param); - break; - - case RES_MTT: - err = mtt_free_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param, &vhcr->out_param); - break; - - case RES_MPT: - err = mpt_free_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param); - break; - - case RES_CQ: - err = cq_free_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param, &vhcr->out_param); - break; - - case RES_SRQ: - err = srq_free_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param, &vhcr->out_param); - break; - - case RES_MAC: - err = mac_free_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param, &vhcr->out_param, - (vhcr->in_modifier >> 8) & 0xFF); - break; - - case RES_VLAN: - err = vlan_free_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param, &vhcr->out_param, - (vhcr->in_modifier >> 8) & 0xFF); - break; - - case RES_COUNTER: - err = counter_free_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param, &vhcr->out_param, - (vhcr->in_modifier >> 8) & 0xFF); - break; - - case RES_XRCD: - err = xrcdn_free_res(dev, slave, vhcr->op_modifier, alop, - vhcr->in_param, &vhcr->out_param); - - default: - break; - } - return err; -} - -/* ugly but other choices are uglier */ -static int mr_phys_mpt(struct mlx4_mpt_entry *mpt) -{ - return (be32_to_cpu(mpt->flags) >> 9) & 1; -} - -static int mr_get_mtt_addr(struct mlx4_mpt_entry *mpt) -{ - return (int)be64_to_cpu(mpt->mtt_addr) & 0xfffffff8; -} - -static int mr_get_mtt_size(struct mlx4_mpt_entry *mpt) -{ - return be32_to_cpu(mpt->mtt_sz); -} - -static u32 mr_get_pd(struct mlx4_mpt_entry *mpt) -{ - return be32_to_cpu(mpt->pd_flags) & 0x00ffffff; -} - -static int mr_is_fmr(struct mlx4_mpt_entry *mpt) -{ - return be32_to_cpu(mpt->pd_flags) & MLX4_MPT_PD_FLAG_FAST_REG; -} - -static int mr_is_bind_enabled(struct mlx4_mpt_entry *mpt) -{ - return be32_to_cpu(mpt->flags) & MLX4_MPT_FLAG_BIND_ENABLE; -} - -static int mr_is_region(struct mlx4_mpt_entry *mpt) -{ - return be32_to_cpu(mpt->flags) & MLX4_MPT_FLAG_REGION; -} - -static int qp_get_mtt_addr(struct mlx4_qp_context *qpc) -{ - return be32_to_cpu(qpc->mtt_base_addr_l) & 0xfffffff8; -} - -static int srq_get_mtt_addr(struct mlx4_srq_context *srqc) -{ - return be32_to_cpu(srqc->mtt_base_addr_l) & 0xfffffff8; -} - -static int qp_get_mtt_size(struct mlx4_qp_context *qpc) -{ - int page_shift = (qpc->log_page_size & 0x3f) + 12; - int log_sq_size = (qpc->sq_size_stride >> 3) & 0xf; - int log_sq_sride = qpc->sq_size_stride & 7; - int log_rq_size = (qpc->rq_size_stride >> 3) & 0xf; - int log_rq_stride = qpc->rq_size_stride & 7; - int srq = (be32_to_cpu(qpc->srqn) >> 24) & 1; - int rss = (be32_to_cpu(qpc->flags) >> 13) & 1; - u32 ts = (be32_to_cpu(qpc->flags) >> 16) & 0xff; - int xrc = (ts == MLX4_QP_ST_XRC) ? 1 : 0; - int sq_size; - int rq_size; - int total_pages; - int total_mem; - int page_offset = (be32_to_cpu(qpc->params2) >> 6) & 0x3f; - - sq_size = 1 << (log_sq_size + log_sq_sride + 4); - rq_size = (srq|rss|xrc) ? 0 : (1 << (log_rq_size + log_rq_stride + 4)); - total_mem = sq_size + rq_size; - total_pages = - roundup_pow_of_two((total_mem + (page_offset << 6)) >> - page_shift); - - return total_pages; -} - -static int check_mtt_range(struct mlx4_dev *dev, int slave, int start, - int size, struct res_mtt *mtt) -{ - int res_start = mtt->com.res_id; - int res_size = (1 << mtt->order); - - if (start < res_start || start + size > res_start + res_size) - return -EPERM; - return 0; -} - -int mlx4_SW2HW_MPT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - int index = vhcr->in_modifier; - struct res_mtt *mtt; - struct res_mpt *mpt; - int mtt_base = mr_get_mtt_addr(inbox->buf) / dev->caps.mtt_entry_sz; - int phys; - int id; - u32 pd; - int pd_slave; - - id = index & mpt_mask(dev); - err = mr_res_start_move_to(dev, slave, id, RES_MPT_HW, &mpt); - if (err) - return err; - - /* Currently disable memory windows since this feature isn't tested yet - * under virtualization. - */ - if (!mr_is_region(inbox->buf)) { - err = -ENOSYS; - goto ex_abort; - } - - /* Make sure that the PD bits related to the slave id are zeros. */ - pd = mr_get_pd(inbox->buf); - pd_slave = (pd >> 17) & 0x7f; - if (pd_slave != 0 && pd_slave != slave) { - err = -EPERM; - goto ex_abort; - } - - if (mr_is_fmr(inbox->buf)) { - /* FMR and Bind Enable are forbidden in slave devices. */ - if (mr_is_bind_enabled(inbox->buf)) { - err = -EPERM; - goto ex_abort; - } - /* FMR and Memory Windows are also forbidden. */ - if (!mr_is_region(inbox->buf)) { - err = -EPERM; - goto ex_abort; - } - } - - phys = mr_phys_mpt(inbox->buf); - if (!phys) { - err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); - if (err) - goto ex_abort; - - err = check_mtt_range(dev, slave, mtt_base, - mr_get_mtt_size(inbox->buf), mtt); - if (err) - goto ex_put; - - mpt->mtt = mtt; - } - - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); - if (err) - goto ex_put; - - if (!phys) { - atomic_inc(&mtt->ref_count); - put_res(dev, slave, mtt->com.res_id, RES_MTT); - } - - res_end_move(dev, slave, RES_MPT, id); - return 0; - -ex_put: - if (!phys) - put_res(dev, slave, mtt->com.res_id, RES_MTT); -ex_abort: - res_abort_move(dev, slave, RES_MPT, id); - - return err; -} - -int mlx4_HW2SW_MPT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - int index = vhcr->in_modifier; - struct res_mpt *mpt; - int id; - - id = index & mpt_mask(dev); - err = mr_res_start_move_to(dev, slave, id, RES_MPT_MAPPED, &mpt); - if (err) - return err; - - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); - if (err) - goto ex_abort; - - if (mpt->mtt) - atomic_dec(&mpt->mtt->ref_count); - - res_end_move(dev, slave, RES_MPT, id); - return 0; - -ex_abort: - res_abort_move(dev, slave, RES_MPT, id); - - return err; -} - -int mlx4_QUERY_MPT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - int index = vhcr->in_modifier; - struct res_mpt *mpt; - int id; - - id = index & mpt_mask(dev); - err = get_res(dev, slave, id, RES_MPT, &mpt); - if (err) - return err; - - if (mpt->com.from_state != RES_MPT_HW) { - err = -EBUSY; - goto out; - } - - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); - -out: - put_res(dev, slave, id, RES_MPT); - return err; -} - -static int qp_get_rcqn(struct mlx4_qp_context *qpc) -{ - return be32_to_cpu(qpc->cqn_recv) & 0xffffff; -} - -static int qp_get_scqn(struct mlx4_qp_context *qpc) -{ - return be32_to_cpu(qpc->cqn_send) & 0xffffff; -} - -static u32 qp_get_srqn(struct mlx4_qp_context *qpc) -{ - return be32_to_cpu(qpc->srqn) & 0x1ffffff; -} - -static void adjust_proxy_tun_qkey(struct mlx4_dev *dev, struct mlx4_vhcr *vhcr, - struct mlx4_qp_context *context) -{ - u32 qpn = vhcr->in_modifier & 0xffffff; - u32 qkey = 0; - - if (mlx4_get_parav_qkey(dev, qpn, &qkey)) - return; - - /* adjust qkey in qp context */ - context->qkey = cpu_to_be32(qkey); -} - -int mlx4_RST2INIT_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - int qpn = vhcr->in_modifier & 0x7fffff; - struct res_mtt *mtt; - struct res_qp *qp; - struct mlx4_qp_context *qpc = inbox->buf + 8; - int mtt_base = qp_get_mtt_addr(qpc) / dev->caps.mtt_entry_sz; - int mtt_size = qp_get_mtt_size(qpc); - struct res_cq *rcq; - struct res_cq *scq; - int rcqn = qp_get_rcqn(qpc); - int scqn = qp_get_scqn(qpc); - u32 srqn = qp_get_srqn(qpc) & 0xffffff; - int use_srq = (qp_get_srqn(qpc) >> 24) & 1; - struct res_srq *srq; - int local_qpn = be32_to_cpu(qpc->local_qpn) & 0xffffff; - - err = qp_res_start_move_to(dev, slave, qpn, RES_QP_HW, &qp, 0); - if (err) - return err; - qp->local_qpn = local_qpn; - qp->sched_queue = 0; - qp->param3 = 0; - qp->vlan_control = 0; - qp->fvl_rx = 0; - qp->pri_path_fl = 0; - qp->vlan_index = 0; - qp->feup = 0; - qp->qpc_flags = be32_to_cpu(qpc->flags); - - err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); - if (err) - goto ex_abort; - - err = check_mtt_range(dev, slave, mtt_base, mtt_size, mtt); - if (err) - goto ex_put_mtt; - - err = get_res(dev, slave, rcqn, RES_CQ, &rcq); - if (err) - goto ex_put_mtt; - - if (scqn != rcqn) { - err = get_res(dev, slave, scqn, RES_CQ, &scq); - if (err) - goto ex_put_rcq; - } else - scq = rcq; - - if (use_srq) { - err = get_res(dev, slave, srqn, RES_SRQ, &srq); - if (err) - goto ex_put_scq; - } - - adjust_proxy_tun_qkey(dev, vhcr, qpc); - update_pkey_index(dev, slave, inbox); - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); - if (err) - goto ex_put_srq; - atomic_inc(&mtt->ref_count); - qp->mtt = mtt; - atomic_inc(&rcq->ref_count); - qp->rcq = rcq; - atomic_inc(&scq->ref_count); - qp->scq = scq; - - if (scqn != rcqn) - put_res(dev, slave, scqn, RES_CQ); - - if (use_srq) { - atomic_inc(&srq->ref_count); - put_res(dev, slave, srqn, RES_SRQ); - qp->srq = srq; - } - put_res(dev, slave, rcqn, RES_CQ); - put_res(dev, slave, mtt_base, RES_MTT); - res_end_move(dev, slave, RES_QP, qpn); - - return 0; - -ex_put_srq: - if (use_srq) - put_res(dev, slave, srqn, RES_SRQ); -ex_put_scq: - if (scqn != rcqn) - put_res(dev, slave, scqn, RES_CQ); -ex_put_rcq: - put_res(dev, slave, rcqn, RES_CQ); -ex_put_mtt: - put_res(dev, slave, mtt_base, RES_MTT); -ex_abort: - res_abort_move(dev, slave, RES_QP, qpn); - - return err; -} - -static int eq_get_mtt_addr(struct mlx4_eq_context *eqc) -{ - return be32_to_cpu(eqc->mtt_base_addr_l) & 0xfffffff8; -} - -static int eq_get_mtt_size(struct mlx4_eq_context *eqc) -{ - int log_eq_size = eqc->log_eq_size & 0x1f; - int page_shift = (eqc->log_page_size & 0x3f) + 12; - - if (log_eq_size + 5 < page_shift) - return 1; - - return 1 << (log_eq_size + 5 - page_shift); -} - -static int cq_get_mtt_addr(struct mlx4_cq_context *cqc) -{ - return be32_to_cpu(cqc->mtt_base_addr_l) & 0xfffffff8; -} - -static int cq_get_mtt_size(struct mlx4_cq_context *cqc) -{ - int log_cq_size = (be32_to_cpu(cqc->logsize_usrpage) >> 24) & 0x1f; - int page_shift = (cqc->log_page_size & 0x3f) + 12; - - if (log_cq_size + 5 < page_shift) - return 1; - - return 1 << (log_cq_size + 5 - page_shift); -} - -int mlx4_SW2HW_EQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - int eqn = vhcr->in_modifier; - int res_id = (slave << 8) | eqn; - struct mlx4_eq_context *eqc = inbox->buf; - int mtt_base = eq_get_mtt_addr(eqc) / dev->caps.mtt_entry_sz; - int mtt_size = eq_get_mtt_size(eqc); - struct res_eq *eq; - struct res_mtt *mtt; - - err = add_res_range(dev, slave, res_id, 1, RES_EQ, 0); - if (err) - return err; - err = eq_res_start_move_to(dev, slave, res_id, RES_EQ_HW, &eq); - if (err) - goto out_add; - - err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); - if (err) - goto out_move; - - err = check_mtt_range(dev, slave, mtt_base, mtt_size, mtt); - if (err) - goto out_put; - - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); - if (err) - goto out_put; - - atomic_inc(&mtt->ref_count); - eq->mtt = mtt; - put_res(dev, slave, mtt->com.res_id, RES_MTT); - res_end_move(dev, slave, RES_EQ, res_id); - return 0; - -out_put: - put_res(dev, slave, mtt->com.res_id, RES_MTT); -out_move: - res_abort_move(dev, slave, RES_EQ, res_id); -out_add: - rem_res_range(dev, slave, res_id, 1, RES_EQ, 0); - return err; -} - -static int get_containing_mtt(struct mlx4_dev *dev, int slave, int start, - int len, struct res_mtt **res) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct res_mtt *mtt; - int err = -EINVAL; - - spin_lock_irq(mlx4_tlock(dev)); - list_for_each_entry(mtt, &tracker->slave_list[slave].res_list[RES_MTT], - com.list) { - if (!check_mtt_range(dev, slave, start, len, mtt)) { - *res = mtt; - mtt->com.from_state = mtt->com.state; - mtt->com.state = RES_MTT_BUSY; - err = 0; - break; - } - } - spin_unlock_irq(mlx4_tlock(dev)); - - return err; -} - -static int verify_qp_parameters(struct mlx4_dev *dev, - struct mlx4_cmd_mailbox *inbox, - enum qp_transition transition, u8 slave) -{ - u32 qp_type; - struct mlx4_qp_context *qp_ctx; - enum mlx4_qp_optpar optpar; - int port; - int num_gids; - - qp_ctx = inbox->buf + 8; - qp_type = (be32_to_cpu(qp_ctx->flags) >> 16) & 0xff; - optpar = be32_to_cpu(*(__be32 *) inbox->buf); - - switch (qp_type) { - case MLX4_QP_ST_RC: - case MLX4_QP_ST_UC: - switch (transition) { - case QP_TRANS_INIT2RTR: - case QP_TRANS_RTR2RTS: - case QP_TRANS_RTS2RTS: - case QP_TRANS_SQD2SQD: - case QP_TRANS_SQD2RTS: - if (slave != mlx4_master_func_num(dev)) - if (optpar & MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH) { - port = (qp_ctx->pri_path.sched_queue >> 6 & 1) + 1; - if (dev->caps.port_mask[port] != MLX4_PORT_TYPE_IB) - num_gids = mlx4_get_slave_num_gids(dev, slave); - else - num_gids = 1; - if (qp_ctx->pri_path.mgid_index >= num_gids) - return -EINVAL; - } - if (optpar & MLX4_QP_OPTPAR_ALT_ADDR_PATH) { - port = (qp_ctx->alt_path.sched_queue >> 6 & 1) + 1; - if (dev->caps.port_mask[port] != MLX4_PORT_TYPE_IB) - num_gids = mlx4_get_slave_num_gids(dev, slave); - else - num_gids = 1; - if (qp_ctx->alt_path.mgid_index >= num_gids) - return -EINVAL; - } - break; - default: - break; - } - - break; - default: - break; - } - - return 0; -} - -int mlx4_WRITE_MTT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - struct mlx4_mtt mtt; - __be64 *page_list = inbox->buf; - u64 *pg_list = (u64 *)page_list; - int i; - struct res_mtt *rmtt = NULL; - int start = be64_to_cpu(page_list[0]); - int npages = vhcr->in_modifier; - int err; - - err = get_containing_mtt(dev, slave, start, npages, &rmtt); - if (err) - return err; - - /* Call the SW implementation of write_mtt: - * - Prepare a dummy mtt struct - * - Translate inbox contents to simple addresses in host endianness */ - mtt.offset = 0; /* TBD this is broken but I don't handle it since - we don't really use it */ - mtt.order = 0; - mtt.page_shift = 0; - for (i = 0; i < npages; ++i) - pg_list[i + 2] = (be64_to_cpu(page_list[i + 2]) & ~1ULL); - - err = __mlx4_write_mtt(dev, &mtt, be64_to_cpu(page_list[0]), npages, - ((u64 *)page_list + 2)); - - if (rmtt) - put_res(dev, slave, rmtt->com.res_id, RES_MTT); - - return err; -} - -int mlx4_HW2SW_EQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int eqn = vhcr->in_modifier; - int res_id = eqn | (slave << 8); - struct res_eq *eq; - int err; - - err = eq_res_start_move_to(dev, slave, res_id, RES_EQ_RESERVED, &eq); - if (err) - return err; - - err = get_res(dev, slave, eq->mtt->com.res_id, RES_MTT, NULL); - if (err) - goto ex_abort; - - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); - if (err) - goto ex_put; - - atomic_dec(&eq->mtt->ref_count); - put_res(dev, slave, eq->mtt->com.res_id, RES_MTT); - res_end_move(dev, slave, RES_EQ, res_id); - rem_res_range(dev, slave, res_id, 1, RES_EQ, 0); - - return 0; - -ex_put: - put_res(dev, slave, eq->mtt->com.res_id, RES_MTT); -ex_abort: - res_abort_move(dev, slave, RES_EQ, res_id); - - return err; -} - -int mlx4_GEN_EQE(struct mlx4_dev *dev, int slave, struct mlx4_eqe *eqe) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_slave_event_eq_info *event_eq; - struct mlx4_cmd_mailbox *mailbox; - u32 in_modifier = 0; - int err; - int res_id; - struct res_eq *req; - - if (!priv->mfunc.master.slave_state) - return -EINVAL; - - /* check for slave valid, slave not PF, and slave active */ - if (slave < 0 || slave >= dev->num_slaves || - slave == dev->caps.function || - !priv->mfunc.master.slave_state[slave].active) - return 0; - - event_eq = &priv->mfunc.master.slave_state[slave].event_eq[eqe->type]; - - /* Create the event only if the slave is registered */ - if (event_eq->eqn < 0) - return 0; - - mutex_lock(&priv->mfunc.master.gen_eqe_mutex[slave]); - res_id = (slave << 8) | event_eq->eqn; - err = get_res(dev, slave, res_id, RES_EQ, &req); - if (err) - goto unlock; - - if (req->com.from_state != RES_EQ_HW) { - err = -EINVAL; - goto put; - } - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) { - err = PTR_ERR(mailbox); - goto put; - } - - if (eqe->type == MLX4_EVENT_TYPE_CMD) { - ++event_eq->token; - eqe->event.cmd.token = cpu_to_be16(event_eq->token); - } - - memcpy(mailbox->buf, (u8 *) eqe, 28); - - in_modifier = (slave & 0xff) | ((event_eq->eqn & 0xff) << 16); - - err = mlx4_cmd(dev, mailbox->dma, in_modifier, 0, - MLX4_CMD_GEN_EQE, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_NATIVE); - - put_res(dev, slave, res_id, RES_EQ); - mutex_unlock(&priv->mfunc.master.gen_eqe_mutex[slave]); - mlx4_free_cmd_mailbox(dev, mailbox); - return err; - -put: - put_res(dev, slave, res_id, RES_EQ); - -unlock: - mutex_unlock(&priv->mfunc.master.gen_eqe_mutex[slave]); - return err; -} - -int mlx4_QUERY_EQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int eqn = vhcr->in_modifier; - int res_id = eqn | (slave << 8); - struct res_eq *eq; - int err; - - err = get_res(dev, slave, res_id, RES_EQ, &eq); - if (err) - return err; - - if (eq->com.from_state != RES_EQ_HW) { - err = -EINVAL; - goto ex_put; - } - - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); - -ex_put: - put_res(dev, slave, res_id, RES_EQ); - return err; -} - -int mlx4_SW2HW_CQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - int cqn = vhcr->in_modifier; - struct mlx4_cq_context *cqc = inbox->buf; - int mtt_base = cq_get_mtt_addr(cqc) / dev->caps.mtt_entry_sz; - struct res_cq *cq; - struct res_mtt *mtt; - - err = cq_res_start_move_to(dev, slave, cqn, RES_CQ_HW, &cq); - if (err) - return err; - err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); - if (err) - goto out_move; - err = check_mtt_range(dev, slave, mtt_base, cq_get_mtt_size(cqc), mtt); - if (err) - goto out_put; - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); - if (err) - goto out_put; - atomic_inc(&mtt->ref_count); - cq->mtt = mtt; - put_res(dev, slave, mtt->com.res_id, RES_MTT); - res_end_move(dev, slave, RES_CQ, cqn); - return 0; - -out_put: - put_res(dev, slave, mtt->com.res_id, RES_MTT); -out_move: - res_abort_move(dev, slave, RES_CQ, cqn); - return err; -} - -int mlx4_HW2SW_CQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - int cqn = vhcr->in_modifier; - struct res_cq *cq; - - err = cq_res_start_move_to(dev, slave, cqn, RES_CQ_ALLOCATED, &cq); - if (err) - return err; - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); - if (err) - goto out_move; - atomic_dec(&cq->mtt->ref_count); - res_end_move(dev, slave, RES_CQ, cqn); - return 0; - -out_move: - res_abort_move(dev, slave, RES_CQ, cqn); - return err; -} - -int mlx4_QUERY_CQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int cqn = vhcr->in_modifier; - struct res_cq *cq; - int err; - - err = get_res(dev, slave, cqn, RES_CQ, &cq); - if (err) - return err; - - if (cq->com.from_state != RES_CQ_HW) - goto ex_put; - - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); -ex_put: - put_res(dev, slave, cqn, RES_CQ); - - return err; -} - -static int handle_resize(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd, - struct res_cq *cq) -{ - int err; - struct res_mtt *orig_mtt; - struct res_mtt *mtt; - struct mlx4_cq_context *cqc = inbox->buf; - int mtt_base = cq_get_mtt_addr(cqc) / dev->caps.mtt_entry_sz; - - err = get_res(dev, slave, cq->mtt->com.res_id, RES_MTT, &orig_mtt); - if (err) - return err; - - if (orig_mtt != cq->mtt) { - err = -EINVAL; - goto ex_put; - } - - err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); - if (err) - goto ex_put; - - err = check_mtt_range(dev, slave, mtt_base, cq_get_mtt_size(cqc), mtt); - if (err) - goto ex_put1; - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); - if (err) - goto ex_put1; - atomic_dec(&orig_mtt->ref_count); - put_res(dev, slave, orig_mtt->com.res_id, RES_MTT); - atomic_inc(&mtt->ref_count); - cq->mtt = mtt; - put_res(dev, slave, mtt->com.res_id, RES_MTT); - return 0; - -ex_put1: - put_res(dev, slave, mtt->com.res_id, RES_MTT); -ex_put: - put_res(dev, slave, orig_mtt->com.res_id, RES_MTT); - - return err; - -} - -int mlx4_MODIFY_CQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int cqn = vhcr->in_modifier; - struct res_cq *cq; - int err; - - err = get_res(dev, slave, cqn, RES_CQ, &cq); - if (err) - return err; - - if (cq->com.from_state != RES_CQ_HW) - goto ex_put; - - if (vhcr->op_modifier == 0) { - err = handle_resize(dev, slave, vhcr, inbox, outbox, cmd, cq); - goto ex_put; - } - - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); -ex_put: - put_res(dev, slave, cqn, RES_CQ); - - return err; -} - -static int srq_get_mtt_size(struct mlx4_srq_context *srqc) -{ - int log_srq_size = (be32_to_cpu(srqc->state_logsize_srqn) >> 24) & 0xf; - int log_rq_stride = srqc->logstride & 7; - int page_shift = (srqc->log_page_size & 0x3f) + 12; - - if (log_srq_size + log_rq_stride + 4 < page_shift) - return 1; - - return 1 << (log_srq_size + log_rq_stride + 4 - page_shift); -} - -int mlx4_SW2HW_SRQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - int srqn = vhcr->in_modifier; - struct res_mtt *mtt; - struct res_srq *srq; - struct mlx4_srq_context *srqc = inbox->buf; - int mtt_base = srq_get_mtt_addr(srqc) / dev->caps.mtt_entry_sz; - - if (srqn != (be32_to_cpu(srqc->state_logsize_srqn) & 0xffffff)) - return -EINVAL; - - err = srq_res_start_move_to(dev, slave, srqn, RES_SRQ_HW, &srq); - if (err) - return err; - err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); - if (err) - goto ex_abort; - err = check_mtt_range(dev, slave, mtt_base, srq_get_mtt_size(srqc), - mtt); - if (err) - goto ex_put_mtt; - - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); - if (err) - goto ex_put_mtt; - - atomic_inc(&mtt->ref_count); - srq->mtt = mtt; - put_res(dev, slave, mtt->com.res_id, RES_MTT); - res_end_move(dev, slave, RES_SRQ, srqn); - return 0; - -ex_put_mtt: - put_res(dev, slave, mtt->com.res_id, RES_MTT); -ex_abort: - res_abort_move(dev, slave, RES_SRQ, srqn); - - return err; -} - -int mlx4_HW2SW_SRQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - int srqn = vhcr->in_modifier; - struct res_srq *srq; - - err = srq_res_start_move_to(dev, slave, srqn, RES_SRQ_ALLOCATED, &srq); - if (err) - return err; - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); - if (err) - goto ex_abort; - atomic_dec(&srq->mtt->ref_count); - if (srq->cq) - atomic_dec(&srq->cq->ref_count); - res_end_move(dev, slave, RES_SRQ, srqn); - - return 0; - -ex_abort: - res_abort_move(dev, slave, RES_SRQ, srqn); - - return err; -} - -int mlx4_QUERY_SRQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - int srqn = vhcr->in_modifier; - struct res_srq *srq; - - err = get_res(dev, slave, srqn, RES_SRQ, &srq); - if (err) - return err; - if (srq->com.from_state != RES_SRQ_HW) { - err = -EBUSY; - goto out; - } - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); -out: - put_res(dev, slave, srqn, RES_SRQ); - return err; -} - -int mlx4_ARM_SRQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - int srqn = vhcr->in_modifier; - struct res_srq *srq; - - err = get_res(dev, slave, srqn, RES_SRQ, &srq); - if (err) - return err; - - if (srq->com.from_state != RES_SRQ_HW) { - err = -EBUSY; - goto out; - } - - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); -out: - put_res(dev, slave, srqn, RES_SRQ); - return err; -} - -int mlx4_GEN_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - int qpn = vhcr->in_modifier & 0x7fffff; - struct res_qp *qp; - - err = get_res(dev, slave, qpn, RES_QP, &qp); - if (err) - return err; - if (qp->com.from_state != RES_QP_HW) { - err = -EBUSY; - goto out; - } - - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); -out: - put_res(dev, slave, qpn, RES_QP); - return err; -} - -int mlx4_INIT2INIT_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - struct mlx4_qp_context *context = inbox->buf + 8; - adjust_proxy_tun_qkey(dev, vhcr, context); - update_pkey_index(dev, slave, inbox); - return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); -} - -static int roce_verify_mac(struct mlx4_dev *dev, int slave, - struct mlx4_qp_context *qpc, - struct mlx4_cmd_mailbox *inbox) -{ - u64 mac; - int port; - u32 ts = (be32_to_cpu(qpc->flags) >> 16) & 0xff; - u8 sched = *(u8 *)(inbox->buf + 64); - u8 smac_ix; - - port = (sched >> 6 & 1) + 1; - if (mlx4_is_eth(dev, port) && (ts != MLX4_QP_ST_MLX)) { - smac_ix = qpc->pri_path.grh_mylmc & 0x7f; - if (mac_find_smac_ix_in_slave(dev, slave, port, smac_ix, &mac)) - return -ENOENT; - } - return 0; -} - -int mlx4_INIT2RTR_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - struct mlx4_qp_context *qpc = inbox->buf + 8; - int qpn = vhcr->in_modifier & 0x7fffff; - struct res_qp *qp; - u8 orig_sched_queue; - __be32 orig_param3 = qpc->param3; - u8 orig_vlan_control = qpc->pri_path.vlan_control; - u8 orig_fvl_rx = qpc->pri_path.fvl_rx; - u8 orig_pri_path_fl = qpc->pri_path.fl; - u8 orig_vlan_index = qpc->pri_path.vlan_index; - u8 orig_feup = qpc->pri_path.feup; - - err = verify_qp_parameters(dev, inbox, QP_TRANS_INIT2RTR, slave); - if (err) - return err; - - if (roce_verify_mac(dev, slave, qpc, inbox)) - return -EINVAL; - - update_pkey_index(dev, slave, inbox); - update_gid(dev, inbox, (u8)slave); - adjust_proxy_tun_qkey(dev, vhcr, qpc); - orig_sched_queue = qpc->pri_path.sched_queue; - - err = get_res(dev, slave, qpn, RES_QP, &qp); - if (err) - return err; - if (qp->com.from_state != RES_QP_HW) { - err = -EBUSY; - goto out; - } - - /* do not modify vport QP params for RSS QPs */ - if (!(qp->qpc_flags & (1 << MLX4_RSS_QPC_FLAG_OFFSET))) { - err = update_vport_qp_param(dev, inbox, slave, qpn); - if (err) - goto out; - } - - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); -out: - /* if no error, save sched queue value passed in by VF. This is - * essentially the QOS value provided by the VF. This will be useful - * if we allow dynamic changes from VST back to VGT - */ - if (!err) { - qp->sched_queue = orig_sched_queue; - qp->param3 = orig_param3; - qp->vlan_control = orig_vlan_control; - qp->fvl_rx = orig_fvl_rx; - qp->pri_path_fl = orig_pri_path_fl; - qp->vlan_index = orig_vlan_index; - qp->feup = orig_feup; - } - put_res(dev, slave, qpn, RES_QP); - return err; -} - -int mlx4_RTR2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - struct mlx4_qp_context *context = inbox->buf + 8; - - err = verify_qp_parameters(dev, inbox, QP_TRANS_RTR2RTS, slave); - if (err) - return err; - - update_pkey_index(dev, slave, inbox); - update_gid(dev, inbox, (u8)slave); - adjust_proxy_tun_qkey(dev, vhcr, context); - return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); -} - -int mlx4_RTS2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - struct mlx4_qp_context *context = inbox->buf + 8; - - err = verify_qp_parameters(dev, inbox, QP_TRANS_RTS2RTS, slave); - if (err) - return err; - - update_pkey_index(dev, slave, inbox); - update_gid(dev, inbox, (u8)slave); - adjust_proxy_tun_qkey(dev, vhcr, context); - return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); -} - - -int mlx4_SQERR2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - struct mlx4_qp_context *context = inbox->buf + 8; - adjust_proxy_tun_qkey(dev, vhcr, context); - return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); -} - -int mlx4_SQD2SQD_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - struct mlx4_qp_context *context = inbox->buf + 8; - - err = verify_qp_parameters(dev, inbox, QP_TRANS_SQD2SQD, slave); - if (err) - return err; - - adjust_proxy_tun_qkey(dev, vhcr, context); - update_gid(dev, inbox, (u8)slave); - update_pkey_index(dev, slave, inbox); - return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); -} - -int mlx4_SQD2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - struct mlx4_qp_context *context = inbox->buf + 8; - - err = verify_qp_parameters(dev, inbox, QP_TRANS_SQD2RTS, slave); - if (err) - return err; - - adjust_proxy_tun_qkey(dev, vhcr, context); - update_gid(dev, inbox, (u8)slave); - update_pkey_index(dev, slave, inbox); - return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); -} - -int mlx4_2RST_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - int qpn = vhcr->in_modifier & 0x7fffff; - struct res_qp *qp; - - err = qp_res_start_move_to(dev, slave, qpn, RES_QP_MAPPED, &qp, 0); - if (err) - return err; - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); - if (err) - goto ex_abort; - - atomic_dec(&qp->mtt->ref_count); - atomic_dec(&qp->rcq->ref_count); - atomic_dec(&qp->scq->ref_count); - if (qp->srq) - atomic_dec(&qp->srq->ref_count); - res_end_move(dev, slave, RES_QP, qpn); - return 0; - -ex_abort: - res_abort_move(dev, slave, RES_QP, qpn); - - return err; -} - -static struct res_gid *find_gid(struct mlx4_dev *dev, int slave, - struct res_qp *rqp, u8 *gid) -{ - struct res_gid *res; - - list_for_each_entry(res, &rqp->mcg_list, list) { - if (!memcmp(res->gid, gid, 16)) - return res; - } - return NULL; -} - -static int add_mcg_res(struct mlx4_dev *dev, int slave, struct res_qp *rqp, - u8 *gid, enum mlx4_protocol prot, - enum mlx4_steer_type steer, u64 reg_id) -{ - struct res_gid *res; - int err; - - res = kzalloc(sizeof *res, GFP_KERNEL); - if (!res) - return -ENOMEM; - - spin_lock_irq(&rqp->mcg_spl); - if (find_gid(dev, slave, rqp, gid)) { - kfree(res); - err = -EEXIST; - } else { - memcpy(res->gid, gid, 16); - res->prot = prot; - res->steer = steer; - res->reg_id = reg_id; - list_add_tail(&res->list, &rqp->mcg_list); - err = 0; - } - spin_unlock_irq(&rqp->mcg_spl); - - return err; -} - -static int rem_mcg_res(struct mlx4_dev *dev, int slave, struct res_qp *rqp, - u8 *gid, enum mlx4_protocol prot, - enum mlx4_steer_type steer, u64 *reg_id) -{ - struct res_gid *res; - int err; - - spin_lock_irq(&rqp->mcg_spl); - res = find_gid(dev, slave, rqp, gid); - if (!res || res->prot != prot || res->steer != steer) - err = -EINVAL; - else { - *reg_id = res->reg_id; - list_del(&res->list); - kfree(res); - err = 0; - } - spin_unlock_irq(&rqp->mcg_spl); - - return err; -} - -static int qp_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], - int block_loopback, enum mlx4_protocol prot, - enum mlx4_steer_type type, u64 *reg_id) -{ - switch (dev->caps.steering_mode) { - case MLX4_STEERING_MODE_DEVICE_MANAGED: - return mlx4_trans_to_dmfs_attach(dev, qp, gid, gid[5], - block_loopback, prot, - reg_id); - case MLX4_STEERING_MODE_B0: - return mlx4_qp_attach_common(dev, qp, gid, - block_loopback, prot, type); - default: - return -EINVAL; - } -} - -static int qp_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], - enum mlx4_protocol prot, enum mlx4_steer_type type, - u64 reg_id) -{ - switch (dev->caps.steering_mode) { - case MLX4_STEERING_MODE_DEVICE_MANAGED: - return mlx4_flow_detach(dev, reg_id); - case MLX4_STEERING_MODE_B0: - return mlx4_qp_detach_common(dev, qp, gid, prot, type); - default: - return -EINVAL; - } -} - -int mlx4_QP_ATTACH_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - struct mlx4_qp qp; /* dummy for calling attach/detach */ - u8 *gid = inbox->buf; - enum mlx4_protocol prot = (vhcr->in_modifier >> 28) & 0x7; - int err; - int qpn; - struct res_qp *rqp; - u64 reg_id = 0; - int attach = vhcr->op_modifier; - int block_loopback = vhcr->in_modifier >> 31; - u8 steer_type_mask = 2; - enum mlx4_steer_type type = (gid[7] & steer_type_mask) >> 1; - - qpn = vhcr->in_modifier & 0xffffff; - err = get_res(dev, slave, qpn, RES_QP, &rqp); - if (err) - return err; - - qp.qpn = qpn; - if (attach) { - err = qp_attach(dev, &qp, gid, block_loopback, prot, - type, ®_id); - if (err) { - pr_err("Fail to attach rule to qp 0x%x\n", qpn); - goto ex_put; - } - err = add_mcg_res(dev, slave, rqp, gid, prot, type, reg_id); - if (err) - goto ex_detach; - } else { - err = rem_mcg_res(dev, slave, rqp, gid, prot, type, ®_id); - if (err) - goto ex_put; - - err = qp_detach(dev, &qp, gid, prot, type, reg_id); - if (err) - pr_err("Fail to detach rule from qp 0x%x reg_id = 0x%llx\n", - qpn, (unsigned long long)reg_id); - } - put_res(dev, slave, qpn, RES_QP); - return err; - -ex_detach: - qp_detach(dev, &qp, gid, prot, type, reg_id); -ex_put: - put_res(dev, slave, qpn, RES_QP); - return err; -} - -/* - * MAC validation for Flow Steering rules. - * VF can attach rules only with a mac address which is assigned to it. - */ -static int validate_eth_header_mac(int slave, struct _rule_hw *eth_header, - struct list_head *rlist) -{ - struct mac_res *res, *tmp; - __be64 be_mac; - - /* make sure it isn't multicast or broadcast mac*/ - if (!is_multicast_ether_addr(eth_header->eth.dst_mac) && - !is_broadcast_ether_addr(eth_header->eth.dst_mac)) { - list_for_each_entry_safe(res, tmp, rlist, list) { - be_mac = cpu_to_be64(res->mac << 16); - if (!memcmp(&be_mac, eth_header->eth.dst_mac, ETH_ALEN)) - return 0; - } - pr_err("MAC %pM doesn't belong to VF %d, Steering rule rejected\n", - eth_header->eth.dst_mac, slave); - return -EINVAL; - } - return 0; -} - -/* - * In case of missing eth header, append eth header with a MAC address - * assigned to the VF. - */ -static int add_eth_header(struct mlx4_dev *dev, int slave, - struct mlx4_cmd_mailbox *inbox, - struct list_head *rlist, int header_id) -{ - struct mac_res *res, *tmp; - u8 port; - struct mlx4_net_trans_rule_hw_ctrl *ctrl; - struct mlx4_net_trans_rule_hw_eth *eth_header; - struct mlx4_net_trans_rule_hw_ipv4 *ip_header; - struct mlx4_net_trans_rule_hw_tcp_udp *l4_header; - __be64 be_mac = 0; - __be64 mac_msk = cpu_to_be64(MLX4_MAC_MASK << 16); - - ctrl = (struct mlx4_net_trans_rule_hw_ctrl *)inbox->buf; - port = ctrl->port; - eth_header = (struct mlx4_net_trans_rule_hw_eth *)(ctrl + 1); - - /* Clear a space in the inbox for eth header */ - switch (header_id) { - case MLX4_NET_TRANS_RULE_ID_IPV4: - ip_header = - (struct mlx4_net_trans_rule_hw_ipv4 *)(eth_header + 1); - memmove(ip_header, eth_header, - sizeof(*ip_header) + sizeof(*l4_header)); - break; - case MLX4_NET_TRANS_RULE_ID_TCP: - case MLX4_NET_TRANS_RULE_ID_UDP: - l4_header = (struct mlx4_net_trans_rule_hw_tcp_udp *) - (eth_header + 1); - memmove(l4_header, eth_header, sizeof(*l4_header)); - break; - default: - return -EINVAL; - } - list_for_each_entry_safe(res, tmp, rlist, list) { - if (port == res->port) { - be_mac = cpu_to_be64(res->mac << 16); - break; - } - } - if (!be_mac) { - pr_err("Failed adding eth header to FS rule, Can't find matching MAC for port %d .\n", - port); - return -EINVAL; - } - - memset(eth_header, 0, sizeof(*eth_header)); - eth_header->size = sizeof(*eth_header) >> 2; - eth_header->id = cpu_to_be16(__sw_id_hw[MLX4_NET_TRANS_RULE_ID_ETH]); - memcpy(eth_header->dst_mac, &be_mac, ETH_ALEN); - memcpy(eth_header->dst_mac_msk, &mac_msk, ETH_ALEN); - - return 0; - -} - -int mlx4_QP_FLOW_STEERING_ATTACH_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct list_head *rlist = &tracker->slave_list[slave].res_list[RES_MAC]; - int err; - int qpn; - struct res_qp *rqp; - struct mlx4_net_trans_rule_hw_ctrl *ctrl; - struct _rule_hw *rule_header; - int header_id; - - if (dev->caps.steering_mode != - MLX4_STEERING_MODE_DEVICE_MANAGED) - return -EOPNOTSUPP; - - ctrl = (struct mlx4_net_trans_rule_hw_ctrl *)inbox->buf; - qpn = be32_to_cpu(ctrl->qpn) & 0xffffff; - err = get_res(dev, slave, qpn, RES_QP, &rqp); - if (err) { - pr_err("Steering rule with qpn 0x%x rejected.\n", qpn); - return err; - } - rule_header = (struct _rule_hw *)(ctrl + 1); - header_id = map_hw_to_sw_id(be16_to_cpu(rule_header->id)); - - switch (header_id) { - case MLX4_NET_TRANS_RULE_ID_ETH: - if (validate_eth_header_mac(slave, rule_header, rlist)) { - err = -EINVAL; - goto err_put; - } - break; - case MLX4_NET_TRANS_RULE_ID_IB: - break; - case MLX4_NET_TRANS_RULE_ID_IPV4: - case MLX4_NET_TRANS_RULE_ID_TCP: - case MLX4_NET_TRANS_RULE_ID_UDP: - pr_warn("Can't attach FS rule without L2 headers, adding L2 header.\n"); - if (add_eth_header(dev, slave, inbox, rlist, header_id)) { - err = -EINVAL; - goto err_put; - } - vhcr->in_modifier += - sizeof(struct mlx4_net_trans_rule_hw_eth) >> 2; - break; - default: - pr_err("Corrupted mailbox.\n"); - err = -EINVAL; - goto err_put; - } - - err = mlx4_cmd_imm(dev, inbox->dma, &vhcr->out_param, - vhcr->in_modifier, 0, - MLX4_QP_FLOW_STEERING_ATTACH, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - if (err) - goto err_put; - - err = add_res_range(dev, slave, vhcr->out_param, 1, RES_FS_RULE, qpn); - if (err) { - mlx4_err(dev, "Fail to add flow steering resources.\n "); - /* detach rule*/ - mlx4_cmd(dev, vhcr->out_param, 0, 0, - MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - goto err_put; - } - atomic_inc(&rqp->ref_count); -err_put: - put_res(dev, slave, qpn, RES_QP); - return err; -} - -int mlx4_QP_FLOW_STEERING_DETACH_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - struct res_qp *rqp; - struct res_fs_rule *rrule; - - if (dev->caps.steering_mode != - MLX4_STEERING_MODE_DEVICE_MANAGED) - return -EOPNOTSUPP; - - err = get_res(dev, slave, vhcr->in_param, RES_FS_RULE, &rrule); - if (err) - return err; - /* Release the rule form busy state before removal */ - put_res(dev, slave, vhcr->in_param, RES_FS_RULE); - err = get_res(dev, slave, rrule->qpn, RES_QP, &rqp); - if (err) - return err; - - err = mlx4_cmd(dev, vhcr->in_param, 0, 0, - MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - if (!err) { - err = rem_res_range(dev, slave, vhcr->in_param, 1, RES_FS_RULE, - 0); - atomic_dec(&rqp->ref_count); - - if (err) { - mlx4_err(dev, "Fail to remove flow steering resources.\n "); - goto out; - } - } - -out: - put_res(dev, slave, rrule->qpn, RES_QP); - return err; -} - -enum { - BUSY_MAX_RETRIES = 10 -}; - -int mlx4_QUERY_IF_STAT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err; - - err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); - - return err; -} - -static void detach_qp(struct mlx4_dev *dev, int slave, struct res_qp *rqp) -{ - struct res_gid *rgid; - struct res_gid *tmp; - struct mlx4_qp qp; /* dummy for calling attach/detach */ - - list_for_each_entry_safe(rgid, tmp, &rqp->mcg_list, list) { - switch (dev->caps.steering_mode) { - case MLX4_STEERING_MODE_DEVICE_MANAGED: - mlx4_flow_detach(dev, rgid->reg_id); - break; - case MLX4_STEERING_MODE_B0: - qp.qpn = rqp->local_qpn; - (void) mlx4_qp_detach_common(dev, &qp, rgid->gid, - rgid->prot, rgid->steer); - break; - } - list_del(&rgid->list); - kfree(rgid); - } -} - -static int _move_all_busy(struct mlx4_dev *dev, int slave, - enum mlx4_resource type, int print) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = - &priv->mfunc.master.res_tracker; - struct list_head *rlist = &tracker->slave_list[slave].res_list[type]; - struct res_common *r; - struct res_common *tmp; - int busy; - - busy = 0; - spin_lock_irq(mlx4_tlock(dev)); - list_for_each_entry_safe(r, tmp, rlist, list) { - if (r->owner == slave) { - if (!r->removing) { - if (r->state == RES_ANY_BUSY) { - if (print) - mlx4_dbg(dev, - "%s id 0x%llx is busy\n", - ResourceType(type), - (unsigned long long)r->res_id); - ++busy; - } else { - r->from_state = r->state; - r->state = RES_ANY_BUSY; - r->removing = 1; - } - } - } - } - spin_unlock_irq(mlx4_tlock(dev)); - - return busy; -} - -static int move_all_busy(struct mlx4_dev *dev, int slave, - enum mlx4_resource type) -{ - unsigned long begin; - int busy; - - begin = jiffies; - do { - busy = _move_all_busy(dev, slave, type, 0); - if (time_after(jiffies, begin + 5 * HZ)) - break; - if (busy) - cond_resched(); - } while (busy); - - if (busy) - busy = _move_all_busy(dev, slave, type, 1); - - return busy; -} -static void rem_slave_qps(struct mlx4_dev *dev, int slave) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct list_head *qp_list = - &tracker->slave_list[slave].res_list[RES_QP]; - struct res_qp *qp; - struct res_qp *tmp; - int state; - u64 in_param; - int qpn; - int err; - - err = move_all_busy(dev, slave, RES_QP); - if (err) - mlx4_warn(dev, "rem_slave_qps: Could not move all qps to busy" - "for slave %d\n", slave); - - spin_lock_irq(mlx4_tlock(dev)); - list_for_each_entry_safe(qp, tmp, qp_list, com.list) { - spin_unlock_irq(mlx4_tlock(dev)); - if (qp->com.owner == slave) { - qpn = qp->com.res_id; - detach_qp(dev, slave, qp); - state = qp->com.from_state; - while (state != 0) { - switch (state) { - case RES_QP_RESERVED: - spin_lock_irq(mlx4_tlock(dev)); - rb_erase(&qp->com.node, - &tracker->res_tree[RES_QP]); - list_del(&qp->com.list); - spin_unlock_irq(mlx4_tlock(dev)); - if (!valid_reserved(dev, slave, qpn)) { - __mlx4_qp_release_range(dev, qpn, 1); - mlx4_release_resource(dev, slave, - RES_QP, 1, 0); - } - kfree(qp); - state = 0; - break; - case RES_QP_MAPPED: - if (!valid_reserved(dev, slave, qpn)) - __mlx4_qp_free_icm(dev, qpn); - state = RES_QP_RESERVED; - break; - case RES_QP_HW: - in_param = slave; - err = mlx4_cmd(dev, in_param, - qp->local_qpn, 2, - MLX4_CMD_2RST_QP, - MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - if (err) - mlx4_dbg(dev, "rem_slave_qps: failed" - " to move slave %d qpn %d to" - " reset\n", slave, - qp->local_qpn); - atomic_dec(&qp->rcq->ref_count); - atomic_dec(&qp->scq->ref_count); - atomic_dec(&qp->mtt->ref_count); - if (qp->srq) - atomic_dec(&qp->srq->ref_count); - state = RES_QP_MAPPED; - break; - default: - state = 0; - } - } - } - spin_lock_irq(mlx4_tlock(dev)); - } - spin_unlock_irq(mlx4_tlock(dev)); -} - -static void rem_slave_srqs(struct mlx4_dev *dev, int slave) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct list_head *srq_list = - &tracker->slave_list[slave].res_list[RES_SRQ]; - struct res_srq *srq; - struct res_srq *tmp; - int state; - u64 in_param; - LIST_HEAD(tlist); - int srqn; - int err; - - err = move_all_busy(dev, slave, RES_SRQ); - if (err) - mlx4_warn(dev, "rem_slave_srqs: Could not move all srqs to " - "busy for slave %d\n", slave); - - spin_lock_irq(mlx4_tlock(dev)); - list_for_each_entry_safe(srq, tmp, srq_list, com.list) { - spin_unlock_irq(mlx4_tlock(dev)); - if (srq->com.owner == slave) { - srqn = srq->com.res_id; - state = srq->com.from_state; - while (state != 0) { - switch (state) { - case RES_SRQ_ALLOCATED: - __mlx4_srq_free_icm(dev, srqn); - spin_lock_irq(mlx4_tlock(dev)); - rb_erase(&srq->com.node, - &tracker->res_tree[RES_SRQ]); - list_del(&srq->com.list); - spin_unlock_irq(mlx4_tlock(dev)); - mlx4_release_resource(dev, slave, - RES_SRQ, 1, 0); - kfree(srq); - state = 0; - break; - - case RES_SRQ_HW: - in_param = slave; - err = mlx4_cmd(dev, in_param, srqn, 1, - MLX4_CMD_HW2SW_SRQ, - MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - if (err) - mlx4_dbg(dev, "rem_slave_srqs: failed" - " to move slave %d srq %d to" - " SW ownership\n", - slave, srqn); - - atomic_dec(&srq->mtt->ref_count); - if (srq->cq) - atomic_dec(&srq->cq->ref_count); - state = RES_SRQ_ALLOCATED; - break; - - default: - state = 0; - } - } - } - spin_lock_irq(mlx4_tlock(dev)); - } - spin_unlock_irq(mlx4_tlock(dev)); -} - -static void rem_slave_cqs(struct mlx4_dev *dev, int slave) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct list_head *cq_list = - &tracker->slave_list[slave].res_list[RES_CQ]; - struct res_cq *cq; - struct res_cq *tmp; - int state; - u64 in_param; - LIST_HEAD(tlist); - int cqn; - int err; - - err = move_all_busy(dev, slave, RES_CQ); - if (err) - mlx4_warn(dev, "rem_slave_cqs: Could not move all cqs to " - "busy for slave %d\n", slave); - - spin_lock_irq(mlx4_tlock(dev)); - list_for_each_entry_safe(cq, tmp, cq_list, com.list) { - spin_unlock_irq(mlx4_tlock(dev)); - if (cq->com.owner == slave && !atomic_read(&cq->ref_count)) { - cqn = cq->com.res_id; - state = cq->com.from_state; - while (state != 0) { - switch (state) { - case RES_CQ_ALLOCATED: - __mlx4_cq_free_icm(dev, cqn); - spin_lock_irq(mlx4_tlock(dev)); - rb_erase(&cq->com.node, - &tracker->res_tree[RES_CQ]); - list_del(&cq->com.list); - spin_unlock_irq(mlx4_tlock(dev)); - mlx4_release_resource(dev, slave, - RES_CQ, 1, 0); - kfree(cq); - state = 0; - break; - - case RES_CQ_HW: - in_param = slave; - err = mlx4_cmd(dev, in_param, cqn, 1, - MLX4_CMD_HW2SW_CQ, - MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - if (err) - mlx4_dbg(dev, "rem_slave_cqs: failed" - " to move slave %d cq %d to" - " SW ownership\n", - slave, cqn); - atomic_dec(&cq->mtt->ref_count); - state = RES_CQ_ALLOCATED; - break; - - default: - state = 0; - } - } - } - spin_lock_irq(mlx4_tlock(dev)); - } - spin_unlock_irq(mlx4_tlock(dev)); -} - -static void rem_slave_mrs(struct mlx4_dev *dev, int slave) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct list_head *mpt_list = - &tracker->slave_list[slave].res_list[RES_MPT]; - struct res_mpt *mpt; - struct res_mpt *tmp; - int state; - u64 in_param; - LIST_HEAD(tlist); - int mptn; - int err; - - err = move_all_busy(dev, slave, RES_MPT); - if (err) - mlx4_warn(dev, "rem_slave_mrs: Could not move all mpts to " - "busy for slave %d\n", slave); - - spin_lock_irq(mlx4_tlock(dev)); - list_for_each_entry_safe(mpt, tmp, mpt_list, com.list) { - spin_unlock_irq(mlx4_tlock(dev)); - if (mpt->com.owner == slave) { - mptn = mpt->com.res_id; - state = mpt->com.from_state; - while (state != 0) { - switch (state) { - case RES_MPT_RESERVED: - __mlx4_mpt_release(dev, mpt->key); - spin_lock_irq(mlx4_tlock(dev)); - rb_erase(&mpt->com.node, - &tracker->res_tree[RES_MPT]); - list_del(&mpt->com.list); - spin_unlock_irq(mlx4_tlock(dev)); - mlx4_release_resource(dev, slave, - RES_MPT, 1, 0); - kfree(mpt); - state = 0; - break; - - case RES_MPT_MAPPED: - __mlx4_mpt_free_icm(dev, mpt->key); - state = RES_MPT_RESERVED; - break; - - case RES_MPT_HW: - in_param = slave; - err = mlx4_cmd(dev, in_param, mptn, 0, - MLX4_CMD_HW2SW_MPT, - MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - if (err) - mlx4_dbg(dev, "rem_slave_mrs: failed" - " to move slave %d mpt %d to" - " SW ownership\n", - slave, mptn); - if (mpt->mtt) - atomic_dec(&mpt->mtt->ref_count); - state = RES_MPT_MAPPED; - break; - default: - state = 0; - } - } - } - spin_lock_irq(mlx4_tlock(dev)); - } - spin_unlock_irq(mlx4_tlock(dev)); -} - -static void rem_slave_mtts(struct mlx4_dev *dev, int slave) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = - &priv->mfunc.master.res_tracker; - struct list_head *mtt_list = - &tracker->slave_list[slave].res_list[RES_MTT]; - struct res_mtt *mtt; - struct res_mtt *tmp; - int state; - LIST_HEAD(tlist); - int base; - int err; - - err = move_all_busy(dev, slave, RES_MTT); - if (err) - mlx4_warn(dev, "rem_slave_mtts: Could not move all mtts to " - "busy for slave %d\n", slave); - - spin_lock_irq(mlx4_tlock(dev)); - list_for_each_entry_safe(mtt, tmp, mtt_list, com.list) { - spin_unlock_irq(mlx4_tlock(dev)); - if (mtt->com.owner == slave) { - base = mtt->com.res_id; - state = mtt->com.from_state; - while (state != 0) { - switch (state) { - case RES_MTT_ALLOCATED: - __mlx4_free_mtt_range(dev, base, - mtt->order); - spin_lock_irq(mlx4_tlock(dev)); - rb_erase(&mtt->com.node, - &tracker->res_tree[RES_MTT]); - list_del(&mtt->com.list); - spin_unlock_irq(mlx4_tlock(dev)); - mlx4_release_resource(dev, slave, RES_MTT, - 1 << mtt->order, 0); - kfree(mtt); - state = 0; - break; - - default: - state = 0; - } - } - } - spin_lock_irq(mlx4_tlock(dev)); - } - spin_unlock_irq(mlx4_tlock(dev)); -} - -static void rem_slave_fs_rule(struct mlx4_dev *dev, int slave) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = - &priv->mfunc.master.res_tracker; - struct list_head *fs_rule_list = - &tracker->slave_list[slave].res_list[RES_FS_RULE]; - struct res_fs_rule *fs_rule; - struct res_fs_rule *tmp; - int state; - u64 base; - int err; - - err = move_all_busy(dev, slave, RES_FS_RULE); - if (err) - mlx4_warn(dev, "rem_slave_fs_rule: Could not move all mtts to busy for slave %d\n", - slave); - - spin_lock_irq(mlx4_tlock(dev)); - list_for_each_entry_safe(fs_rule, tmp, fs_rule_list, com.list) { - spin_unlock_irq(mlx4_tlock(dev)); - if (fs_rule->com.owner == slave) { - base = fs_rule->com.res_id; - state = fs_rule->com.from_state; - while (state != 0) { - switch (state) { - case RES_FS_RULE_ALLOCATED: - /* detach rule */ - err = mlx4_cmd(dev, base, 0, 0, - MLX4_QP_FLOW_STEERING_DETACH, - MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - - spin_lock_irq(mlx4_tlock(dev)); - rb_erase(&fs_rule->com.node, - &tracker->res_tree[RES_FS_RULE]); - list_del(&fs_rule->com.list); - spin_unlock_irq(mlx4_tlock(dev)); - kfree(fs_rule); - state = 0; - break; - - default: - state = 0; - } - } - } - spin_lock_irq(mlx4_tlock(dev)); - } - spin_unlock_irq(mlx4_tlock(dev)); -} - -static void rem_slave_eqs(struct mlx4_dev *dev, int slave) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct list_head *eq_list = - &tracker->slave_list[slave].res_list[RES_EQ]; - struct res_eq *eq; - struct res_eq *tmp; - int err; - int state; - LIST_HEAD(tlist); - int eqn; - struct mlx4_cmd_mailbox *mailbox; - - err = move_all_busy(dev, slave, RES_EQ); - if (err) - mlx4_warn(dev, "rem_slave_eqs: Could not move all eqs to " - "busy for slave %d\n", slave); - - spin_lock_irq(mlx4_tlock(dev)); - list_for_each_entry_safe(eq, tmp, eq_list, com.list) { - spin_unlock_irq(mlx4_tlock(dev)); - if (eq->com.owner == slave) { - eqn = eq->com.res_id; - state = eq->com.from_state; - while (state != 0) { - switch (state) { - case RES_EQ_RESERVED: - spin_lock_irq(mlx4_tlock(dev)); - rb_erase(&eq->com.node, - &tracker->res_tree[RES_EQ]); - list_del(&eq->com.list); - spin_unlock_irq(mlx4_tlock(dev)); - kfree(eq); - state = 0; - break; - - case RES_EQ_HW: - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) { - cond_resched(); - continue; - } - err = mlx4_cmd_box(dev, slave, 0, - eqn & 0xff, 0, - MLX4_CMD_HW2SW_EQ, - MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - if (err) - mlx4_dbg(dev, "rem_slave_eqs: failed" - " to move slave %d eqs %d to" - " SW ownership\n", slave, eqn); - mlx4_free_cmd_mailbox(dev, mailbox); - atomic_dec(&eq->mtt->ref_count); - state = RES_EQ_RESERVED; - break; - - default: - state = 0; - } - } - } - spin_lock_irq(mlx4_tlock(dev)); - } - spin_unlock_irq(mlx4_tlock(dev)); -} - -static void rem_slave_counters(struct mlx4_dev *dev, int slave) -{ - __mlx4_slave_counters_free(dev, slave); -} - -static void rem_slave_xrcdns(struct mlx4_dev *dev, int slave) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; - struct list_head *xrcdn_list = - &tracker->slave_list[slave].res_list[RES_XRCD]; - struct res_xrcdn *xrcd; - struct res_xrcdn *tmp; - int err; - int xrcdn; - - err = move_all_busy(dev, slave, RES_XRCD); - if (err) - mlx4_warn(dev, "rem_slave_xrcdns: Could not move all xrcdns to " - "busy for slave %d\n", slave); - - spin_lock_irq(mlx4_tlock(dev)); - list_for_each_entry_safe(xrcd, tmp, xrcdn_list, com.list) { - if (xrcd->com.owner == slave) { - xrcdn = xrcd->com.res_id; - rb_erase(&xrcd->com.node, &tracker->res_tree[RES_XRCD]); - list_del(&xrcd->com.list); - kfree(xrcd); - __mlx4_xrcd_free(dev, xrcdn); - } - } - spin_unlock_irq(mlx4_tlock(dev)); -} - -void mlx4_delete_all_resources_for_slave(struct mlx4_dev *dev, int slave) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - mutex_lock(&priv->mfunc.master.res_tracker.slave_list[slave].mutex); - rem_slave_macs(dev, slave); - rem_slave_vlans(dev, slave); - rem_slave_fs_rule(dev, slave); - rem_slave_qps(dev, slave); - rem_slave_srqs(dev, slave); - rem_slave_cqs(dev, slave); - rem_slave_mrs(dev, slave); - rem_slave_eqs(dev, slave); - rem_slave_mtts(dev, slave); - rem_slave_counters(dev, slave); - rem_slave_xrcdns(dev, slave); - mutex_unlock(&priv->mfunc.master.res_tracker.slave_list[slave].mutex); -} - -void mlx4_vf_immed_vlan_work_handler(struct work_struct *_work) -{ - struct mlx4_vf_immed_vlan_work *work = - container_of(_work, struct mlx4_vf_immed_vlan_work, work); - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_update_qp_context *upd_context; - struct mlx4_dev *dev = &work->priv->dev; - struct mlx4_resource_tracker *tracker = - &work->priv->mfunc.master.res_tracker; - struct list_head *qp_list = - &tracker->slave_list[work->slave].res_list[RES_QP]; - struct res_qp *qp; - struct res_qp *tmp; - u64 qp_path_mask_vlan_ctrl = - ((1ULL << MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_UNTAGGED) | - (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_1P) | - (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_TAGGED) | - (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_UNTAGGED) | - (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_1P) | - (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_TAGGED)); - - u64 qp_path_mask = ((1ULL << MLX4_UPD_QP_PATH_MASK_VLAN_INDEX) | - (1ULL << MLX4_UPD_QP_PATH_MASK_FVL) | - (1ULL << MLX4_UPD_QP_PATH_MASK_CV) | - (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_HIDE_CQE_VLAN) | - (1ULL << MLX4_UPD_QP_PATH_MASK_FEUP) | - (1ULL << MLX4_UPD_QP_PATH_MASK_FVL_RX) | - (1ULL << MLX4_UPD_QP_PATH_MASK_SCHED_QUEUE)); - - int err; - int port, errors = 0; - u8 vlan_control; - - if (mlx4_is_slave(dev)) { - mlx4_warn(dev, "Trying to update-qp in slave %d\n", - work->slave); - goto out; - } - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - goto out; - - if (!work->vlan_id) - vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED | - MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED; - else - vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED | - MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED | - MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED; - - upd_context = mailbox->buf; - upd_context->qp_mask = cpu_to_be64(MLX4_UPD_QP_MASK_VSD); - - spin_lock_irq(mlx4_tlock(dev)); - list_for_each_entry_safe(qp, tmp, qp_list, com.list) { - spin_unlock_irq(mlx4_tlock(dev)); - if (qp->com.owner == work->slave) { - if (qp->com.from_state != RES_QP_HW || - !qp->sched_queue || /* no INIT2RTR trans yet */ - mlx4_is_qp_reserved(dev, qp->local_qpn) || - qp->qpc_flags & (1 << MLX4_RSS_QPC_FLAG_OFFSET)) { - spin_lock_irq(mlx4_tlock(dev)); - continue; - } - port = (qp->sched_queue >> 6 & 1) + 1; - if (port != work->port) { - spin_lock_irq(mlx4_tlock(dev)); - continue; - } - if (MLX4_QP_ST_RC == ((qp->qpc_flags >> 16) & 0xff)) - upd_context->primary_addr_path_mask = cpu_to_be64(qp_path_mask); - else - upd_context->primary_addr_path_mask = - cpu_to_be64(qp_path_mask | qp_path_mask_vlan_ctrl); - if (work->vlan_id == MLX4_VGT) { - upd_context->qp_context.param3 = qp->param3; - upd_context->qp_context.pri_path.vlan_control = qp->vlan_control; - upd_context->qp_context.pri_path.fvl_rx = qp->fvl_rx; - upd_context->qp_context.pri_path.vlan_index = qp->vlan_index; - upd_context->qp_context.pri_path.fl = qp->pri_path_fl; - upd_context->qp_context.pri_path.feup = qp->feup; - upd_context->qp_context.pri_path.sched_queue = - qp->sched_queue; - } else { - upd_context->qp_context.param3 = qp->param3 & ~cpu_to_be32(MLX4_STRIP_VLAN); - upd_context->qp_context.pri_path.vlan_control = vlan_control; - upd_context->qp_context.pri_path.vlan_index = work->vlan_ix; - upd_context->qp_context.pri_path.fvl_rx = - qp->fvl_rx | MLX4_FVL_RX_FORCE_ETH_VLAN; - upd_context->qp_context.pri_path.fl = - qp->pri_path_fl | MLX4_FL_CV | MLX4_FL_ETH_HIDE_CQE_VLAN; - upd_context->qp_context.pri_path.feup = - qp->feup | MLX4_FEUP_FORCE_ETH_UP | MLX4_FVL_FORCE_ETH_VLAN; - upd_context->qp_context.pri_path.sched_queue = - qp->sched_queue & 0xC7; - upd_context->qp_context.pri_path.sched_queue |= - ((work->qos & 0x7) << 3); - } - - err = mlx4_cmd(dev, mailbox->dma, - qp->local_qpn & 0xffffff, - 0, MLX4_CMD_UPDATE_QP, - MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); - if (err) { - mlx4_info(dev, "UPDATE_QP failed for slave %d, " - "port %d, qpn %d (%d)\n", - work->slave, port, qp->local_qpn, - err); - errors++; - } - } - spin_lock_irq(mlx4_tlock(dev)); - } - spin_unlock_irq(mlx4_tlock(dev)); - mlx4_free_cmd_mailbox(dev, mailbox); - - if (errors) - mlx4_err(dev, "%d UPDATE_QP failures for slave %d, port %d\n", - errors, work->slave, work->port); - - /* unregister previous vlan_id if needed and we had no errors - * while updating the QPs - */ - if (work->flags & MLX4_VF_IMMED_VLAN_FLAG_VLAN && !errors && - NO_INDX != work->orig_vlan_ix) - __mlx4_unregister_vlan(&work->priv->dev, work->port, - work->orig_vlan_id); -out: - kfree(work); - return; -} - Property changes on: stable/11/sys/ofed/drivers/net/mlx4/resource_tracker.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/en_main.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/en_main.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/en_main.c (nonexistent) @@ -1,343 +0,0 @@ -/* - * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#define LINUXKPI_PARAM_PREFIX mlx4_ - -#include -#include -#include -#include - -#include -#include -#include - -#include "mlx4_en.h" - -/* Mellanox ConnectX HCA Ethernet driver */ - -#define MLX4_EN_PARM_INT(X, def_val, desc) \ - static unsigned int X = def_val;\ - module_param(X , uint, 0444); \ - MODULE_PARM_DESC(X, desc); - - -/* - * Device scope module parameters - */ - -/* Enable RSS UDP traffic */ -MLX4_EN_PARM_INT(udp_rss, 1, - "Enable RSS for incoming UDP traffic"); - -/* Priority pausing */ -MLX4_EN_PARM_INT(pfctx, 0, "Priority based Flow Control policy on TX[7:0]." - " Per priority bit mask"); -MLX4_EN_PARM_INT(pfcrx, 0, "Priority based Flow Control policy on RX[7:0]." - " Per priority bit mask"); - -#define MAX_PFC_TX 0xff -#define MAX_PFC_RX 0xff - - -static int mlx4_en_get_profile(struct mlx4_en_dev *mdev) -{ - struct mlx4_en_profile *params = &mdev->profile; - int i; - - params->udp_rss = udp_rss; - params->num_tx_rings_p_up = min_t(int, mp_ncpus, - MLX4_EN_MAX_TX_RING_P_UP); - if (params->udp_rss && !(mdev->dev->caps.flags - & MLX4_DEV_CAP_FLAG_UDP_RSS)) { - mlx4_warn(mdev, "UDP RSS is not supported on this device.\n"); - params->udp_rss = 0; - } - for (i = 1; i <= MLX4_MAX_PORTS; i++) { - params->prof[i].rx_pause = 1; - params->prof[i].rx_ppp = pfcrx; - params->prof[i].tx_pause = 1; - params->prof[i].tx_ppp = pfctx; - params->prof[i].tx_ring_size = MLX4_EN_DEF_TX_RING_SIZE; - params->prof[i].rx_ring_size = MLX4_EN_DEF_RX_RING_SIZE; - params->prof[i].tx_ring_num = params->num_tx_rings_p_up * - MLX4_EN_NUM_UP; - params->prof[i].rss_rings = 0; - } - - return 0; -} - -static void *mlx4_en_get_netdev(struct mlx4_dev *dev, void *ctx, u8 port) -{ - struct mlx4_en_dev *endev = ctx; - - return endev->pndev[port]; -} - -static void mlx4_en_event(struct mlx4_dev *dev, void *endev_ptr, - enum mlx4_dev_event event, unsigned long port) -{ - struct mlx4_en_dev *mdev = (struct mlx4_en_dev *) endev_ptr; - struct mlx4_en_priv *priv; - - switch (event) { - case MLX4_DEV_EVENT_PORT_UP: - case MLX4_DEV_EVENT_PORT_DOWN: - if (!mdev->pndev[port]) - return; - priv = netdev_priv(mdev->pndev[port]); - /* To prevent races, we poll the link state in a separate - task rather than changing it here */ - priv->link_state = event; - queue_work(mdev->workqueue, &priv->linkstate_task); - break; - - case MLX4_DEV_EVENT_CATASTROPHIC_ERROR: - mlx4_err(mdev, "Internal error detected, restarting device\n"); - break; - - case MLX4_DEV_EVENT_SLAVE_INIT: - case MLX4_DEV_EVENT_SLAVE_SHUTDOWN: - break; - default: - if (port < 1 || port > dev->caps.num_ports || - !mdev->pndev[port]) - return; - mlx4_warn(mdev, "Unhandled event %d for port %d\n", event, - (int) port); - } -} - -static void mlx4_en_remove(struct mlx4_dev *dev, void *endev_ptr) -{ - struct mlx4_en_dev *mdev = endev_ptr; - int i, ret; - - mutex_lock(&mdev->state_lock); - mdev->device_up = false; - mutex_unlock(&mdev->state_lock); - - mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) - if (mdev->pndev[i]) - mlx4_en_destroy_netdev(mdev->pndev[i]); - - flush_workqueue(mdev->workqueue); - destroy_workqueue(mdev->workqueue); - ret = mlx4_mr_free(dev, &mdev->mr); - if (ret) - mlx4_err(mdev, "Error deregistering MR. The system may have become unstable."); - iounmap(mdev->uar_map); - mlx4_uar_free(dev, &mdev->priv_uar); - mlx4_pd_free(dev, mdev->priv_pdn); - kfree(mdev); -} - -static void *mlx4_en_add(struct mlx4_dev *dev) -{ - struct mlx4_en_dev *mdev; - int i; - int err; - - mdev = kzalloc(sizeof *mdev, GFP_KERNEL); - if (!mdev) { - dev_err(&dev->pdev->dev, "Device struct alloc failed, " - "aborting.\n"); - err = -ENOMEM; - goto err_free_res; - } - - if (mlx4_pd_alloc(dev, &mdev->priv_pdn)) - goto err_free_dev; - - if (mlx4_uar_alloc(dev, &mdev->priv_uar)) - goto err_pd; - - mdev->uar_map = ioremap((phys_addr_t) mdev->priv_uar.pfn << PAGE_SHIFT, - PAGE_SIZE); - if (!mdev->uar_map) - goto err_uar; - spin_lock_init(&mdev->uar_lock); - - mdev->dev = dev; - mdev->dma_device = &(dev->pdev->dev); - mdev->pdev = dev->pdev; - mdev->device_up = false; - - mdev->LSO_support = !!(dev->caps.flags & (1 << 15)); - if (!mdev->LSO_support) - mlx4_warn(mdev, "LSO not supported, please upgrade to later " - "FW version to enable LSO\n"); - - if (mlx4_mr_alloc(mdev->dev, mdev->priv_pdn, 0, ~0ull, - MLX4_PERM_LOCAL_WRITE | MLX4_PERM_LOCAL_READ, - 0, 0, &mdev->mr)) { - mlx4_err(mdev, "Failed allocating memory region\n"); - goto err_map; - } - if (mlx4_mr_enable(mdev->dev, &mdev->mr)) { - mlx4_err(mdev, "Failed enabling memory region\n"); - goto err_mr; - } - - /* Build device profile according to supplied module parameters */ - err = mlx4_en_get_profile(mdev); - if (err) { - mlx4_err(mdev, "Bad module parameters, aborting.\n"); - goto err_mr; - } - - /* Configure which ports to start according to module parameters */ - mdev->port_cnt = 0; - mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) - mdev->port_cnt++; - - - mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { - if (!dev->caps.comp_pool) { - mdev->profile.prof[i].rx_ring_num = - rounddown_pow_of_two(max_t(int, MIN_RX_RINGS, - min_t(int, - dev->caps.num_comp_vectors, - DEF_RX_RINGS))); - } else { - mdev->profile.prof[i].rx_ring_num = rounddown_pow_of_two( - min_t(int, dev->caps.comp_pool / - dev->caps.num_ports, MAX_MSIX_P_PORT)); - } - } - - /* Create our own workqueue for reset/multicast tasks - * Note: we cannot use the shared workqueue because of deadlocks caused - * by the rtnl lock */ - mdev->workqueue = create_singlethread_workqueue("mlx4_en"); - if (!mdev->workqueue) { - err = -ENOMEM; - goto err_mr; - } - - /* At this stage all non-port specific tasks are complete: - * mark the card state as up */ - mutex_init(&mdev->state_lock); - mdev->device_up = true; - - /* Setup ports */ - - /* Create a netdev for each port */ - mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { - mlx4_info(mdev, "Activating port:%d\n", i); - if (mlx4_en_init_netdev(mdev, i, &mdev->profile.prof[i])) - mdev->pndev[i] = NULL; - } - - return mdev; - -err_mr: - err = mlx4_mr_free(dev, &mdev->mr); - if (err) - mlx4_err(mdev, "Error deregistering MR. The system may have become unstable."); -err_map: - if (mdev->uar_map) - iounmap(mdev->uar_map); -err_uar: - mlx4_uar_free(dev, &mdev->priv_uar); -err_pd: - mlx4_pd_free(dev, mdev->priv_pdn); -err_free_dev: - kfree(mdev); -err_free_res: - return NULL; -} - -static struct mlx4_interface mlx4_en_interface = { - .add = mlx4_en_add, - .remove = mlx4_en_remove, - .event = mlx4_en_event, - .get_dev = mlx4_en_get_netdev, - .protocol = MLX4_PROT_ETH, -}; - -static void mlx4_en_verify_params(void) -{ - if (pfctx > MAX_PFC_TX) { - pr_warn("mlx4_en: WARNING: illegal module parameter pfctx 0x%x - " - "should be in range 0-0x%x, will be changed to default (0)\n", - pfctx, MAX_PFC_TX); - pfctx = 0; - } - - if (pfcrx > MAX_PFC_RX) { - pr_warn("mlx4_en: WARNING: illegal module parameter pfcrx 0x%x - " - "should be in range 0-0x%x, will be changed to default (0)\n", - pfcrx, MAX_PFC_RX); - pfcrx = 0; - } -} - - -static int __init mlx4_en_init(void) -{ - mlx4_en_verify_params(); - -#ifdef CONFIG_DEBUG_FS - int err = 0; - err = mlx4_en_register_debugfs(); - if (err) - pr_err(KERN_ERR "Failed to register debugfs\n"); -#endif - return mlx4_register_interface(&mlx4_en_interface); -} - -static void __exit mlx4_en_cleanup(void) -{ - mlx4_unregister_interface(&mlx4_en_interface); -#ifdef CONFIG_DEBUG_FS - mlx4_en_unregister_debugfs(); -#endif -} - -module_init(mlx4_en_init); -module_exit(mlx4_en_cleanup); - -static int -mlxen_evhand(module_t mod, int event, void *arg) -{ - return (0); -} -static moduledata_t mlxen_mod = { - .name = "mlxen", - .evhand = mlxen_evhand, -}; -DECLARE_MODULE(mlxen, mlxen_mod, SI_SUB_OFED_PREINIT, SI_ORDER_ANY); -MODULE_DEPEND(mlxen, mlx4, 1, 1, 1); -MODULE_DEPEND(mlxen, linuxkpi, 1, 1, 1); Property changes on: stable/11/sys/ofed/drivers/net/mlx4/en_main.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/reset.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/reset.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/reset.c (nonexistent) @@ -1,185 +0,0 @@ -/* - * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include -#include - -#include "mlx4.h" - -int mlx4_reset(struct mlx4_dev *dev) -{ - void __iomem *reset; - u32 *hca_header = NULL; - int pcie_cap; - u16 devctl; - u16 linkctl; - u16 vendor; - unsigned long end; - u32 sem; - int i; - int err = 0; - -#define MLX4_RESET_BASE 0xf0000 -#define MLX4_RESET_SIZE 0x400 -#define MLX4_SEM_OFFSET 0x3fc -#define MLX4_RESET_OFFSET 0x10 -#define MLX4_RESET_VALUE swab32(1) - -#define MLX4_SEM_TIMEOUT_JIFFIES (10 * HZ) -#define MLX4_RESET_TIMEOUT_JIFFIES (2 * HZ) - - /* - * Reset the chip. This is somewhat ugly because we have to - * save off the PCI header before reset and then restore it - * after the chip reboots. We skip config space offsets 22 - * and 23 since those have a special meaning. - */ - - /* Do we need to save off the full 4K PCI Express header?? */ - hca_header = kmalloc(256, GFP_KERNEL); - if (!hca_header) { - err = -ENOMEM; - mlx4_err(dev, "Couldn't allocate memory to save HCA " - "PCI header, aborting.\n"); - goto out; - } - - pcie_cap = pci_pcie_cap(dev->pdev); - - for (i = 0; i < 64; ++i) { - if (i == 22 || i == 23) - continue; - if (pci_read_config_dword(dev->pdev, i * 4, hca_header + i)) { - err = -ENODEV; - mlx4_err(dev, "Couldn't save HCA " - "PCI header, aborting.\n"); - goto out; - } - } - - reset = ioremap(pci_resource_start(dev->pdev, 0) + MLX4_RESET_BASE, - MLX4_RESET_SIZE); - if (!reset) { - err = -ENOMEM; - mlx4_err(dev, "Couldn't map HCA reset register, aborting.\n"); - goto out; - } - - /* grab HW semaphore to lock out flash updates */ - end = jiffies + MLX4_SEM_TIMEOUT_JIFFIES; - do { - sem = readl(reset + MLX4_SEM_OFFSET); - if (!sem) - break; - - msleep(1); - } while (time_before(jiffies, end)); - - if (sem) { - mlx4_err(dev, "Failed to obtain HW semaphore, aborting\n"); - err = -EAGAIN; - iounmap(reset); - goto out; - } - - /* actually hit reset */ - writel(MLX4_RESET_VALUE, reset + MLX4_RESET_OFFSET); - iounmap(reset); - - /* wait half a second before accessing device */ - msleep(500); - - end = jiffies + MLX4_RESET_TIMEOUT_JIFFIES; - do { - if (!pci_read_config_word(dev->pdev, PCI_VENDOR_ID, &vendor) && - vendor != 0xffff) - break; - - msleep(1); - } while (time_before(jiffies, end)); - - if (vendor == 0xffff) { - err = -ENODEV; - mlx4_err(dev, "PCI device did not come back after reset, " - "aborting.\n"); - goto out; - } - - /* Now restore the PCI headers */ - if (pcie_cap) { - devctl = hca_header[(pcie_cap + PCI_EXP_DEVCTL) / 4]; - if (pcie_capability_write_word(dev->pdev, PCI_EXP_DEVCTL, - devctl)) { - err = -ENODEV; - mlx4_err(dev, "Couldn't restore HCA PCI Express " - "Device Control register, aborting.\n"); - goto out; - } - linkctl = hca_header[(pcie_cap + PCI_EXP_LNKCTL) / 4]; - if (pcie_capability_write_word(dev->pdev, PCI_EXP_LNKCTL, - linkctl)) { - err = -ENODEV; - mlx4_err(dev, "Couldn't restore HCA PCI Express " - "Link control register, aborting.\n"); - goto out; - } - } - - for (i = 0; i < 16; ++i) { - if (i * 4 == PCI_COMMAND) - continue; - - if (pci_write_config_dword(dev->pdev, i * 4, hca_header[i])) { - err = -ENODEV; - mlx4_err(dev, "Couldn't restore HCA reg %x, " - "aborting.\n", i); - goto out; - } - } - - if (pci_write_config_dword(dev->pdev, PCI_COMMAND, - hca_header[PCI_COMMAND / 4])) { - err = -ENODEV; - mlx4_err(dev, "Couldn't restore HCA COMMAND, " - "aborting.\n"); - goto out; - } - -out: - kfree(hca_header); - - return err; -} Property changes on: stable/11/sys/ofed/drivers/net/mlx4/reset.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/en_netdev.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/en_netdev.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/en_netdev.c (nonexistent) @@ -1,2799 +0,0 @@ -/* - * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include -#include -#include -#ifdef CONFIG_NET_RX_BUSY_POLL -#include -#endif - -#include -#include - -#include -#include -#include -#include - -#include -#include - -#include "mlx4_en.h" -#include "en_port.h" - -static void mlx4_en_sysctl_stat(struct mlx4_en_priv *priv); -static void mlx4_en_sysctl_conf(struct mlx4_en_priv *priv); - -#ifdef CONFIG_NET_RX_BUSY_POLL -/* must be called with local_bh_disable()d */ -static int mlx4_en_low_latency_recv(struct napi_struct *napi) -{ - struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi); - struct net_device *dev = cq->dev; - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_rx_ring *rx_ring = priv->rx_ring[cq->ring]; - int done; - - if (!priv->port_up) - return LL_FLUSH_FAILED; - - if (!mlx4_en_cq_lock_poll(cq)) - return LL_FLUSH_BUSY; - - done = mlx4_en_process_rx_cq(dev, cq, 4); -#ifdef LL_EXTENDED_STATS - if (done) - rx_ring->cleaned += done; - else - rx_ring->misses++; -#endif - - mlx4_en_cq_unlock_poll(cq); - - return done; -} -#endif /* CONFIG_NET_RX_BUSY_POLL */ - -#ifdef CONFIG_RFS_ACCEL - -struct mlx4_en_filter { - struct list_head next; - struct work_struct work; - - u8 ip_proto; - __be32 src_ip; - __be32 dst_ip; - __be16 src_port; - __be16 dst_port; - - int rxq_index; - struct mlx4_en_priv *priv; - u32 flow_id; /* RFS infrastructure id */ - int id; /* mlx4_en driver id */ - u64 reg_id; /* Flow steering API id */ - u8 activated; /* Used to prevent expiry before filter - * is attached - */ - struct hlist_node filter_chain; -}; - -static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv); - -static enum mlx4_net_trans_rule_id mlx4_ip_proto_to_trans_rule_id(u8 ip_proto) -{ - switch (ip_proto) { - case IPPROTO_UDP: - return MLX4_NET_TRANS_RULE_ID_UDP; - case IPPROTO_TCP: - return MLX4_NET_TRANS_RULE_ID_TCP; - default: - return -EPROTONOSUPPORT; - } -}; - -static void mlx4_en_filter_work(struct work_struct *work) -{ - struct mlx4_en_filter *filter = container_of(work, - struct mlx4_en_filter, - work); - struct mlx4_en_priv *priv = filter->priv; - struct mlx4_spec_list spec_tcp_udp = { - .id = mlx4_ip_proto_to_trans_rule_id(filter->ip_proto), - { - .tcp_udp = { - .dst_port = filter->dst_port, - .dst_port_msk = (__force __be16)-1, - .src_port = filter->src_port, - .src_port_msk = (__force __be16)-1, - }, - }, - }; - struct mlx4_spec_list spec_ip = { - .id = MLX4_NET_TRANS_RULE_ID_IPV4, - { - .ipv4 = { - .dst_ip = filter->dst_ip, - .dst_ip_msk = (__force __be32)-1, - .src_ip = filter->src_ip, - .src_ip_msk = (__force __be32)-1, - }, - }, - }; - struct mlx4_spec_list spec_eth = { - .id = MLX4_NET_TRANS_RULE_ID_ETH, - }; - struct mlx4_net_trans_rule rule = { - .list = LIST_HEAD_INIT(rule.list), - .queue_mode = MLX4_NET_TRANS_Q_LIFO, - .exclusive = 1, - .allow_loopback = 1, - .promisc_mode = MLX4_FS_REGULAR, - .port = priv->port, - .priority = MLX4_DOMAIN_RFS, - }; - int rc; - __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); - - if (spec_tcp_udp.id < 0) { - en_warn(priv, "RFS: ignoring unsupported ip protocol (%d)\n", - filter->ip_proto); - goto ignore; - } - list_add_tail(&spec_eth.list, &rule.list); - list_add_tail(&spec_ip.list, &rule.list); - list_add_tail(&spec_tcp_udp.list, &rule.list); - - rule.qpn = priv->rss_map.qps[filter->rxq_index].qpn; - memcpy(spec_eth.eth.dst_mac, priv->dev->dev_addr, ETH_ALEN); - memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN); - - filter->activated = 0; - - if (filter->reg_id) { - rc = mlx4_flow_detach(priv->mdev->dev, filter->reg_id); - if (rc && rc != -ENOENT) - en_err(priv, "Error detaching flow. rc = %d\n", rc); - } - - rc = mlx4_flow_attach(priv->mdev->dev, &rule, &filter->reg_id); - if (rc) - en_err(priv, "Error attaching flow. err = %d\n", rc); - -ignore: - mlx4_en_filter_rfs_expire(priv); - - filter->activated = 1; -} - -static inline struct hlist_head * -filter_hash_bucket(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip, - __be16 src_port, __be16 dst_port) -{ - unsigned long l; - int bucket_idx; - - l = (__force unsigned long)src_port | - ((__force unsigned long)dst_port << 2); - l ^= (__force unsigned long)(src_ip ^ dst_ip); - - bucket_idx = hash_long(l, MLX4_EN_FILTER_HASH_SHIFT); - - return &priv->filter_hash[bucket_idx]; -} - -static struct mlx4_en_filter * -mlx4_en_filter_alloc(struct mlx4_en_priv *priv, int rxq_index, __be32 src_ip, - __be32 dst_ip, u8 ip_proto, __be16 src_port, - __be16 dst_port, u32 flow_id) -{ - struct mlx4_en_filter *filter = NULL; - - filter = kzalloc(sizeof(struct mlx4_en_filter), GFP_ATOMIC); - if (!filter) - return NULL; - - filter->priv = priv; - filter->rxq_index = rxq_index; - INIT_WORK(&filter->work, mlx4_en_filter_work); - - filter->src_ip = src_ip; - filter->dst_ip = dst_ip; - filter->ip_proto = ip_proto; - filter->src_port = src_port; - filter->dst_port = dst_port; - - filter->flow_id = flow_id; - - filter->id = priv->last_filter_id++ % RPS_NO_FILTER; - - list_add_tail(&filter->next, &priv->filters); - hlist_add_head(&filter->filter_chain, - filter_hash_bucket(priv, src_ip, dst_ip, src_port, - dst_port)); - - return filter; -} - -static void mlx4_en_filter_free(struct mlx4_en_filter *filter) -{ - struct mlx4_en_priv *priv = filter->priv; - int rc; - - list_del(&filter->next); - - rc = mlx4_flow_detach(priv->mdev->dev, filter->reg_id); - if (rc && rc != -ENOENT) - en_err(priv, "Error detaching flow. rc = %d\n", rc); - - kfree(filter); -} - -static inline struct mlx4_en_filter * -mlx4_en_filter_find(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip, - u8 ip_proto, __be16 src_port, __be16 dst_port) -{ - struct mlx4_en_filter *filter; - struct mlx4_en_filter *ret = NULL; - - hlist_for_each_entry(filter, - filter_hash_bucket(priv, src_ip, dst_ip, - src_port, dst_port), - filter_chain) { - if (filter->src_ip == src_ip && - filter->dst_ip == dst_ip && - filter->ip_proto == ip_proto && - filter->src_port == src_port && - filter->dst_port == dst_port) { - ret = filter; - break; - } - } - - return ret; -} - -static int -mlx4_en_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, - u16 rxq_index, u32 flow_id) -{ - struct mlx4_en_priv *priv = netdev_priv(net_dev); - struct mlx4_en_filter *filter; - const struct iphdr *ip; - const __be16 *ports; - u8 ip_proto; - __be32 src_ip; - __be32 dst_ip; - __be16 src_port; - __be16 dst_port; - int nhoff = skb_network_offset(skb); - int ret = 0; - - if (skb->protocol != htons(ETH_P_IP)) - return -EPROTONOSUPPORT; - - ip = (const struct iphdr *)(skb->data + nhoff); - if (ip_is_fragment(ip)) - return -EPROTONOSUPPORT; - - if ((ip->protocol != IPPROTO_TCP) && (ip->protocol != IPPROTO_UDP)) - return -EPROTONOSUPPORT; - ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl); - - ip_proto = ip->protocol; - src_ip = ip->saddr; - dst_ip = ip->daddr; - src_port = ports[0]; - dst_port = ports[1]; - - spin_lock_bh(&priv->filters_lock); - filter = mlx4_en_filter_find(priv, src_ip, dst_ip, ip_proto, - src_port, dst_port); - if (filter) { - if (filter->rxq_index == rxq_index) - goto out; - - filter->rxq_index = rxq_index; - } else { - filter = mlx4_en_filter_alloc(priv, rxq_index, - src_ip, dst_ip, ip_proto, - src_port, dst_port, flow_id); - if (!filter) { - ret = -ENOMEM; - goto err; - } - } - - queue_work(priv->mdev->workqueue, &filter->work); - -out: - ret = filter->id; -err: - spin_unlock_bh(&priv->filters_lock); - - return ret; -} - -void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *rx_ring) -{ - struct mlx4_en_filter *filter, *tmp; - LIST_HEAD(del_list); - - spin_lock_bh(&priv->filters_lock); - list_for_each_entry_safe(filter, tmp, &priv->filters, next) { - list_move(&filter->next, &del_list); - hlist_del(&filter->filter_chain); - } - spin_unlock_bh(&priv->filters_lock); - - list_for_each_entry_safe(filter, tmp, &del_list, next) { - cancel_work_sync(&filter->work); - mlx4_en_filter_free(filter); - } -} - -static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv) -{ - struct mlx4_en_filter *filter = NULL, *tmp, *last_filter = NULL; - LIST_HEAD(del_list); - int i = 0; - - spin_lock_bh(&priv->filters_lock); - list_for_each_entry_safe(filter, tmp, &priv->filters, next) { - if (i > MLX4_EN_FILTER_EXPIRY_QUOTA) - break; - - if (filter->activated && - !work_pending(&filter->work) && - rps_may_expire_flow(priv->dev, - filter->rxq_index, filter->flow_id, - filter->id)) { - list_move(&filter->next, &del_list); - hlist_del(&filter->filter_chain); - } else - last_filter = filter; - - i++; - } - - if (last_filter && (&last_filter->next != priv->filters.next)) - list_move(&priv->filters, &last_filter->next); - - spin_unlock_bh(&priv->filters_lock); - - list_for_each_entry_safe(filter, tmp, &del_list, next) - mlx4_en_filter_free(filter); -} -#endif - -static void mlx4_en_vlan_rx_add_vid(void *arg, struct net_device *dev, u16 vid) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int err; - int idx; - - if (arg != priv) - return; - - en_dbg(HW, priv, "adding VLAN:%d\n", vid); - - set_bit(vid, priv->active_vlans); - - /* Add VID to port VLAN filter */ - mutex_lock(&mdev->state_lock); - if (mdev->device_up && priv->port_up) { - err = mlx4_SET_VLAN_FLTR(mdev->dev, priv); - if (err) - en_err(priv, "Failed configuring VLAN filter\n"); - } - if (mlx4_register_vlan(mdev->dev, priv->port, vid, &idx)) - en_dbg(HW, priv, "failed adding vlan %d\n", vid); - mutex_unlock(&mdev->state_lock); - -} - -static void mlx4_en_vlan_rx_kill_vid(void *arg, struct net_device *dev, u16 vid) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int err; - - if (arg != priv) - return; - - en_dbg(HW, priv, "Killing VID:%d\n", vid); - - clear_bit(vid, priv->active_vlans); - - /* Remove VID from port VLAN filter */ - mutex_lock(&mdev->state_lock); - mlx4_unregister_vlan(mdev->dev, priv->port, vid); - - if (mdev->device_up && priv->port_up) { - err = mlx4_SET_VLAN_FLTR(mdev->dev, priv); - if (err) - en_err(priv, "Failed configuring VLAN filter\n"); - } - mutex_unlock(&mdev->state_lock); - -} - -static int mlx4_en_uc_steer_add(struct mlx4_en_priv *priv, - unsigned char *mac, int *qpn, u64 *reg_id) -{ - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_dev *dev = mdev->dev; - int err; - - switch (dev->caps.steering_mode) { - case MLX4_STEERING_MODE_B0: { - struct mlx4_qp qp; - u8 gid[16] = {0}; - - qp.qpn = *qpn; - memcpy(&gid[10], mac, ETH_ALEN); - gid[5] = priv->port; - - err = mlx4_unicast_attach(dev, &qp, gid, 0, MLX4_PROT_ETH); - break; - } - case MLX4_STEERING_MODE_DEVICE_MANAGED: { - struct mlx4_spec_list spec_eth = { {NULL} }; - __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); - - struct mlx4_net_trans_rule rule = { - .queue_mode = MLX4_NET_TRANS_Q_FIFO, - .exclusive = 0, - .allow_loopback = 1, - .promisc_mode = MLX4_FS_REGULAR, - .priority = MLX4_DOMAIN_NIC, - }; - - rule.port = priv->port; - rule.qpn = *qpn; - INIT_LIST_HEAD(&rule.list); - - spec_eth.id = MLX4_NET_TRANS_RULE_ID_ETH; - memcpy(spec_eth.eth.dst_mac, mac, ETH_ALEN); - memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN); - list_add_tail(&spec_eth.list, &rule.list); - - err = mlx4_flow_attach(dev, &rule, reg_id); - break; - } - default: - return -EINVAL; - } - if (err) - en_warn(priv, "Failed Attaching Unicast\n"); - - return err; -} - -static void mlx4_en_uc_steer_release(struct mlx4_en_priv *priv, - unsigned char *mac, int qpn, u64 reg_id) -{ - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_dev *dev = mdev->dev; - - switch (dev->caps.steering_mode) { - case MLX4_STEERING_MODE_B0: { - struct mlx4_qp qp; - u8 gid[16] = {0}; - - qp.qpn = qpn; - memcpy(&gid[10], mac, ETH_ALEN); - gid[5] = priv->port; - - mlx4_unicast_detach(dev, &qp, gid, MLX4_PROT_ETH); - break; - } - case MLX4_STEERING_MODE_DEVICE_MANAGED: { - mlx4_flow_detach(dev, reg_id); - break; - } - default: - en_err(priv, "Invalid steering mode.\n"); - } -} - -static int mlx4_en_get_qp(struct mlx4_en_priv *priv) -{ - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_dev *dev = mdev->dev; - struct mlx4_mac_entry *entry; - int index = 0; - int err = 0; - u64 reg_id; - int *qpn = &priv->base_qpn; - u64 mac = mlx4_mac_to_u64(IF_LLADDR(priv->dev)); - - en_dbg(DRV, priv, "Registering MAC: %pM for adding\n", - IF_LLADDR(priv->dev)); - index = mlx4_register_mac(dev, priv->port, mac); - if (index < 0) { - err = index; - en_err(priv, "Failed adding MAC: %pM\n", - IF_LLADDR(priv->dev)); - return err; - } - - if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) { - int base_qpn = mlx4_get_base_qpn(dev, priv->port); - *qpn = base_qpn + index; - return 0; - } - - err = mlx4_qp_reserve_range(dev, 1, 1, qpn, 0); - en_dbg(DRV, priv, "Reserved qp %d\n", *qpn); - if (err) { - en_err(priv, "Failed to reserve qp for mac registration\n"); - goto qp_err; - } - - err = mlx4_en_uc_steer_add(priv, IF_LLADDR(priv->dev), qpn, ®_id); - if (err) - goto steer_err; - - entry = kmalloc(sizeof(*entry), GFP_KERNEL); - if (!entry) { - err = -ENOMEM; - goto alloc_err; - } - memcpy(entry->mac, IF_LLADDR(priv->dev), sizeof(entry->mac)); - entry->reg_id = reg_id; - - hlist_add_head(&entry->hlist, - &priv->mac_hash[entry->mac[MLX4_EN_MAC_HASH_IDX]]); - - return 0; - -alloc_err: - mlx4_en_uc_steer_release(priv, IF_LLADDR(priv->dev), *qpn, reg_id); - -steer_err: - mlx4_qp_release_range(dev, *qpn, 1); - -qp_err: - mlx4_unregister_mac(dev, priv->port, mac); - return err; -} - -static void mlx4_en_put_qp(struct mlx4_en_priv *priv) -{ - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_dev *dev = mdev->dev; - int qpn = priv->base_qpn; - u64 mac; - - if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) { - mac = mlx4_mac_to_u64(IF_LLADDR(priv->dev)); - en_dbg(DRV, priv, "Registering MAC: %pM for deleting\n", - IF_LLADDR(priv->dev)); - mlx4_unregister_mac(dev, priv->port, mac); - } else { - struct mlx4_mac_entry *entry; - struct hlist_node *tmp; - struct hlist_head *bucket; - unsigned int i; - - for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) { - bucket = &priv->mac_hash[i]; - hlist_for_each_entry_safe(entry, tmp, bucket, hlist) { - mac = mlx4_mac_to_u64(entry->mac); - en_dbg(DRV, priv, "Registering MAC: %pM for deleting\n", - entry->mac); - mlx4_en_uc_steer_release(priv, entry->mac, - qpn, entry->reg_id); - - mlx4_unregister_mac(dev, priv->port, mac); - hlist_del(&entry->hlist); - kfree(entry); - } - } - - en_dbg(DRV, priv, "Releasing qp: port %d, qpn %d\n", - priv->port, qpn); - mlx4_qp_release_range(dev, qpn, 1); - priv->flags &= ~MLX4_EN_FLAG_FORCE_PROMISC; - } -} - -static void mlx4_en_clear_list(struct net_device *dev) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_mc_list *tmp, *mc_to_del; - - list_for_each_entry_safe(mc_to_del, tmp, &priv->mc_list, list) { - list_del(&mc_to_del->list); - kfree(mc_to_del); - } -} - -static void mlx4_en_cache_mclist(struct net_device *dev) -{ - struct ifmultiaddr *ifma; - struct mlx4_en_mc_list *tmp; - struct mlx4_en_priv *priv = netdev_priv(dev); - - if_maddr_rlock(dev); - TAILQ_FOREACH(ifma, &dev->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - if (((struct sockaddr_dl *)ifma->ifma_addr)->sdl_alen != - ETHER_ADDR_LEN) - continue; - /* Make sure the list didn't grow. */ - tmp = kzalloc(sizeof(struct mlx4_en_mc_list), GFP_ATOMIC); - if (tmp == NULL) { - en_err(priv, "Failed to allocate multicast list\n"); - break; - } - memcpy(tmp->addr, - LLADDR((struct sockaddr_dl *)ifma->ifma_addr), ETH_ALEN); - list_add_tail(&tmp->list, &priv->mc_list); - } - if_maddr_runlock(dev); -} - -static void update_mclist_flags(struct mlx4_en_priv *priv, - struct list_head *dst, - struct list_head *src) -{ - struct mlx4_en_mc_list *dst_tmp, *src_tmp, *new_mc; - bool found; - - /* Find all the entries that should be removed from dst, - * These are the entries that are not found in src - */ - list_for_each_entry(dst_tmp, dst, list) { - found = false; - list_for_each_entry(src_tmp, src, list) { - if (!memcmp(dst_tmp->addr, src_tmp->addr, ETH_ALEN)) { - found = true; - break; - } - } - if (!found) - dst_tmp->action = MCLIST_REM; - } - - /* Add entries that exist in src but not in dst - * mark them as need to add - */ - list_for_each_entry(src_tmp, src, list) { - found = false; - list_for_each_entry(dst_tmp, dst, list) { - if (!memcmp(dst_tmp->addr, src_tmp->addr, ETH_ALEN)) { - dst_tmp->action = MCLIST_NONE; - found = true; - break; - } - } - if (!found) { - new_mc = kmalloc(sizeof(struct mlx4_en_mc_list), - GFP_KERNEL); - if (!new_mc) { - en_err(priv, "Failed to allocate current multicast list\n"); - return; - } - memcpy(new_mc, src_tmp, - sizeof(struct mlx4_en_mc_list)); - new_mc->action = MCLIST_ADD; - list_add_tail(&new_mc->list, dst); - } - } -} - -static void mlx4_en_set_rx_mode(struct net_device *dev) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - - if (!priv->port_up) - return; - - queue_work(priv->mdev->workqueue, &priv->rx_mode_task); -} - -static void mlx4_en_set_promisc_mode(struct mlx4_en_priv *priv, - struct mlx4_en_dev *mdev) -{ - int err = 0; - if (!(priv->flags & MLX4_EN_FLAG_PROMISC)) { - priv->flags |= MLX4_EN_FLAG_PROMISC; - - /* Enable promiscouos mode */ - switch (mdev->dev->caps.steering_mode) { - case MLX4_STEERING_MODE_DEVICE_MANAGED: - err = mlx4_flow_steer_promisc_add(mdev->dev, - priv->port, - priv->base_qpn, - MLX4_FS_ALL_DEFAULT); - if (err) - en_err(priv, "Failed enabling promiscuous mode\n"); - priv->flags |= MLX4_EN_FLAG_MC_PROMISC; - break; - - case MLX4_STEERING_MODE_B0: - err = mlx4_unicast_promisc_add(mdev->dev, - priv->base_qpn, - priv->port); - if (err) - en_err(priv, "Failed enabling unicast promiscuous mode\n"); - - /* Add the default qp number as multicast - * promisc - */ - if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) { - err = mlx4_multicast_promisc_add(mdev->dev, - priv->base_qpn, - priv->port); - if (err) - en_err(priv, "Failed enabling multicast promiscuous mode\n"); - priv->flags |= MLX4_EN_FLAG_MC_PROMISC; - } - break; - - case MLX4_STEERING_MODE_A0: - err = mlx4_SET_PORT_qpn_calc(mdev->dev, - priv->port, - priv->base_qpn, - 1); - if (err) - en_err(priv, "Failed enabling promiscuous mode\n"); - break; - } - - /* Disable port multicast filter (unconditionally) */ - err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, - 0, MLX4_MCAST_DISABLE); - if (err) - en_err(priv, "Failed disabling multicast filter\n"); - } -} - -static void mlx4_en_clear_promisc_mode(struct mlx4_en_priv *priv, - struct mlx4_en_dev *mdev) -{ - int err = 0; - - priv->flags &= ~MLX4_EN_FLAG_PROMISC; - - /* Disable promiscouos mode */ - switch (mdev->dev->caps.steering_mode) { - case MLX4_STEERING_MODE_DEVICE_MANAGED: - err = mlx4_flow_steer_promisc_remove(mdev->dev, - priv->port, - MLX4_FS_ALL_DEFAULT); - if (err) - en_err(priv, "Failed disabling promiscuous mode\n"); - priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; - break; - - case MLX4_STEERING_MODE_B0: - err = mlx4_unicast_promisc_remove(mdev->dev, - priv->base_qpn, - priv->port); - if (err) - en_err(priv, "Failed disabling unicast promiscuous mode\n"); - /* Disable Multicast promisc */ - if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { - err = mlx4_multicast_promisc_remove(mdev->dev, - priv->base_qpn, - priv->port); - if (err) - en_err(priv, "Failed disabling multicast promiscuous mode\n"); - priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; - } - break; - - case MLX4_STEERING_MODE_A0: - err = mlx4_SET_PORT_qpn_calc(mdev->dev, - priv->port, - priv->base_qpn, 0); - if (err) - en_err(priv, "Failed disabling promiscuous mode\n"); - break; - } -} - -static void mlx4_en_do_multicast(struct mlx4_en_priv *priv, - struct net_device *dev, - struct mlx4_en_dev *mdev) -{ - struct mlx4_en_mc_list *mclist, *tmp; - u8 mc_list[16] = {0}; - int err = 0; - u64 mcast_addr = 0; - - - /* Enable/disable the multicast filter according to IFF_ALLMULTI */ - if (dev->if_flags & IFF_ALLMULTI) { - err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, - 0, MLX4_MCAST_DISABLE); - if (err) - en_err(priv, "Failed disabling multicast filter\n"); - - /* Add the default qp number as multicast promisc */ - if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) { - switch (mdev->dev->caps.steering_mode) { - case MLX4_STEERING_MODE_DEVICE_MANAGED: - err = mlx4_flow_steer_promisc_add(mdev->dev, - priv->port, - priv->base_qpn, - MLX4_FS_MC_DEFAULT); - break; - - case MLX4_STEERING_MODE_B0: - err = mlx4_multicast_promisc_add(mdev->dev, - priv->base_qpn, - priv->port); - break; - - case MLX4_STEERING_MODE_A0: - break; - } - if (err) - en_err(priv, "Failed entering multicast promisc mode\n"); - priv->flags |= MLX4_EN_FLAG_MC_PROMISC; - } - } else { - /* Disable Multicast promisc */ - if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { - switch (mdev->dev->caps.steering_mode) { - case MLX4_STEERING_MODE_DEVICE_MANAGED: - err = mlx4_flow_steer_promisc_remove(mdev->dev, - priv->port, - MLX4_FS_MC_DEFAULT); - break; - - case MLX4_STEERING_MODE_B0: - err = mlx4_multicast_promisc_remove(mdev->dev, - priv->base_qpn, - priv->port); - break; - - case MLX4_STEERING_MODE_A0: - break; - } - if (err) - en_err(priv, "Failed disabling multicast promiscuous mode\n"); - priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; - } - - err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, - 0, MLX4_MCAST_DISABLE); - if (err) - en_err(priv, "Failed disabling multicast filter\n"); - - /* Flush mcast filter and init it with broadcast address */ - mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, ETH_BCAST, - 1, MLX4_MCAST_CONFIG); - - /* Update multicast list - we cache all addresses so they won't - * change while HW is updated holding the command semaphor */ - mlx4_en_cache_mclist(dev); - list_for_each_entry(mclist, &priv->mc_list, list) { - mcast_addr = mlx4_mac_to_u64(mclist->addr); - mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, - mcast_addr, 0, MLX4_MCAST_CONFIG); - } - err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, - 0, MLX4_MCAST_ENABLE); - if (err) - en_err(priv, "Failed enabling multicast filter\n"); - - update_mclist_flags(priv, &priv->curr_list, &priv->mc_list); - list_for_each_entry_safe(mclist, tmp, &priv->curr_list, list) { - if (mclist->action == MCLIST_REM) { - /* detach this address and delete from list */ - memcpy(&mc_list[10], mclist->addr, ETH_ALEN); - mc_list[5] = priv->port; - err = mlx4_multicast_detach(mdev->dev, - &priv->rss_map.indir_qp, - mc_list, - MLX4_PROT_ETH, - mclist->reg_id); - if (err) - en_err(priv, "Fail to detach multicast address\n"); - - /* remove from list */ - list_del(&mclist->list); - kfree(mclist); - } else if (mclist->action == MCLIST_ADD) { - /* attach the address */ - memcpy(&mc_list[10], mclist->addr, ETH_ALEN); - /* needed for B0 steering support */ - mc_list[5] = priv->port; - err = mlx4_multicast_attach(mdev->dev, - &priv->rss_map.indir_qp, - mc_list, - priv->port, 0, - MLX4_PROT_ETH, - &mclist->reg_id); - if (err) - en_err(priv, "Fail to attach multicast address\n"); - - } - } - } -} - -static void mlx4_en_do_set_rx_mode(struct work_struct *work) -{ - struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, - rx_mode_task); - struct mlx4_en_dev *mdev = priv->mdev; - struct net_device *dev = priv->dev; - - - mutex_lock(&mdev->state_lock); - if (!mdev->device_up) { - en_dbg(HW, priv, "Card is not up, ignoring rx mode change.\n"); - goto out; - } - if (!priv->port_up) { - en_dbg(HW, priv, "Port is down, ignoring rx mode change.\n"); - goto out; - } - if (!mlx4_en_QUERY_PORT(mdev, priv->port)) { - if (priv->port_state.link_state) { - priv->last_link_state = MLX4_DEV_EVENT_PORT_UP; - /* update netif baudrate */ - priv->dev->if_baudrate = - IF_Mbps(priv->port_state.link_speed); - /* Important note: the following call for if_link_state_change - * is needed for interface up scenario (start port, link state - * change) */ - if_link_state_change(priv->dev, LINK_STATE_UP); - en_dbg(HW, priv, "Link Up\n"); - } - } - - /* Promsicuous mode: disable all filters */ - if ((dev->if_flags & IFF_PROMISC) || - (priv->flags & MLX4_EN_FLAG_FORCE_PROMISC)) { - mlx4_en_set_promisc_mode(priv, mdev); - goto out; - } - - /* Not in promiscuous mode */ - if (priv->flags & MLX4_EN_FLAG_PROMISC) - mlx4_en_clear_promisc_mode(priv, mdev); - - mlx4_en_do_multicast(priv, dev, mdev); -out: - mutex_unlock(&mdev->state_lock); -} - -#ifdef CONFIG_NET_POLL_CONTROLLER -static void mlx4_en_netpoll(struct net_device *dev) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_cq *cq; - unsigned long flags; - int i; - - for (i = 0; i < priv->rx_ring_num; i++) { - cq = priv->rx_cq[i]; - spin_lock_irqsave(&cq->lock, flags); - napi_synchronize(&cq->napi); - mlx4_en_process_rx_cq(dev, cq, 0); - spin_unlock_irqrestore(&cq->lock, flags); - } -} -#endif - -static void mlx4_en_watchdog_timeout(void *arg) -{ - struct mlx4_en_priv *priv = arg; - struct mlx4_en_dev *mdev = priv->mdev; - - en_dbg(DRV, priv, "Scheduling watchdog\n"); - queue_work(mdev->workqueue, &priv->watchdog_task); - if (priv->port_up) - callout_reset(&priv->watchdog_timer, MLX4_EN_WATCHDOG_TIMEOUT, - mlx4_en_watchdog_timeout, priv); -} - - - -static void mlx4_en_set_default_moderation(struct mlx4_en_priv *priv) -{ - struct mlx4_en_cq *cq; - int i; - - /* If we haven't received a specific coalescing setting - * (module param), we set the moderation parameters as follows: - * - moder_cnt is set to the number of mtu sized packets to - * satisfy our coelsing target. - * - moder_time is set to a fixed value. - */ - priv->rx_frames = MLX4_EN_RX_COAL_TARGET / priv->dev->if_mtu + 1; - priv->rx_usecs = MLX4_EN_RX_COAL_TIME; - priv->tx_frames = MLX4_EN_TX_COAL_PKTS; - priv->tx_usecs = MLX4_EN_TX_COAL_TIME; - en_dbg(INTR, priv, "Default coalesing params for mtu: %u - " - "rx_frames:%d rx_usecs:%d\n", - (unsigned)priv->dev->if_mtu, priv->rx_frames, priv->rx_usecs); - - /* Setup cq moderation params */ - for (i = 0; i < priv->rx_ring_num; i++) { - cq = priv->rx_cq[i]; - cq->moder_cnt = priv->rx_frames; - cq->moder_time = priv->rx_usecs; - priv->last_moder_time[i] = MLX4_EN_AUTO_CONF; - priv->last_moder_packets[i] = 0; - priv->last_moder_bytes[i] = 0; - } - - for (i = 0; i < priv->tx_ring_num; i++) { - cq = priv->tx_cq[i]; - cq->moder_cnt = priv->tx_frames; - cq->moder_time = priv->tx_usecs; - } - - /* Reset auto-moderation params */ - priv->pkt_rate_low = MLX4_EN_RX_RATE_LOW; - priv->rx_usecs_low = MLX4_EN_RX_COAL_TIME_LOW; - priv->pkt_rate_high = MLX4_EN_RX_RATE_HIGH; - priv->rx_usecs_high = MLX4_EN_RX_COAL_TIME_HIGH; - priv->sample_interval = MLX4_EN_SAMPLE_INTERVAL; - priv->adaptive_rx_coal = 1; - priv->last_moder_jiffies = 0; - priv->last_moder_tx_packets = 0; -} - -static void mlx4_en_auto_moderation(struct mlx4_en_priv *priv) -{ - unsigned long period = (unsigned long) (jiffies - priv->last_moder_jiffies); - struct mlx4_en_cq *cq; - unsigned long packets; - unsigned long rate; - unsigned long avg_pkt_size; - unsigned long rx_packets; - unsigned long rx_bytes; - unsigned long rx_pkt_diff; - int moder_time; - int ring, err; - - if (!priv->adaptive_rx_coal || period < priv->sample_interval * HZ) - return; - - for (ring = 0; ring < priv->rx_ring_num; ring++) { - spin_lock(&priv->stats_lock); - rx_packets = priv->rx_ring[ring]->packets; - rx_bytes = priv->rx_ring[ring]->bytes; - spin_unlock(&priv->stats_lock); - - rx_pkt_diff = ((unsigned long) (rx_packets - - priv->last_moder_packets[ring])); - packets = rx_pkt_diff; - rate = packets * HZ / period; - avg_pkt_size = packets ? ((unsigned long) (rx_bytes - - priv->last_moder_bytes[ring])) / packets : 0; - - /* Apply auto-moderation only when packet rate - * exceeds a rate that it matters */ - if (rate > (MLX4_EN_RX_RATE_THRESH / priv->rx_ring_num) && - avg_pkt_size > MLX4_EN_AVG_PKT_SMALL) { - if (rate < priv->pkt_rate_low) - moder_time = priv->rx_usecs_low; - else if (rate > priv->pkt_rate_high) - moder_time = priv->rx_usecs_high; - else - moder_time = (rate - priv->pkt_rate_low) * - (priv->rx_usecs_high - priv->rx_usecs_low) / - (priv->pkt_rate_high - priv->pkt_rate_low) + - priv->rx_usecs_low; - } else { - moder_time = priv->rx_usecs_low; - } - - if (moder_time != priv->last_moder_time[ring]) { - priv->last_moder_time[ring] = moder_time; - cq = priv->rx_cq[ring]; - cq->moder_time = moder_time; - err = mlx4_en_set_cq_moder(priv, cq); - if (err) - en_err(priv, "Failed modifying moderation for cq:%d\n", - ring); - } - priv->last_moder_packets[ring] = rx_packets; - priv->last_moder_bytes[ring] = rx_bytes; - } - - priv->last_moder_jiffies = jiffies; -} - -static void mlx4_en_do_get_stats(struct work_struct *work) -{ - struct delayed_work *delay = to_delayed_work(work); - struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv, - stats_task); - struct mlx4_en_dev *mdev = priv->mdev; - int err; - - mutex_lock(&mdev->state_lock); - if (mdev->device_up) { - if (priv->port_up) { - err = mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 0); - if (err) - en_dbg(HW, priv, "Could not update stats\n"); - - mlx4_en_auto_moderation(priv); - } - - queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY); - } - mutex_unlock(&mdev->state_lock); -} - -/* mlx4_en_service_task - Run service task for tasks that needed to be done - * periodically - */ -static void mlx4_en_service_task(struct work_struct *work) -{ - struct delayed_work *delay = to_delayed_work(work); - struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv, - service_task); - struct mlx4_en_dev *mdev = priv->mdev; - - mutex_lock(&mdev->state_lock); - if (mdev->device_up) { - queue_delayed_work(mdev->workqueue, &priv->service_task, - SERVICE_TASK_DELAY); - } - mutex_unlock(&mdev->state_lock); -} - -static void mlx4_en_linkstate(struct work_struct *work) -{ - struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, - linkstate_task); - struct mlx4_en_dev *mdev = priv->mdev; - int linkstate = priv->link_state; - - mutex_lock(&mdev->state_lock); - /* If observable port state changed set carrier state and - * report to system log */ - if (priv->last_link_state != linkstate) { - if (linkstate == MLX4_DEV_EVENT_PORT_DOWN) { - en_info(priv, "Link Down\n"); - if_link_state_change(priv->dev, LINK_STATE_DOWN); - /* update netif baudrate */ - priv->dev->if_baudrate = 0; - - /* make sure the port is up before notifying the OS. - * This is tricky since we get here on INIT_PORT and - * in such case we can't tell the OS the port is up. - * To solve this there is a call to if_link_state_change - * in set_rx_mode. - * */ - } else if (priv->port_up && (linkstate == MLX4_DEV_EVENT_PORT_UP)){ - if (mlx4_en_QUERY_PORT(priv->mdev, priv->port)) - en_info(priv, "Query port failed\n"); - priv->dev->if_baudrate = - IF_Mbps(priv->port_state.link_speed); - en_info(priv, "Link Up\n"); - if_link_state_change(priv->dev, LINK_STATE_UP); - } - } - priv->last_link_state = linkstate; - mutex_unlock(&mdev->state_lock); -} - - -int mlx4_en_start_port(struct net_device *dev) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_cq *cq; - struct mlx4_en_tx_ring *tx_ring; - int rx_index = 0; - int tx_index = 0; - int err = 0; - int i; - int j; - u8 mc_list[16] = {0}; - - - if (priv->port_up) { - en_dbg(DRV, priv, "start port called while port already up\n"); - return 0; - } - - INIT_LIST_HEAD(&priv->mc_list); - INIT_LIST_HEAD(&priv->curr_list); - INIT_LIST_HEAD(&priv->ethtool_list); - - /* Calculate Rx buf size */ - dev->if_mtu = min(dev->if_mtu, priv->max_mtu); - mlx4_en_calc_rx_buf(dev); - en_dbg(DRV, priv, "Rx buf size:%d\n", priv->rx_mb_size); - - /* Configure rx cq's and rings */ - err = mlx4_en_activate_rx_rings(priv); - if (err) { - en_err(priv, "Failed to activate RX rings\n"); - return err; - } - for (i = 0; i < priv->rx_ring_num; i++) { - cq = priv->rx_cq[i]; - - mlx4_en_cq_init_lock(cq); - err = mlx4_en_activate_cq(priv, cq, i); - if (err) { - en_err(priv, "Failed activating Rx CQ\n"); - goto cq_err; - } - for (j = 0; j < cq->size; j++) - cq->buf[j].owner_sr_opcode = MLX4_CQE_OWNER_MASK; - err = mlx4_en_set_cq_moder(priv, cq); - if (err) { - en_err(priv, "Failed setting cq moderation parameters"); - mlx4_en_deactivate_cq(priv, cq); - goto cq_err; - } - mlx4_en_arm_cq(priv, cq); - priv->rx_ring[i]->cqn = cq->mcq.cqn; - ++rx_index; - } - - /* Set qp number */ - en_dbg(DRV, priv, "Getting qp number for port %d\n", priv->port); - err = mlx4_en_get_qp(priv); - if (err) { - en_err(priv, "Failed getting eth qp\n"); - goto cq_err; - } - mdev->mac_removed[priv->port] = 0; - - /* gets default allocated counter index from func cap */ - /* or sink counter index if no resources */ - priv->counter_index = mdev->dev->caps.def_counter_index[priv->port - 1]; - - en_dbg(DRV, priv, "%s: default counter index %d for port %d\n", - __func__, priv->counter_index, priv->port); - - err = mlx4_en_config_rss_steer(priv); - if (err) { - en_err(priv, "Failed configuring rss steering\n"); - goto mac_err; - } - - err = mlx4_en_create_drop_qp(priv); - if (err) - goto rss_err; - - /* Configure tx cq's and rings */ - for (i = 0; i < priv->tx_ring_num; i++) { - /* Configure cq */ - cq = priv->tx_cq[i]; - err = mlx4_en_activate_cq(priv, cq, i); - if (err) { - en_err(priv, "Failed activating Tx CQ\n"); - goto tx_err; - } - err = mlx4_en_set_cq_moder(priv, cq); - if (err) { - en_err(priv, "Failed setting cq moderation parameters"); - mlx4_en_deactivate_cq(priv, cq); - goto tx_err; - } - en_dbg(DRV, priv, "Resetting index of collapsed CQ:%d to -1\n", i); - cq->buf->wqe_index = cpu_to_be16(0xffff); - - /* Configure ring */ - tx_ring = priv->tx_ring[i]; - - err = mlx4_en_activate_tx_ring(priv, tx_ring, cq->mcq.cqn, - i / priv->num_tx_rings_p_up); - if (err) { - en_err(priv, "Failed activating Tx ring %d\n", i); - mlx4_en_deactivate_cq(priv, cq); - goto tx_err; - } - - /* Arm CQ for TX completions */ - mlx4_en_arm_cq(priv, cq); - - /* Set initial ownership of all Tx TXBBs to SW (1) */ - for (j = 0; j < tx_ring->buf_size; j += STAMP_STRIDE) - *((u32 *) (tx_ring->buf + j)) = 0xffffffff; - ++tx_index; - } - - /* Configure port */ - err = mlx4_SET_PORT_general(mdev->dev, priv->port, - priv->rx_mb_size, - priv->prof->tx_pause, - priv->prof->tx_ppp, - priv->prof->rx_pause, - priv->prof->rx_ppp); - if (err) { - en_err(priv, "Failed setting port general configurations for port %d, with error %d\n", - priv->port, err); - goto tx_err; - } - /* Set default qp number */ - err = mlx4_SET_PORT_qpn_calc(mdev->dev, priv->port, priv->base_qpn, 0); - if (err) { - en_err(priv, "Failed setting default qp numbers\n"); - goto tx_err; - } - - /* Init port */ - en_dbg(HW, priv, "Initializing port\n"); - err = mlx4_INIT_PORT(mdev->dev, priv->port); - if (err) { - en_err(priv, "Failed Initializing port\n"); - goto tx_err; - } - - /* Attach rx QP to bradcast address */ - memset(&mc_list[10], 0xff, ETH_ALEN); - mc_list[5] = priv->port; /* needed for B0 steering support */ - if (mlx4_multicast_attach(mdev->dev, &priv->rss_map.indir_qp, mc_list, - priv->port, 0, MLX4_PROT_ETH, - &priv->broadcast_id)) - mlx4_warn(mdev, "Failed Attaching Broadcast\n"); - - /* Must redo promiscuous mode setup. */ - priv->flags &= ~(MLX4_EN_FLAG_PROMISC | MLX4_EN_FLAG_MC_PROMISC); - - /* Schedule multicast task to populate multicast list */ - queue_work(mdev->workqueue, &priv->rx_mode_task); - - priv->port_up = true; - - /* Enable the queues. */ - dev->if_drv_flags &= ~IFF_DRV_OACTIVE; - dev->if_drv_flags |= IFF_DRV_RUNNING; -#ifdef CONFIG_DEBUG_FS - mlx4_en_create_debug_files(priv); -#endif - callout_reset(&priv->watchdog_timer, MLX4_EN_WATCHDOG_TIMEOUT, - mlx4_en_watchdog_timeout, priv); - - - return 0; - -tx_err: - while (tx_index--) { - mlx4_en_deactivate_tx_ring(priv, priv->tx_ring[tx_index]); - mlx4_en_deactivate_cq(priv, priv->tx_cq[tx_index]); - } - mlx4_en_destroy_drop_qp(priv); -rss_err: - mlx4_en_release_rss_steer(priv); -mac_err: - mlx4_en_put_qp(priv); -cq_err: - while (rx_index--) - mlx4_en_deactivate_cq(priv, priv->rx_cq[rx_index]); - for (i = 0; i < priv->rx_ring_num; i++) - mlx4_en_deactivate_rx_ring(priv, priv->rx_ring[i]); - - return err; /* need to close devices */ -} - - -void mlx4_en_stop_port(struct net_device *dev) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_mc_list *mclist, *tmp; - int i; - u8 mc_list[16] = {0}; - - if (!priv->port_up) { - en_dbg(DRV, priv, "stop port called while port already down\n"); - return; - } - -#ifdef CONFIG_DEBUG_FS - mlx4_en_delete_debug_files(priv); -#endif - - /* close port*/ - mlx4_CLOSE_PORT(mdev->dev, priv->port); - - /* Set port as not active */ - priv->port_up = false; - if (priv->counter_index != 0xff) { - mlx4_counter_free(mdev->dev, priv->port, priv->counter_index); - priv->counter_index = 0xff; - } - - /* Promsicuous mode */ - if (mdev->dev->caps.steering_mode == - MLX4_STEERING_MODE_DEVICE_MANAGED) { - priv->flags &= ~(MLX4_EN_FLAG_PROMISC | - MLX4_EN_FLAG_MC_PROMISC); - mlx4_flow_steer_promisc_remove(mdev->dev, - priv->port, - MLX4_FS_ALL_DEFAULT); - mlx4_flow_steer_promisc_remove(mdev->dev, - priv->port, - MLX4_FS_MC_DEFAULT); - } else if (priv->flags & MLX4_EN_FLAG_PROMISC) { - priv->flags &= ~MLX4_EN_FLAG_PROMISC; - - /* Disable promiscouos mode */ - mlx4_unicast_promisc_remove(mdev->dev, priv->base_qpn, - priv->port); - - /* Disable Multicast promisc */ - if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { - mlx4_multicast_promisc_remove(mdev->dev, priv->base_qpn, - priv->port); - priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; - } - } - - /* Detach All multicasts */ - memset(&mc_list[10], 0xff, ETH_ALEN); - mc_list[5] = priv->port; /* needed for B0 steering support */ - mlx4_multicast_detach(mdev->dev, &priv->rss_map.indir_qp, mc_list, - MLX4_PROT_ETH, priv->broadcast_id); - list_for_each_entry(mclist, &priv->curr_list, list) { - memcpy(&mc_list[10], mclist->addr, ETH_ALEN); - mc_list[5] = priv->port; - mlx4_multicast_detach(mdev->dev, &priv->rss_map.indir_qp, - mc_list, MLX4_PROT_ETH, mclist->reg_id); - } - mlx4_en_clear_list(dev); - list_for_each_entry_safe(mclist, tmp, &priv->curr_list, list) { - list_del(&mclist->list); - kfree(mclist); - } - - /* Flush multicast filter */ - mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 1, MLX4_MCAST_CONFIG); - mlx4_en_destroy_drop_qp(priv); - - /* Free TX Rings */ - for (i = 0; i < priv->tx_ring_num; i++) { - mlx4_en_deactivate_tx_ring(priv, priv->tx_ring[i]); - mlx4_en_deactivate_cq(priv, priv->tx_cq[i]); - } - msleep(10); - - for (i = 0; i < priv->tx_ring_num; i++) - mlx4_en_free_tx_buf(dev, priv->tx_ring[i]); - - /* Free RSS qps */ - mlx4_en_release_rss_steer(priv); - - /* Unregister Mac address for the port */ - mlx4_en_put_qp(priv); - mdev->mac_removed[priv->port] = 1; - - /* Free RX Rings */ - for (i = 0; i < priv->rx_ring_num; i++) { - struct mlx4_en_cq *cq = priv->rx_cq[i]; - mlx4_en_deactivate_rx_ring(priv, priv->rx_ring[i]); - mlx4_en_deactivate_cq(priv, cq); - } - - callout_stop(&priv->watchdog_timer); - - dev->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); -} - -static void mlx4_en_restart(struct work_struct *work) -{ - struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, - watchdog_task); - struct mlx4_en_dev *mdev = priv->mdev; - struct net_device *dev = priv->dev; - struct mlx4_en_tx_ring *ring; - int i; - - - if (priv->blocked == 0 || priv->port_up == 0) - return; - for (i = 0; i < priv->tx_ring_num; i++) { - ring = priv->tx_ring[i]; - if (ring->blocked && - ring->watchdog_time + MLX4_EN_WATCHDOG_TIMEOUT < ticks) - goto reset; - } - return; - -reset: - priv->port_stats.tx_timeout++; - en_dbg(DRV, priv, "Watchdog task called for port %d\n", priv->port); - - mutex_lock(&mdev->state_lock); - if (priv->port_up) { - mlx4_en_stop_port(dev); - //for (i = 0; i < priv->tx_ring_num; i++) - // netdev_tx_reset_queue(priv->tx_ring[i]->tx_queue); - if (mlx4_en_start_port(dev)) - en_err(priv, "Failed restarting port %d\n", priv->port); - } - mutex_unlock(&mdev->state_lock); -} - -static void mlx4_en_clear_stats(struct net_device *dev) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int i; - - if (!mlx4_is_slave(mdev->dev)) - if (mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 1)) - en_dbg(HW, priv, "Failed dumping statistics\n"); - - memset(&priv->pstats, 0, sizeof(priv->pstats)); - memset(&priv->pkstats, 0, sizeof(priv->pkstats)); - memset(&priv->port_stats, 0, sizeof(priv->port_stats)); - memset(&priv->vport_stats, 0, sizeof(priv->vport_stats)); - - for (i = 0; i < priv->tx_ring_num; i++) { - priv->tx_ring[i]->bytes = 0; - priv->tx_ring[i]->packets = 0; - priv->tx_ring[i]->tx_csum = 0; - priv->tx_ring[i]->oversized_packets = 0; - } - for (i = 0; i < priv->rx_ring_num; i++) { - priv->rx_ring[i]->bytes = 0; - priv->rx_ring[i]->packets = 0; - priv->rx_ring[i]->csum_ok = 0; - priv->rx_ring[i]->csum_none = 0; - } -} - -static void mlx4_en_open(void* arg) -{ - - struct mlx4_en_priv *priv; - struct mlx4_en_dev *mdev; - struct net_device *dev; - int err = 0; - - priv = arg; - mdev = priv->mdev; - dev = priv->dev; - - - mutex_lock(&mdev->state_lock); - - if (!mdev->device_up) { - en_err(priv, "Cannot open - device down/disabled\n"); - goto out; - } - - /* Reset HW statistics and SW counters */ - mlx4_en_clear_stats(dev); - - err = mlx4_en_start_port(dev); - if (err) - en_err(priv, "Failed starting port:%d\n", priv->port); - -out: - mutex_unlock(&mdev->state_lock); - return; -} - -void mlx4_en_free_resources(struct mlx4_en_priv *priv) -{ - int i; - -#ifdef CONFIG_RFS_ACCEL - if (priv->dev->rx_cpu_rmap) { - free_irq_cpu_rmap(priv->dev->rx_cpu_rmap); - priv->dev->rx_cpu_rmap = NULL; - } -#endif - - for (i = 0; i < priv->tx_ring_num; i++) { - if (priv->tx_ring && priv->tx_ring[i]) - mlx4_en_destroy_tx_ring(priv, &priv->tx_ring[i]); - if (priv->tx_cq && priv->tx_cq[i]) - mlx4_en_destroy_cq(priv, &priv->tx_cq[i]); - } - - for (i = 0; i < priv->rx_ring_num; i++) { - if (priv->rx_ring[i]) - mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i], - priv->prof->rx_ring_size, priv->stride); - if (priv->rx_cq[i]) - mlx4_en_destroy_cq(priv, &priv->rx_cq[i]); - } - - if (priv->stat_sysctl != NULL) - sysctl_ctx_free(&priv->stat_ctx); -} - -int mlx4_en_alloc_resources(struct mlx4_en_priv *priv) -{ - struct mlx4_en_port_profile *prof = priv->prof; - int i; - int node = 0; - - /* Create rx Rings */ - for (i = 0; i < priv->rx_ring_num; i++) { - if (mlx4_en_create_cq(priv, &priv->rx_cq[i], - prof->rx_ring_size, i, RX, node)) - goto err; - - if (mlx4_en_create_rx_ring(priv, &priv->rx_ring[i], - prof->rx_ring_size, node)) - goto err; - } - - /* Create tx Rings */ - for (i = 0; i < priv->tx_ring_num; i++) { - if (mlx4_en_create_cq(priv, &priv->tx_cq[i], - prof->tx_ring_size, i, TX, node)) - goto err; - - if (mlx4_en_create_tx_ring(priv, &priv->tx_ring[i], - prof->tx_ring_size, TXBB_SIZE, node, i)) - goto err; - } - -#ifdef CONFIG_RFS_ACCEL - priv->dev->rx_cpu_rmap = alloc_irq_cpu_rmap(priv->rx_ring_num); - if (!priv->dev->rx_cpu_rmap) - goto err; -#endif - /* Re-create stat sysctls in case the number of rings changed. */ - mlx4_en_sysctl_stat(priv); - return 0; - -err: - en_err(priv, "Failed to allocate NIC resources\n"); - for (i = 0; i < priv->rx_ring_num; i++) { - if (priv->rx_ring[i]) - mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i], - prof->rx_ring_size, - priv->stride); - if (priv->rx_cq[i]) - mlx4_en_destroy_cq(priv, &priv->rx_cq[i]); - } - for (i = 0; i < priv->tx_ring_num; i++) { - if (priv->tx_ring[i]) - mlx4_en_destroy_tx_ring(priv, &priv->tx_ring[i]); - if (priv->tx_cq[i]) - mlx4_en_destroy_cq(priv, &priv->tx_cq[i]); - } - priv->port_up = false; - return -ENOMEM; -} - -struct en_port_attribute { - struct attribute attr; - ssize_t (*show)(struct en_port *, struct en_port_attribute *, char *buf); - ssize_t (*store)(struct en_port *, struct en_port_attribute *, char *buf, size_t count); -}; - -#define PORT_ATTR_RO(_name) \ -struct en_port_attribute en_port_attr_##_name = __ATTR_RO(_name) - -#define EN_PORT_ATTR(_name, _mode, _show, _store) \ -struct en_port_attribute en_port_attr_##_name = __ATTR(_name, _mode, _show, _store) - -void mlx4_en_destroy_netdev(struct net_device *dev) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - - en_dbg(DRV, priv, "Destroying netdev on port:%d\n", priv->port); - - if (priv->vlan_attach != NULL) - EVENTHANDLER_DEREGISTER(vlan_config, priv->vlan_attach); - if (priv->vlan_detach != NULL) - EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach); - - /* Unregister device - this will close the port if it was up */ - if (priv->registered) { - mutex_lock(&mdev->state_lock); - ether_ifdetach(dev); - mutex_unlock(&mdev->state_lock); - } - - mutex_lock(&mdev->state_lock); - mlx4_en_stop_port(dev); - mutex_unlock(&mdev->state_lock); - - if (priv->allocated) - mlx4_free_hwq_res(mdev->dev, &priv->res, MLX4_EN_PAGE_SIZE); - - cancel_delayed_work(&priv->stats_task); - cancel_delayed_work(&priv->service_task); - /* flush any pending task for this netdev */ - flush_workqueue(mdev->workqueue); - callout_drain(&priv->watchdog_timer); - - /* Detach the netdev so tasks would not attempt to access it */ - mutex_lock(&mdev->state_lock); - mdev->pndev[priv->port] = NULL; - mutex_unlock(&mdev->state_lock); - - - mlx4_en_free_resources(priv); - - /* freeing the sysctl conf cannot be called from within mlx4_en_free_resources */ - if (priv->conf_sysctl != NULL) - sysctl_ctx_free(&priv->conf_ctx); - - kfree(priv->tx_ring); - kfree(priv->tx_cq); - - kfree(priv); - if_free(dev); - -} - -static int mlx4_en_change_mtu(struct net_device *dev, int new_mtu) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int err = 0; - - en_dbg(DRV, priv, "Change MTU called - current:%u new:%u\n", - (unsigned)dev->if_mtu, (unsigned)new_mtu); - - if ((new_mtu < MLX4_EN_MIN_MTU) || (new_mtu > priv->max_mtu)) { - en_err(priv, "Bad MTU size:%d, max %u.\n", new_mtu, - priv->max_mtu); - return -EPERM; - } - mutex_lock(&mdev->state_lock); - dev->if_mtu = new_mtu; - if (dev->if_drv_flags & IFF_DRV_RUNNING) { - if (!mdev->device_up) { - /* NIC is probably restarting - let watchdog task reset - * * the port */ - en_dbg(DRV, priv, "Change MTU called with card down!?\n"); - } else { - mlx4_en_stop_port(dev); - err = mlx4_en_start_port(dev); - if (err) { - en_err(priv, "Failed restarting port:%d\n", - priv->port); - queue_work(mdev->workqueue, &priv->watchdog_task); - } - } - } - mutex_unlock(&mdev->state_lock); - return 0; -} - -static int mlx4_en_calc_media(struct mlx4_en_priv *priv) -{ - int trans_type; - int active; - - active = IFM_ETHER; - if (priv->last_link_state == MLX4_DEV_EVENT_PORT_DOWN) - return (active); - active |= IFM_FDX; - trans_type = priv->port_state.transciver; - /* XXX I don't know all of the transceiver values. */ - switch (priv->port_state.link_speed) { - case 1000: - active |= IFM_1000_T; - break; - case 10000: - if (trans_type > 0 && trans_type <= 0xC) - active |= IFM_10G_SR; - else if (trans_type == 0x80 || trans_type == 0) - active |= IFM_10G_CX4; - break; - case 40000: - active |= IFM_40G_CR4; - break; - } - if (priv->prof->tx_pause) - active |= IFM_ETH_TXPAUSE; - if (priv->prof->rx_pause) - active |= IFM_ETH_RXPAUSE; - - return (active); -} - -static void mlx4_en_media_status(struct ifnet *dev, struct ifmediareq *ifmr) -{ - struct mlx4_en_priv *priv; - - priv = dev->if_softc; - ifmr->ifm_status = IFM_AVALID; - if (priv->last_link_state != MLX4_DEV_EVENT_PORT_DOWN) - ifmr->ifm_status |= IFM_ACTIVE; - ifmr->ifm_active = mlx4_en_calc_media(priv); - - return; -} - -static int mlx4_en_media_change(struct ifnet *dev) -{ - struct mlx4_en_priv *priv; - struct ifmedia *ifm; - int rxpause; - int txpause; - int error; - - priv = dev->if_softc; - ifm = &priv->media; - rxpause = txpause = 0; - error = 0; - - if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) - return (EINVAL); - switch (IFM_SUBTYPE(ifm->ifm_media)) { - case IFM_AUTO: - break; - case IFM_10G_SR: - case IFM_10G_CX4: - case IFM_1000_T: - case IFM_40G_CR4: - if ((IFM_SUBTYPE(ifm->ifm_media) - == IFM_SUBTYPE(mlx4_en_calc_media(priv))) - && (ifm->ifm_media & IFM_FDX)) - break; - /* Fallthrough */ - default: - printf("%s: Only auto media type\n", if_name(dev)); - return (EINVAL); - } - /* Allow user to set/clear pause */ - if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_RXPAUSE) - rxpause = 1; - if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_TXPAUSE) - txpause = 1; - if (priv->prof->tx_pause != txpause || priv->prof->rx_pause != rxpause) { - priv->prof->tx_pause = txpause; - priv->prof->rx_pause = rxpause; - error = -mlx4_SET_PORT_general(priv->mdev->dev, priv->port, - priv->rx_mb_size + ETHER_CRC_LEN, priv->prof->tx_pause, - priv->prof->tx_ppp, priv->prof->rx_pause, - priv->prof->rx_ppp); - } - return (error); -} - -static int mlx4_en_ioctl(struct ifnet *dev, u_long command, caddr_t data) -{ - struct mlx4_en_priv *priv; - struct mlx4_en_dev *mdev; - struct ifreq *ifr; - int error; - int mask; - - error = 0; - mask = 0; - priv = dev->if_softc; - mdev = priv->mdev; - ifr = (struct ifreq *) data; - switch (command) { - - case SIOCSIFMTU: - error = -mlx4_en_change_mtu(dev, ifr->ifr_mtu); - break; - case SIOCSIFFLAGS: - if (dev->if_flags & IFF_UP) { - if ((dev->if_drv_flags & IFF_DRV_RUNNING) == 0) { - mutex_lock(&mdev->state_lock); - mlx4_en_start_port(dev); - mutex_unlock(&mdev->state_lock); - } else { - mlx4_en_set_rx_mode(dev); - } - } else { - mutex_lock(&mdev->state_lock); - if (dev->if_drv_flags & IFF_DRV_RUNNING) { - mlx4_en_stop_port(dev); - if_link_state_change(dev, LINK_STATE_DOWN); - } - mutex_unlock(&mdev->state_lock); - } - break; - case SIOCADDMULTI: - case SIOCDELMULTI: - mlx4_en_set_rx_mode(dev); - break; - case SIOCSIFMEDIA: - case SIOCGIFMEDIA: - error = ifmedia_ioctl(dev, ifr, &priv->media, command); - break; - case SIOCSIFCAP: - mutex_lock(&mdev->state_lock); - mask = ifr->ifr_reqcap ^ dev->if_capenable; - if (mask & IFCAP_TXCSUM) { - dev->if_capenable ^= IFCAP_TXCSUM; - dev->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); - - if (IFCAP_TSO4 & dev->if_capenable && - !(IFCAP_TXCSUM & dev->if_capenable)) { - dev->if_capenable &= ~IFCAP_TSO4; - dev->if_hwassist &= ~CSUM_IP_TSO; - if_printf(dev, - "tso4 disabled due to -txcsum.\n"); - } - } - if (mask & IFCAP_TXCSUM_IPV6) { - dev->if_capenable ^= IFCAP_TXCSUM_IPV6; - dev->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); - - if (IFCAP_TSO6 & dev->if_capenable && - !(IFCAP_TXCSUM_IPV6 & dev->if_capenable)) { - dev->if_capenable &= ~IFCAP_TSO6; - dev->if_hwassist &= ~CSUM_IP6_TSO; - if_printf(dev, - "tso6 disabled due to -txcsum6.\n"); - } - } - if (mask & IFCAP_RXCSUM) - dev->if_capenable ^= IFCAP_RXCSUM; - if (mask & IFCAP_RXCSUM_IPV6) - dev->if_capenable ^= IFCAP_RXCSUM_IPV6; - - if (mask & IFCAP_TSO4) { - if (!(IFCAP_TSO4 & dev->if_capenable) && - !(IFCAP_TXCSUM & dev->if_capenable)) { - if_printf(dev, "enable txcsum first.\n"); - error = EAGAIN; - goto out; - } - dev->if_capenable ^= IFCAP_TSO4; - dev->if_hwassist ^= CSUM_IP_TSO; - } - if (mask & IFCAP_TSO6) { - if (!(IFCAP_TSO6 & dev->if_capenable) && - !(IFCAP_TXCSUM_IPV6 & dev->if_capenable)) { - if_printf(dev, "enable txcsum6 first.\n"); - error = EAGAIN; - goto out; - } - dev->if_capenable ^= IFCAP_TSO6; - dev->if_hwassist ^= CSUM_IP6_TSO; - } - if (mask & IFCAP_LRO) - dev->if_capenable ^= IFCAP_LRO; - if (mask & IFCAP_VLAN_HWTAGGING) - dev->if_capenable ^= IFCAP_VLAN_HWTAGGING; - if (mask & IFCAP_VLAN_HWFILTER) - dev->if_capenable ^= IFCAP_VLAN_HWFILTER; - if (mask & IFCAP_WOL_MAGIC) - dev->if_capenable ^= IFCAP_WOL_MAGIC; - if (dev->if_drv_flags & IFF_DRV_RUNNING) - mlx4_en_start_port(dev); -out: - mutex_unlock(&mdev->state_lock); - VLAN_CAPABILITIES(dev); - break; -#if __FreeBSD_version >= 1100036 - case SIOCGI2C: { - struct ifi2creq i2c; - - error = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); - if (error) - break; - if (i2c.len > sizeof(i2c.data)) { - error = EINVAL; - break; - } - /* - * Note that we ignore i2c.addr here. The driver hardcodes - * the address to 0x50, while standard expects it to be 0xA0. - */ - error = mlx4_get_module_info(mdev->dev, priv->port, - i2c.offset, i2c.len, i2c.data); - if (error < 0) { - error = -error; - break; - } - error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); - break; - } -#endif - default: - error = ether_ioctl(dev, command, data); - break; - } - - return (error); -} - - -int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, - struct mlx4_en_port_profile *prof) -{ - struct net_device *dev; - struct mlx4_en_priv *priv; - uint8_t dev_addr[ETHER_ADDR_LEN]; - int err; - int i; - - priv = kzalloc(sizeof(*priv), GFP_KERNEL); - dev = priv->dev = if_alloc(IFT_ETHER); - if (dev == NULL) { - en_err(priv, "Net device allocation failed\n"); - kfree(priv); - return -ENOMEM; - } - dev->if_softc = priv; - if_initname(dev, "mlxen", (device_get_unit( - mdev->pdev->dev.bsddev) * MLX4_MAX_PORTS) + port - 1); - dev->if_mtu = ETHERMTU; - dev->if_init = mlx4_en_open; - dev->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; - dev->if_ioctl = mlx4_en_ioctl; - dev->if_transmit = mlx4_en_transmit; - dev->if_qflush = mlx4_en_qflush; - dev->if_snd.ifq_maxlen = prof->tx_ring_size; - - /* - * Initialize driver private data - */ - priv->counter_index = 0xff; - spin_lock_init(&priv->stats_lock); - INIT_WORK(&priv->rx_mode_task, mlx4_en_do_set_rx_mode); - INIT_WORK(&priv->watchdog_task, mlx4_en_restart); - INIT_WORK(&priv->linkstate_task, mlx4_en_linkstate); - INIT_DELAYED_WORK(&priv->stats_task, mlx4_en_do_get_stats); - INIT_DELAYED_WORK(&priv->service_task, mlx4_en_service_task); - callout_init(&priv->watchdog_timer, 1); -#ifdef CONFIG_RFS_ACCEL - INIT_LIST_HEAD(&priv->filters); - spin_lock_init(&priv->filters_lock); -#endif - - priv->msg_enable = MLX4_EN_MSG_LEVEL; - priv->dev = dev; - priv->mdev = mdev; - priv->ddev = &mdev->pdev->dev; - priv->prof = prof; - priv->port = port; - priv->port_up = false; - priv->flags = prof->flags; - - priv->num_tx_rings_p_up = mdev->profile.num_tx_rings_p_up; - priv->tx_ring_num = prof->tx_ring_num; - priv->tx_ring = kcalloc(MAX_TX_RINGS, - sizeof(struct mlx4_en_tx_ring *), GFP_KERNEL); - if (!priv->tx_ring) { - err = -ENOMEM; - goto out; - } - priv->tx_cq = kcalloc(sizeof(struct mlx4_en_cq *), MAX_TX_RINGS, - GFP_KERNEL); - if (!priv->tx_cq) { - err = -ENOMEM; - goto out; - } - - priv->rx_ring_num = prof->rx_ring_num; - priv->cqe_factor = (mdev->dev->caps.cqe_size == 64) ? 1 : 0; - priv->mac_index = -1; - priv->last_ifq_jiffies = 0; - priv->if_counters_rx_errors = 0; - priv->if_counters_rx_no_buffer = 0; -#ifdef CONFIG_MLX4_EN_DCB - if (!mlx4_is_slave(priv->mdev->dev)) { - priv->dcbx_cap = DCB_CAP_DCBX_HOST; - priv->flags |= MLX4_EN_FLAG_DCB_ENABLED; - if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETS_CFG) { - dev->dcbnl_ops = &mlx4_en_dcbnl_ops; - } else { - en_info(priv, "QoS disabled - no HW support\n"); - dev->dcbnl_ops = &mlx4_en_dcbnl_pfc_ops; - } - } -#endif - - for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) - INIT_HLIST_HEAD(&priv->mac_hash[i]); - - /* Query for default mac and max mtu */ - priv->max_mtu = mdev->dev->caps.eth_mtu_cap[priv->port]; - priv->mac = mdev->dev->caps.def_mac[priv->port]; - if (ILLEGAL_MAC(priv->mac)) { -#if BITS_PER_LONG == 64 - en_err(priv, "Port: %d, invalid mac burned: 0x%lx, quiting\n", - priv->port, priv->mac); -#elif BITS_PER_LONG == 32 - en_err(priv, "Port: %d, invalid mac burned: 0x%llx, quiting\n", - priv->port, priv->mac); -#endif - err = -EINVAL; - goto out; - } - - priv->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + - DS_SIZE); - - mlx4_en_sysctl_conf(priv); - - err = mlx4_en_alloc_resources(priv); - if (err) - goto out; - - /* Allocate page for receive rings */ - err = mlx4_alloc_hwq_res(mdev->dev, &priv->res, - MLX4_EN_PAGE_SIZE, MLX4_EN_PAGE_SIZE); - if (err) { - en_err(priv, "Failed to allocate page for rx qps\n"); - goto out; - } - priv->allocated = 1; - - /* - * Set driver features - */ - dev->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6; - dev->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING; - dev->if_capabilities |= IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER; - dev->if_capabilities |= IFCAP_LINKSTATE | IFCAP_JUMBO_MTU; - dev->if_capabilities |= IFCAP_LRO; - dev->if_capabilities |= IFCAP_HWSTATS; - - if (mdev->LSO_support) - dev->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6 | IFCAP_VLAN_HWTSO; - -#if __FreeBSD_version >= 1100000 - /* set TSO limits so that we don't have to drop TX packets */ - dev->if_hw_tsomax = MLX4_EN_TX_MAX_PAYLOAD_SIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN) /* hdr */; - dev->if_hw_tsomaxsegcount = MLX4_EN_TX_MAX_MBUF_FRAGS - 1 /* hdr */; - dev->if_hw_tsomaxsegsize = MLX4_EN_TX_MAX_MBUF_SIZE; -#endif - - dev->if_capenable = dev->if_capabilities; - - dev->if_hwassist = 0; - if (dev->if_capenable & (IFCAP_TSO4 | IFCAP_TSO6)) - dev->if_hwassist |= CSUM_TSO; - if (dev->if_capenable & IFCAP_TXCSUM) - dev->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP); - if (dev->if_capenable & IFCAP_TXCSUM_IPV6) - dev->if_hwassist |= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); - - - /* Register for VLAN events */ - priv->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, - mlx4_en_vlan_rx_add_vid, priv, EVENTHANDLER_PRI_FIRST); - priv->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, - mlx4_en_vlan_rx_kill_vid, priv, EVENTHANDLER_PRI_FIRST); - - mdev->pndev[priv->port] = dev; - - priv->last_link_state = MLX4_DEV_EVENT_PORT_DOWN; - mlx4_en_set_default_moderation(priv); - - /* Set default MAC */ - for (i = 0; i < ETHER_ADDR_LEN; i++) - dev_addr[ETHER_ADDR_LEN - 1 - i] = (u8) (priv->mac >> (8 * i)); - - - ether_ifattach(dev, dev_addr); - if_link_state_change(dev, LINK_STATE_DOWN); - ifmedia_init(&priv->media, IFM_IMASK | IFM_ETH_FMASK, - mlx4_en_media_change, mlx4_en_media_status); - ifmedia_add(&priv->media, IFM_ETHER | IFM_FDX | IFM_1000_T, 0, NULL); - ifmedia_add(&priv->media, IFM_ETHER | IFM_FDX | IFM_10G_SR, 0, NULL); - ifmedia_add(&priv->media, IFM_ETHER | IFM_FDX | IFM_10G_CX4, 0, NULL); - ifmedia_add(&priv->media, IFM_ETHER | IFM_FDX | IFM_40G_CR4, 0, NULL); - ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL); - ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO); - - en_warn(priv, "Using %d TX rings\n", prof->tx_ring_num); - en_warn(priv, "Using %d RX rings\n", prof->rx_ring_num); - - priv->registered = 1; - - en_warn(priv, "Using %d TX rings\n", prof->tx_ring_num); - en_warn(priv, "Using %d RX rings\n", prof->rx_ring_num); - - - priv->rx_mb_size = dev->if_mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN; - err = mlx4_SET_PORT_general(mdev->dev, priv->port, - priv->rx_mb_size, - prof->tx_pause, prof->tx_ppp, - prof->rx_pause, prof->rx_ppp); - if (err) { - en_err(priv, "Failed setting port general configurations " - "for port %d, with error %d\n", priv->port, err); - goto out; - } - - /* Init port */ - en_warn(priv, "Initializing port\n"); - err = mlx4_INIT_PORT(mdev->dev, priv->port); - if (err) { - en_err(priv, "Failed Initializing port\n"); - goto out; - } - - queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY); - - if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) - queue_delayed_work(mdev->workqueue, &priv->service_task, SERVICE_TASK_DELAY); - - return 0; - -out: - mlx4_en_destroy_netdev(dev); - return err; -} - -static int mlx4_en_set_ring_size(struct net_device *dev, - int rx_size, int tx_size) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int port_up = 0; - int err = 0; - - rx_size = roundup_pow_of_two(rx_size); - rx_size = max_t(u32, rx_size, MLX4_EN_MIN_RX_SIZE); - rx_size = min_t(u32, rx_size, MLX4_EN_MAX_RX_SIZE); - tx_size = roundup_pow_of_two(tx_size); - tx_size = max_t(u32, tx_size, MLX4_EN_MIN_TX_SIZE); - tx_size = min_t(u32, tx_size, MLX4_EN_MAX_TX_SIZE); - - if (rx_size == (priv->port_up ? - priv->rx_ring[0]->actual_size : priv->rx_ring[0]->size) && - tx_size == priv->tx_ring[0]->size) - return 0; - mutex_lock(&mdev->state_lock); - if (priv->port_up) { - port_up = 1; - mlx4_en_stop_port(dev); - } - mlx4_en_free_resources(priv); - priv->prof->tx_ring_size = tx_size; - priv->prof->rx_ring_size = rx_size; - err = mlx4_en_alloc_resources(priv); - if (err) { - en_err(priv, "Failed reallocating port resources\n"); - goto out; - } - if (port_up) { - err = mlx4_en_start_port(dev); - if (err) - en_err(priv, "Failed starting port\n"); - } -out: - mutex_unlock(&mdev->state_lock); - return err; -} -static int mlx4_en_set_rx_ring_size(SYSCTL_HANDLER_ARGS) -{ - struct mlx4_en_priv *priv; - int size; - int error; - - priv = arg1; - size = priv->prof->rx_ring_size; - error = sysctl_handle_int(oidp, &size, 0, req); - if (error || !req->newptr) - return (error); - error = -mlx4_en_set_ring_size(priv->dev, size, - priv->prof->tx_ring_size); - return (error); -} - -static int mlx4_en_set_tx_ring_size(SYSCTL_HANDLER_ARGS) -{ - struct mlx4_en_priv *priv; - int size; - int error; - - priv = arg1; - size = priv->prof->tx_ring_size; - error = sysctl_handle_int(oidp, &size, 0, req); - if (error || !req->newptr) - return (error); - error = -mlx4_en_set_ring_size(priv->dev, priv->prof->rx_ring_size, - size); - - return (error); -} - -static int mlx4_en_get_module_info(struct net_device *dev, - struct ethtool_modinfo *modinfo) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int ret; - u8 data[4]; - - /* Read first 2 bytes to get Module & REV ID */ - ret = mlx4_get_module_info(mdev->dev, priv->port, - 0/*offset*/, 2/*size*/, data); - - if (ret < 2) { - en_err(priv, "Failed to read eeprom module first two bytes, error: 0x%x\n", -ret); - return -EIO; - } - - switch (data[0] /* identifier */) { - case MLX4_MODULE_ID_QSFP: - modinfo->type = ETH_MODULE_SFF_8436; - modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN; - break; - case MLX4_MODULE_ID_QSFP_PLUS: - if (data[1] >= 0x3) { /* revision id */ - modinfo->type = ETH_MODULE_SFF_8636; - modinfo->eeprom_len = ETH_MODULE_SFF_8636_LEN; - } else { - modinfo->type = ETH_MODULE_SFF_8436; - modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN; - } - break; - case MLX4_MODULE_ID_QSFP28: - modinfo->type = ETH_MODULE_SFF_8636; - modinfo->eeprom_len = ETH_MODULE_SFF_8636_LEN; - break; - case MLX4_MODULE_ID_SFP: - modinfo->type = ETH_MODULE_SFF_8472; - modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN; - break; - default: - en_err(priv, "mlx4_en_get_module_info : Not recognized cable type\n"); - return -EINVAL; - } - - return 0; -} - -static int mlx4_en_get_module_eeprom(struct net_device *dev, - struct ethtool_eeprom *ee, - u8 *data) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int offset = ee->offset; - int i = 0, ret; - - if (ee->len == 0) - return -EINVAL; - - memset(data, 0, ee->len); - - while (i < ee->len) { - en_dbg(DRV, priv, - "mlx4_get_module_info i(%d) offset(%d) len(%d)\n", - i, offset, ee->len - i); - - ret = mlx4_get_module_info(mdev->dev, priv->port, - offset, ee->len - i, data + i); - - if (!ret) /* Done reading */ - return 0; - - if (ret < 0) { - en_err(priv, - "mlx4_get_module_info i(%d) offset(%d) bytes_to_read(%d) - FAILED (0x%x)\n", - i, offset, ee->len - i, ret); - return -1; - } - - i += ret; - offset += ret; - } - return 0; -} - -static void mlx4_en_print_eeprom(u8 *data, __u32 len) -{ - int i; - int j = 0; - int row = 0; - const int NUM_OF_BYTES = 16; - - printf("\nOffset\t\tValues\n"); - printf("------\t\t------\n"); - while(row < len){ - printf("0x%04x\t\t",row); - for(i=0; i < NUM_OF_BYTES; i++){ - printf("%02x ", data[j]); - row++; - j++; - } - printf("\n"); - } -} - -/* Read cable EEPROM module information by first inspecting the first - * two bytes to get the length and then read the rest of the information. - * The information is printed to dmesg. */ -static int mlx4_en_read_eeprom(SYSCTL_HANDLER_ARGS) -{ - - u8* data; - int error; - int result = 0; - struct mlx4_en_priv *priv; - struct net_device *dev; - struct ethtool_modinfo modinfo; - struct ethtool_eeprom ee; - - error = sysctl_handle_int(oidp, &result, 0, req); - if (error || !req->newptr) - return (error); - - if (result == 1) { - priv = arg1; - dev = priv->dev; - data = kmalloc(PAGE_SIZE, GFP_KERNEL); - - error = mlx4_en_get_module_info(dev, &modinfo); - if (error) { - en_err(priv, - "mlx4_en_get_module_info returned with error - FAILED (0x%x)\n", - -error); - goto out; - } - - ee.len = modinfo.eeprom_len; - ee.offset = 0; - - error = mlx4_en_get_module_eeprom(dev, &ee, data); - if (error) { - en_err(priv, - "mlx4_en_get_module_eeprom returned with error - FAILED (0x%x)\n", - -error); - /* Continue printing partial information in case of an error */ - } - - /* EEPROM information will be printed in dmesg */ - mlx4_en_print_eeprom(data, ee.len); -out: - kfree(data); - } - /* Return zero to prevent sysctl failure. */ - return (0); -} - -static int mlx4_en_set_tx_ppp(SYSCTL_HANDLER_ARGS) -{ - struct mlx4_en_priv *priv; - int ppp; - int error; - - priv = arg1; - ppp = priv->prof->tx_ppp; - error = sysctl_handle_int(oidp, &ppp, 0, req); - if (error || !req->newptr) - return (error); - if (ppp > 0xff || ppp < 0) - return (-EINVAL); - priv->prof->tx_ppp = ppp; - error = -mlx4_SET_PORT_general(priv->mdev->dev, priv->port, - priv->rx_mb_size + ETHER_CRC_LEN, - priv->prof->tx_pause, - priv->prof->tx_ppp, - priv->prof->rx_pause, - priv->prof->rx_ppp); - - return (error); -} - -static int mlx4_en_set_rx_ppp(SYSCTL_HANDLER_ARGS) -{ - struct mlx4_en_priv *priv; - struct mlx4_en_dev *mdev; - int ppp; - int error; - int port_up; - - port_up = 0; - priv = arg1; - mdev = priv->mdev; - ppp = priv->prof->rx_ppp; - error = sysctl_handle_int(oidp, &ppp, 0, req); - if (error || !req->newptr) - return (error); - if (ppp > 0xff || ppp < 0) - return (-EINVAL); - /* See if we have to change the number of tx queues. */ - if (!ppp != !priv->prof->rx_ppp) { - mutex_lock(&mdev->state_lock); - if (priv->port_up) { - port_up = 1; - mlx4_en_stop_port(priv->dev); - } - mlx4_en_free_resources(priv); - priv->prof->rx_ppp = ppp; - error = -mlx4_en_alloc_resources(priv); - if (error) - en_err(priv, "Failed reallocating port resources\n"); - if (error == 0 && port_up) { - error = -mlx4_en_start_port(priv->dev); - if (error) - en_err(priv, "Failed starting port\n"); - } - mutex_unlock(&mdev->state_lock); - return (error); - - } - priv->prof->rx_ppp = ppp; - error = -mlx4_SET_PORT_general(priv->mdev->dev, priv->port, - priv->rx_mb_size + ETHER_CRC_LEN, - priv->prof->tx_pause, - priv->prof->tx_ppp, - priv->prof->rx_pause, - priv->prof->rx_ppp); - - return (error); -} - -static void mlx4_en_sysctl_conf(struct mlx4_en_priv *priv) -{ - struct net_device *dev; - struct sysctl_ctx_list *ctx; - struct sysctl_oid *node; - struct sysctl_oid_list *node_list; - struct sysctl_oid *coal; - struct sysctl_oid_list *coal_list; - const char *pnameunit; - - dev = priv->dev; - ctx = &priv->conf_ctx; - pnameunit = device_get_nameunit(priv->mdev->pdev->dev.bsddev); - - sysctl_ctx_init(ctx); - priv->conf_sysctl = SYSCTL_ADD_NODE(ctx, SYSCTL_STATIC_CHILDREN(_hw), - OID_AUTO, dev->if_xname, CTLFLAG_RD, 0, "mlx4 10gig ethernet"); - node = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(priv->conf_sysctl), OID_AUTO, - "conf", CTLFLAG_RD, NULL, "Configuration"); - node_list = SYSCTL_CHILDREN(node); - - SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "msg_enable", - CTLFLAG_RW, &priv->msg_enable, 0, - "Driver message enable bitfield"); - SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "rx_rings", - CTLFLAG_RD, &priv->rx_ring_num, 0, - "Number of receive rings"); - SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "tx_rings", - CTLFLAG_RD, &priv->tx_ring_num, 0, - "Number of transmit rings"); - SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "rx_size", - CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, - mlx4_en_set_rx_ring_size, "I", "Receive ring size"); - SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "tx_size", - CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, - mlx4_en_set_tx_ring_size, "I", "Transmit ring size"); - SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "tx_ppp", - CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, - mlx4_en_set_tx_ppp, "I", "TX Per-priority pause"); - SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "rx_ppp", - CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, - mlx4_en_set_rx_ppp, "I", "RX Per-priority pause"); - SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "port_num", - CTLFLAG_RD, &priv->port, 0, - "Port Number"); - SYSCTL_ADD_STRING(ctx, node_list, OID_AUTO, "device_name", - CTLFLAG_RD, __DECONST(void *, pnameunit), 0, - "PCI device name"); - - /* Add coalescer configuration. */ - coal = SYSCTL_ADD_NODE(ctx, node_list, OID_AUTO, - "coalesce", CTLFLAG_RD, NULL, "Interrupt coalesce configuration"); - coal_list = SYSCTL_CHILDREN(coal); - SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "pkt_rate_low", - CTLFLAG_RW, &priv->pkt_rate_low, 0, - "Packets per-second for minimum delay"); - SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "rx_usecs_low", - CTLFLAG_RW, &priv->rx_usecs_low, 0, - "Minimum RX delay in micro-seconds"); - SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "pkt_rate_high", - CTLFLAG_RW, &priv->pkt_rate_high, 0, - "Packets per-second for maximum delay"); - SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "rx_usecs_high", - CTLFLAG_RW, &priv->rx_usecs_high, 0, - "Maximum RX delay in micro-seconds"); - SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "sample_interval", - CTLFLAG_RW, &priv->sample_interval, 0, - "adaptive frequency in units of HZ ticks"); - SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "adaptive_rx_coal", - CTLFLAG_RW, &priv->adaptive_rx_coal, 0, - "Enable adaptive rx coalescing"); - /* EEPROM support */ - SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "eeprom_info", - CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, - mlx4_en_read_eeprom, "I", "EEPROM information"); -} - -static void mlx4_en_sysctl_stat(struct mlx4_en_priv *priv) -{ - struct sysctl_ctx_list *ctx; - struct sysctl_oid_list *node_list; - struct sysctl_oid *ring_node; - struct sysctl_oid_list *ring_list; - struct mlx4_en_tx_ring *tx_ring; - struct mlx4_en_rx_ring *rx_ring; - char namebuf[128]; - int i; - - ctx = &priv->stat_ctx; - sysctl_ctx_init(ctx); - priv->stat_sysctl = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(priv->conf_sysctl), OID_AUTO, - "stat", CTLFLAG_RD, NULL, "Statistics"); - node_list = SYSCTL_CHILDREN(priv->stat_sysctl); - -#ifdef MLX4_EN_PERF_STAT - SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "tx_poll", CTLFLAG_RD, - &priv->pstats.tx_poll, "TX Poll calls"); - SYSCTL_ADD_QUAD(ctx, node_list, OID_AUTO, "tx_pktsz_avg", CTLFLAG_RD, - &priv->pstats.tx_pktsz_avg, "TX average packet size"); - SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "inflight_avg", CTLFLAG_RD, - &priv->pstats.inflight_avg, "TX average packets in-flight"); - SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "tx_coal_avg", CTLFLAG_RD, - &priv->pstats.tx_coal_avg, "TX average coalesced completions"); - SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "rx_coal_avg", CTLFLAG_RD, - &priv->pstats.rx_coal_avg, "RX average coalesced completions"); -#endif - - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tso_packets", CTLFLAG_RD, - &priv->port_stats.tso_packets, "TSO packets sent"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "queue_stopped", CTLFLAG_RD, - &priv->port_stats.queue_stopped, "Queue full"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "wake_queue", CTLFLAG_RD, - &priv->port_stats.wake_queue, "Queue resumed after full"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_timeout", CTLFLAG_RD, - &priv->port_stats.tx_timeout, "Transmit timeouts"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_oversized_packets", CTLFLAG_RD, - &priv->port_stats.oversized_packets, "TX oversized packets, m_defrag failed"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_alloc_failed", CTLFLAG_RD, - &priv->port_stats.rx_alloc_failed, "RX failed to allocate mbuf"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_chksum_good", CTLFLAG_RD, - &priv->port_stats.rx_chksum_good, "RX checksum offload success"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_chksum_none", CTLFLAG_RD, - &priv->port_stats.rx_chksum_none, "RX without checksum offload"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_chksum_offload", - CTLFLAG_RD, &priv->port_stats.tx_chksum_offload, - "TX checksum offloads"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "defrag_attempts", CTLFLAG_RD, - &priv->port_stats.defrag_attempts, "Oversized chains defragged"); - - /* Could strdup the names and add in a loop. This is simpler. */ - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_bytes", CTLFLAG_RD, - &priv->pkstats.rx_bytes, "RX Bytes"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_packets", CTLFLAG_RD, - &priv->pkstats.rx_packets, "RX packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_multicast_packets", CTLFLAG_RD, - &priv->pkstats.rx_multicast_packets, "RX Multicast Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_broadcast_packets", CTLFLAG_RD, - &priv->pkstats.rx_broadcast_packets, "RX Broadcast Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_errors", CTLFLAG_RD, - &priv->pkstats.rx_errors, "RX Errors"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_dropped", CTLFLAG_RD, - &priv->pkstats.rx_dropped, "RX Dropped"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_length_errors", CTLFLAG_RD, - &priv->pkstats.rx_length_errors, "RX Length Errors"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_over_errors", CTLFLAG_RD, - &priv->pkstats.rx_over_errors, "RX Over Errors"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_crc_errors", CTLFLAG_RD, - &priv->pkstats.rx_crc_errors, "RX CRC Errors"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_jabbers", CTLFLAG_RD, - &priv->pkstats.rx_jabbers, "RX Jabbers"); - - - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_in_range_length_error", CTLFLAG_RD, - &priv->pkstats.rx_in_range_length_error, "RX IN_Range Length Error"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_out_range_length_error", - CTLFLAG_RD, &priv->pkstats.rx_out_range_length_error, - "RX Out Range Length Error"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_lt_64_bytes_packets", CTLFLAG_RD, - &priv->pkstats.rx_lt_64_bytes_packets, "RX Lt 64 Bytes Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_127_bytes_packets", CTLFLAG_RD, - &priv->pkstats.rx_127_bytes_packets, "RX 127 bytes Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_255_bytes_packets", CTLFLAG_RD, - &priv->pkstats.rx_255_bytes_packets, "RX 255 bytes Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_511_bytes_packets", CTLFLAG_RD, - &priv->pkstats.rx_511_bytes_packets, "RX 511 bytes Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_1023_bytes_packets", CTLFLAG_RD, - &priv->pkstats.rx_1023_bytes_packets, "RX 1023 bytes Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_1518_bytes_packets", CTLFLAG_RD, - &priv->pkstats.rx_1518_bytes_packets, "RX 1518 bytes Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_1522_bytes_packets", CTLFLAG_RD, - &priv->pkstats.rx_1522_bytes_packets, "RX 1522 bytes Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_1548_bytes_packets", CTLFLAG_RD, - &priv->pkstats.rx_1548_bytes_packets, "RX 1548 bytes Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_gt_1548_bytes_packets", CTLFLAG_RD, - &priv->pkstats.rx_gt_1548_bytes_packets, - "RX Greater Then 1548 bytes Packets"); - - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_packets", CTLFLAG_RD, - &priv->pkstats.tx_packets, "TX packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_bytes", CTLFLAG_RD, - &priv->pkstats.tx_bytes, "TX Bytes"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_multicast_packets", CTLFLAG_RD, - &priv->pkstats.tx_multicast_packets, "TX Multicast Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_broadcast_packets", CTLFLAG_RD, - &priv->pkstats.tx_broadcast_packets, "TX Broadcast Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_errors", CTLFLAG_RD, - &priv->pkstats.tx_errors, "TX Errors"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_dropped", CTLFLAG_RD, - &priv->pkstats.tx_dropped, "TX Dropped"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_lt_64_bytes_packets", CTLFLAG_RD, - &priv->pkstats.tx_lt_64_bytes_packets, "TX Less Then 64 Bytes Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_127_bytes_packets", CTLFLAG_RD, - &priv->pkstats.tx_127_bytes_packets, "TX 127 Bytes Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_255_bytes_packets", CTLFLAG_RD, - &priv->pkstats.tx_255_bytes_packets, "TX 255 Bytes Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_511_bytes_packets", CTLFLAG_RD, - &priv->pkstats.tx_511_bytes_packets, "TX 511 Bytes Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_1023_bytes_packets", CTLFLAG_RD, - &priv->pkstats.tx_1023_bytes_packets, "TX 1023 Bytes Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_1518_bytes_packets", CTLFLAG_RD, - &priv->pkstats.tx_1518_bytes_packets, "TX 1518 Bytes Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_1522_bytes_packets", CTLFLAG_RD, - &priv->pkstats.tx_1522_bytes_packets, "TX 1522 Bytes Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_1548_bytes_packets", CTLFLAG_RD, - &priv->pkstats.tx_1548_bytes_packets, "TX 1548 Bytes Packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_gt_1548_bytes_packets", CTLFLAG_RD, - &priv->pkstats.tx_gt_1548_bytes_packets, - "TX Greater Then 1548 Bytes Packets"); - - - - for (i = 0; i < priv->tx_ring_num; i++) { - tx_ring = priv->tx_ring[i]; - snprintf(namebuf, sizeof(namebuf), "tx_ring%d", i); - ring_node = SYSCTL_ADD_NODE(ctx, node_list, OID_AUTO, namebuf, - CTLFLAG_RD, NULL, "TX Ring"); - ring_list = SYSCTL_CHILDREN(ring_node); - SYSCTL_ADD_ULONG(ctx, ring_list, OID_AUTO, "packets", - CTLFLAG_RD, &tx_ring->packets, "TX packets"); - SYSCTL_ADD_ULONG(ctx, ring_list, OID_AUTO, "bytes", - CTLFLAG_RD, &tx_ring->bytes, "TX bytes"); - SYSCTL_ADD_ULONG(ctx, ring_list, OID_AUTO, "tso_packets", - CTLFLAG_RD, &tx_ring->tso_packets, "TSO packets"); - SYSCTL_ADD_ULONG(ctx, ring_list, OID_AUTO, "defrag_attempts", - CTLFLAG_RD, &tx_ring->defrag_attempts, "Oversized chains defragged"); - } - - for (i = 0; i < priv->rx_ring_num; i++) { - rx_ring = priv->rx_ring[i]; - snprintf(namebuf, sizeof(namebuf), "rx_ring%d", i); - ring_node = SYSCTL_ADD_NODE(ctx, node_list, OID_AUTO, namebuf, - CTLFLAG_RD, NULL, "RX Ring"); - ring_list = SYSCTL_CHILDREN(ring_node); - SYSCTL_ADD_ULONG(ctx, ring_list, OID_AUTO, "packets", - CTLFLAG_RD, &rx_ring->packets, "RX packets"); - SYSCTL_ADD_ULONG(ctx, ring_list, OID_AUTO, "bytes", - CTLFLAG_RD, &rx_ring->bytes, "RX bytes"); - SYSCTL_ADD_ULONG(ctx, ring_list, OID_AUTO, "error", - CTLFLAG_RD, &rx_ring->errors, "RX soft errors"); - } -} Property changes on: stable/11/sys/ofed/drivers/net/mlx4/en_netdev.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/en_cq.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/en_cq.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/en_cq.c (nonexistent) @@ -1,214 +0,0 @@ -/* - * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include -#include -#include - -#include "mlx4_en.h" - - -static void mlx4_en_cq_event(struct mlx4_cq *cq, enum mlx4_event event) -{ - return; -} - - -int mlx4_en_create_cq(struct mlx4_en_priv *priv, - struct mlx4_en_cq **pcq, - int entries, int ring, enum cq_type mode, - int node) -{ - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_cq *cq; - int err; - - cq = kzalloc_node(sizeof(struct mlx4_en_cq), GFP_KERNEL, node); - if (!cq) { - cq = kzalloc(sizeof(struct mlx4_en_cq), GFP_KERNEL); - if (!cq) { - en_err(priv, "Failed to allocate CW struture\n"); - return -ENOMEM; - } - } - - cq->size = entries; - cq->buf_size = cq->size * mdev->dev->caps.cqe_size; - - cq->tq = taskqueue_create_fast("mlx4_en_que", M_NOWAIT, - taskqueue_thread_enqueue, &cq->tq); - if (mode == RX) { - TASK_INIT(&cq->cq_task, 0, mlx4_en_rx_que, cq); - taskqueue_start_threads(&cq->tq, 1, PI_NET, "%s rx cq", - if_name(priv->dev)); - - } else { - TASK_INIT(&cq->cq_task, 0, mlx4_en_tx_que, cq); - taskqueue_start_threads(&cq->tq, 1, PI_NET, "%s tx cq", - if_name(priv->dev)); - } - - cq->ring = ring; - cq->is_tx = mode; - spin_lock_init(&cq->lock); - - err = mlx4_alloc_hwq_res(mdev->dev, &cq->wqres, - cq->buf_size, 2 * PAGE_SIZE); - if (err) - goto err_cq; - - err = mlx4_en_map_buffer(&cq->wqres.buf); - if (err) - goto err_res; - - cq->buf = (struct mlx4_cqe *) cq->wqres.buf.direct.buf; - *pcq = cq; - - return 0; - -err_res: - mlx4_free_hwq_res(mdev->dev, &cq->wqres, cq->buf_size); -err_cq: - kfree(cq); - return err; -} - - -int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq, - int cq_idx) -{ - struct mlx4_en_dev *mdev = priv->mdev; - int err = 0; - char name[25]; - int timestamp_en = 0; - - cq->dev = mdev->pndev[priv->port]; - cq->mcq.set_ci_db = cq->wqres.db.db; - cq->mcq.arm_db = cq->wqres.db.db + 1; - *cq->mcq.set_ci_db = 0; - *cq->mcq.arm_db = 0; - memset(cq->buf, 0, cq->buf_size); - - if (cq->is_tx == RX) { - if (mdev->dev->caps.comp_pool) { - if (!cq->vector) { - sprintf(name, "%s-%d", if_name(priv->dev), - cq->ring); - /* Set IRQ for specific name (per ring) */ - if (mlx4_assign_eq(mdev->dev, name, &cq->vector)) { - cq->vector = (cq->ring + 1 + priv->port) - % mdev->dev->caps.num_comp_vectors; - mlx4_warn(mdev, "Failed Assigning an EQ to " - "%s ,Falling back to legacy EQ's\n", - name); - } - } - } else { - cq->vector = (cq->ring + 1 + priv->port) % - mdev->dev->caps.num_comp_vectors; - } - } else { - struct mlx4_en_cq *rx_cq; - /* - * For TX we use the same irq per - * ring we assigned for the RX - */ - cq_idx = cq_idx % priv->rx_ring_num; - rx_cq = priv->rx_cq[cq_idx]; - cq->vector = rx_cq->vector; - } - - if (!cq->is_tx) - cq->size = priv->rx_ring[cq->ring]->actual_size; - err = mlx4_cq_alloc(mdev->dev, cq->size, &cq->wqres.mtt, - &mdev->priv_uar, cq->wqres.db.dma, &cq->mcq, - cq->vector, 0, timestamp_en); - if (err) - return err; - - cq->mcq.comp = cq->is_tx ? mlx4_en_tx_irq : mlx4_en_rx_irq; - cq->mcq.event = mlx4_en_cq_event; - - if (cq->is_tx) { - init_timer(&cq->timer); - cq->timer.function = mlx4_en_poll_tx_cq; - cq->timer.data = (unsigned long) cq; - } - - - return 0; -} - -void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq) -{ - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_cq *cq = *pcq; - - taskqueue_drain(cq->tq, &cq->cq_task); - taskqueue_free(cq->tq); - mlx4_en_unmap_buffer(&cq->wqres.buf); - mlx4_free_hwq_res(mdev->dev, &cq->wqres, cq->buf_size); - if (priv->mdev->dev->caps.comp_pool && cq->vector) - mlx4_release_eq(priv->mdev->dev, cq->vector); - kfree(cq); - *pcq = NULL; -} - -void mlx4_en_deactivate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq) -{ - struct mlx4_en_dev *mdev = priv->mdev; - - taskqueue_drain(cq->tq, &cq->cq_task); - if (cq->is_tx) - del_timer_sync(&cq->timer); - - mlx4_cq_free(mdev->dev, &cq->mcq); -} - - -/* Set rx cq moderation parameters */ -int mlx4_en_set_cq_moder(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq) -{ - return mlx4_cq_modify(priv->mdev->dev, &cq->mcq, - cq->moder_cnt, cq->moder_time); -} - -int mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq) -{ - mlx4_cq_arm(&cq->mcq, MLX4_CQ_DB_REQ_NOT, priv->mdev->uar_map, - &priv->mdev->uar_lock); - - return 0; -} - - Property changes on: stable/11/sys/ofed/drivers/net/mlx4/en_cq.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/cmd.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/cmd.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/cmd.c (nonexistent) @@ -1,2606 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include - -#include "mlx4.h" -#include "fw.h" - -#define CMD_POLL_TOKEN 0xffff -#define INBOX_MASK 0xffffffffffffff00ULL - -#define CMD_CHAN_VER 1 -#define CMD_CHAN_IF_REV 1 - -enum { - /* command completed successfully: */ - CMD_STAT_OK = 0x00, - /* Internal error (such as a bus error) occurred while processing command: */ - CMD_STAT_INTERNAL_ERR = 0x01, - /* Operation/command not supported or opcode modifier not supported: */ - CMD_STAT_BAD_OP = 0x02, - /* Parameter not supported or parameter out of range: */ - CMD_STAT_BAD_PARAM = 0x03, - /* System not enabled or bad system state: */ - CMD_STAT_BAD_SYS_STATE = 0x04, - /* Attempt to access reserved or unallocaterd resource: */ - CMD_STAT_BAD_RESOURCE = 0x05, - /* Requested resource is currently executing a command, or is otherwise busy: */ - CMD_STAT_RESOURCE_BUSY = 0x06, - /* Required capability exceeds device limits: */ - CMD_STAT_EXCEED_LIM = 0x08, - /* Resource is not in the appropriate state or ownership: */ - CMD_STAT_BAD_RES_STATE = 0x09, - /* Index out of range: */ - CMD_STAT_BAD_INDEX = 0x0a, - /* FW image corrupted: */ - CMD_STAT_BAD_NVMEM = 0x0b, - /* Error in ICM mapping (e.g. not enough auxiliary ICM pages to execute command): */ - CMD_STAT_ICM_ERROR = 0x0c, - /* Attempt to modify a QP/EE which is not in the presumed state: */ - CMD_STAT_BAD_QP_STATE = 0x10, - /* Bad segment parameters (Address/Size): */ - CMD_STAT_BAD_SEG_PARAM = 0x20, - /* Memory Region has Memory Windows bound to: */ - CMD_STAT_REG_BOUND = 0x21, - /* HCA local attached memory not present: */ - CMD_STAT_LAM_NOT_PRE = 0x22, - /* Bad management packet (silently discarded): */ - CMD_STAT_BAD_PKT = 0x30, - /* More outstanding CQEs in CQ than new CQ size: */ - CMD_STAT_BAD_SIZE = 0x40, - /* Multi Function device support required: */ - CMD_STAT_MULTI_FUNC_REQ = 0x50, -}; - -enum { - HCR_IN_PARAM_OFFSET = 0x00, - HCR_IN_MODIFIER_OFFSET = 0x08, - HCR_OUT_PARAM_OFFSET = 0x0c, - HCR_TOKEN_OFFSET = 0x14, - HCR_STATUS_OFFSET = 0x18, - - HCR_OPMOD_SHIFT = 12, - HCR_T_BIT = 21, - HCR_E_BIT = 22, - HCR_GO_BIT = 23 -}; - -enum { - GO_BIT_TIMEOUT_MSECS = 10000 -}; - -enum mlx4_vlan_transition { - MLX4_VLAN_TRANSITION_VST_VST = 0, - MLX4_VLAN_TRANSITION_VST_VGT = 1, - MLX4_VLAN_TRANSITION_VGT_VST = 2, - MLX4_VLAN_TRANSITION_VGT_VGT = 3, -}; - - -struct mlx4_cmd_context { - struct completion done; - int result; - int next; - u64 out_param; - u16 token; - u8 fw_status; -}; - -static int mlx4_master_process_vhcr(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr_cmd *in_vhcr); - -static int mlx4_status_to_errno(u8 status) -{ - static const int trans_table[] = { - [CMD_STAT_INTERNAL_ERR] = -EIO, - [CMD_STAT_BAD_OP] = -EPERM, - [CMD_STAT_BAD_PARAM] = -EINVAL, - [CMD_STAT_BAD_SYS_STATE] = -ENXIO, - [CMD_STAT_BAD_RESOURCE] = -EBADF, - [CMD_STAT_RESOURCE_BUSY] = -EBUSY, - [CMD_STAT_EXCEED_LIM] = -ENOMEM, - [CMD_STAT_BAD_RES_STATE] = -EBADF, - [CMD_STAT_BAD_INDEX] = -EBADF, - [CMD_STAT_BAD_NVMEM] = -EFAULT, - [CMD_STAT_ICM_ERROR] = -ENFILE, - [CMD_STAT_BAD_QP_STATE] = -EINVAL, - [CMD_STAT_BAD_SEG_PARAM] = -EFAULT, - [CMD_STAT_REG_BOUND] = -EBUSY, - [CMD_STAT_LAM_NOT_PRE] = -EAGAIN, - [CMD_STAT_BAD_PKT] = -EINVAL, - [CMD_STAT_BAD_SIZE] = -ENOMEM, - [CMD_STAT_MULTI_FUNC_REQ] = -EACCES, - }; - - if (status >= ARRAY_SIZE(trans_table) || - (status != CMD_STAT_OK && trans_table[status] == 0)) - return -EIO; - - return trans_table[status]; -} - -static const char *cmd_to_str(u16 cmd) -{ - switch (cmd) { - case MLX4_CMD_SYS_EN: return "SYS_EN"; - case MLX4_CMD_SYS_DIS: return "SYS_DIS"; - case MLX4_CMD_MAP_FA: return "MAP_FA"; - case MLX4_CMD_UNMAP_FA: return "UNMAP_FA"; - case MLX4_CMD_RUN_FW: return "RUN_FW"; - case MLX4_CMD_MOD_STAT_CFG: return "MOD_STAT_CFG"; - case MLX4_CMD_QUERY_DEV_CAP: return "QUERY_DEV_CAP"; - case MLX4_CMD_QUERY_FW: return "QUERY_FW"; - case MLX4_CMD_ENABLE_LAM: return "ENABLE_LAM"; - case MLX4_CMD_DISABLE_LAM: return "DISABLE_LAM"; - case MLX4_CMD_QUERY_DDR: return "QUERY_DDR"; - case MLX4_CMD_QUERY_ADAPTER: return "QUERY_ADAPTER"; - case MLX4_CMD_INIT_HCA: return "INIT_HCA"; - case MLX4_CMD_CLOSE_HCA: return "CLOSE_HCA"; - case MLX4_CMD_INIT_PORT: return "INIT_PORT"; - case MLX4_CMD_CLOSE_PORT: return "CLOSE_PORT"; - case MLX4_CMD_QUERY_HCA: return "QUERY_HCA"; - case MLX4_CMD_QUERY_PORT: return "QUERY_PORT"; - case MLX4_CMD_SENSE_PORT: return "SENSE_PORT"; - case MLX4_CMD_HW_HEALTH_CHECK: return "HW_HEALTH_CHECK"; - case MLX4_CMD_SET_PORT: return "SET_PORT"; - case MLX4_CMD_SET_NODE: return "SET_NODE"; - case MLX4_CMD_QUERY_FUNC: return "QUERY_FUNC"; - case MLX4_CMD_MAP_ICM: return "MAP_ICM"; - case MLX4_CMD_UNMAP_ICM: return "UNMAP_ICM"; - case MLX4_CMD_MAP_ICM_AUX: return "MAP_ICM_AUX"; - case MLX4_CMD_UNMAP_ICM_AUX: return "UNMAP_ICM_AUX"; - case MLX4_CMD_SET_ICM_SIZE: return "SET_ICM_SIZE"; - /*master notify fw on finish for slave's flr*/ - case MLX4_CMD_INFORM_FLR_DONE: return "INFORM_FLR_DONE"; - case MLX4_CMD_GET_OP_REQ: return "GET_OP_REQ"; - - /* TPT commands */ - case MLX4_CMD_SW2HW_MPT: return "SW2HW_MPT"; - case MLX4_CMD_QUERY_MPT: return "QUERY_MPT"; - case MLX4_CMD_HW2SW_MPT: return "HW2SW_MPT"; - case MLX4_CMD_READ_MTT: return "READ_MTT"; - case MLX4_CMD_WRITE_MTT: return "WRITE_MTT"; - case MLX4_CMD_SYNC_TPT: return "SYNC_TPT"; - - /* EQ commands */ - case MLX4_CMD_MAP_EQ: return "MAP_EQ"; - case MLX4_CMD_SW2HW_EQ: return "SW2HW_EQ"; - case MLX4_CMD_HW2SW_EQ: return "HW2SW_EQ"; - case MLX4_CMD_QUERY_EQ: return "QUERY_EQ"; - - /* CQ commands */ - case MLX4_CMD_SW2HW_CQ: return "SW2HW_CQ"; - case MLX4_CMD_HW2SW_CQ: return "HW2SW_CQ"; - case MLX4_CMD_QUERY_CQ: return "QUERY_CQ:"; - case MLX4_CMD_MODIFY_CQ: return "MODIFY_CQ:"; - - /* SRQ commands */ - case MLX4_CMD_SW2HW_SRQ: return "SW2HW_SRQ"; - case MLX4_CMD_HW2SW_SRQ: return "HW2SW_SRQ"; - case MLX4_CMD_QUERY_SRQ: return "QUERY_SRQ"; - case MLX4_CMD_ARM_SRQ: return "ARM_SRQ"; - - /* QP/EE commands */ - case MLX4_CMD_RST2INIT_QP: return "RST2INIT_QP"; - case MLX4_CMD_INIT2RTR_QP: return "INIT2RTR_QP"; - case MLX4_CMD_RTR2RTS_QP: return "RTR2RTS_QP"; - case MLX4_CMD_RTS2RTS_QP: return "RTS2RTS_QP"; - case MLX4_CMD_SQERR2RTS_QP: return "SQERR2RTS_QP"; - case MLX4_CMD_2ERR_QP: return "2ERR_QP"; - case MLX4_CMD_RTS2SQD_QP: return "RTS2SQD_QP"; - case MLX4_CMD_SQD2SQD_QP: return "SQD2SQD_QP"; - case MLX4_CMD_SQD2RTS_QP: return "SQD2RTS_QP"; - case MLX4_CMD_2RST_QP: return "2RST_QP"; - case MLX4_CMD_QUERY_QP: return "QUERY_QP"; - case MLX4_CMD_INIT2INIT_QP: return "INIT2INIT_QP"; - case MLX4_CMD_SUSPEND_QP: return "SUSPEND_QP"; - case MLX4_CMD_UNSUSPEND_QP: return "UNSUSPEND_QP"; - /* special QP and management commands */ - case MLX4_CMD_CONF_SPECIAL_QP: return "CONF_SPECIAL_QP"; - case MLX4_CMD_MAD_IFC: return "MAD_IFC"; - - /* multicast commands */ - case MLX4_CMD_READ_MCG: return "READ_MCG"; - case MLX4_CMD_WRITE_MCG: return "WRITE_MCG"; - case MLX4_CMD_MGID_HASH: return "MGID_HASH"; - - /* miscellaneous commands */ - case MLX4_CMD_DIAG_RPRT: return "DIAG_RPRT"; - case MLX4_CMD_NOP: return "NOP"; - case MLX4_CMD_ACCESS_MEM: return "ACCESS_MEM"; - case MLX4_CMD_SET_VEP: return "SET_VEP"; - - /* Ethernet specific commands */ - case MLX4_CMD_SET_VLAN_FLTR: return "SET_VLAN_FLTR"; - case MLX4_CMD_SET_MCAST_FLTR: return "SET_MCAST_FLTR"; - case MLX4_CMD_DUMP_ETH_STATS: return "DUMP_ETH_STATS"; - - /* Communication channel commands */ - case MLX4_CMD_ARM_COMM_CHANNEL: return "ARM_COMM_CHANNEL"; - case MLX4_CMD_GEN_EQE: return "GEN_EQE"; - - /* virtual commands */ - case MLX4_CMD_ALLOC_RES: return "ALLOC_RES"; - case MLX4_CMD_FREE_RES: return "FREE_RES"; - case MLX4_CMD_MCAST_ATTACH: return "MCAST_ATTACH"; - case MLX4_CMD_UCAST_ATTACH: return "UCAST_ATTACH"; - case MLX4_CMD_PROMISC: return "PROMISC"; - case MLX4_CMD_QUERY_FUNC_CAP: return "QUERY_FUNC_CAP"; - case MLX4_CMD_QP_ATTACH: return "QP_ATTACH"; - - /* debug commands */ - case MLX4_CMD_QUERY_DEBUG_MSG: return "QUERY_DEBUG_MSG"; - case MLX4_CMD_SET_DEBUG_MSG: return "SET_DEBUG_MSG"; - - /* statistics commands */ - case MLX4_CMD_QUERY_IF_STAT: return "QUERY_IF_STAT"; - case MLX4_CMD_SET_IF_STAT: return "SET_IF_STAT"; - - /* register/delete flow steering network rules */ - case MLX4_QP_FLOW_STEERING_ATTACH: return "QP_FLOW_STEERING_ATTACH"; - case MLX4_QP_FLOW_STEERING_DETACH: return "QP_FLOW_STEERING_DETACH"; - case MLX4_FLOW_STEERING_IB_UC_QP_RANGE: return "FLOW_STEERING_IB_UC_QP_RANGE"; - default: return "OTHER"; - } -} - -static u8 mlx4_errno_to_status(int errno) -{ - switch (errno) { - case -EPERM: - return CMD_STAT_BAD_OP; - case -EINVAL: - return CMD_STAT_BAD_PARAM; - case -ENXIO: - return CMD_STAT_BAD_SYS_STATE; - case -EBUSY: - return CMD_STAT_RESOURCE_BUSY; - case -ENOMEM: - return CMD_STAT_EXCEED_LIM; - case -ENFILE: - return CMD_STAT_ICM_ERROR; - default: - return CMD_STAT_INTERNAL_ERR; - } -} - -static int comm_pending(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - u32 status = readl(&priv->mfunc.comm->slave_read); - - return (swab32(status) >> 31) != priv->cmd.comm_toggle; -} - -static void mlx4_comm_cmd_post(struct mlx4_dev *dev, u8 cmd, u16 param) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - u32 val; - - priv->cmd.comm_toggle ^= 1; - val = param | (cmd << 16) | (priv->cmd.comm_toggle << 31); - __raw_writel((__force u32) cpu_to_be32(val), - &priv->mfunc.comm->slave_write); - mmiowb(); -} - -static int mlx4_comm_cmd_poll(struct mlx4_dev *dev, u8 cmd, u16 param, - unsigned long timeout) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - unsigned long end; - int err = 0; - int ret_from_pending = 0; - - /* First, verify that the master reports correct status */ - if (comm_pending(dev)) { - mlx4_warn(dev, "Communication channel is not idle." - "my toggle is %d (cmd:0x%x)\n", - priv->cmd.comm_toggle, cmd); - return -EAGAIN; - } - - /* Write command */ - down(&priv->cmd.poll_sem); - mlx4_comm_cmd_post(dev, cmd, param); - - end = msecs_to_jiffies(timeout) + jiffies; - while (comm_pending(dev) && time_before(jiffies, end)) - cond_resched(); - ret_from_pending = comm_pending(dev); - if (ret_from_pending) { - /* check if the slave is trying to boot in the middle of - * FLR process. The only non-zero result in the RESET command - * is MLX4_DELAY_RESET_SLAVE*/ - if ((MLX4_COMM_CMD_RESET == cmd)) { - mlx4_warn(dev, "Got slave FLRed from Communication" - " channel (ret:0x%x)\n", ret_from_pending); - err = MLX4_DELAY_RESET_SLAVE; - } else { - mlx4_warn(dev, "Communication channel timed out\n"); - err = -ETIMEDOUT; - } - } - - up(&priv->cmd.poll_sem); - return err; -} - -static int mlx4_comm_cmd_wait(struct mlx4_dev *dev, u8 op, - u16 param, unsigned long timeout) -{ - struct mlx4_cmd *cmd = &mlx4_priv(dev)->cmd; - struct mlx4_cmd_context *context; - unsigned long end; - int err = 0; - - down(&cmd->event_sem); - - end = msecs_to_jiffies(timeout) + jiffies; - while (comm_pending(dev) && time_before(jiffies, end)) - cond_resched(); - if (comm_pending(dev)) { - mlx4_warn(dev, "mlx4_comm_cmd_wait: Comm channel " - "is not idle. My toggle is %d (op: 0x%x)\n", - mlx4_priv(dev)->cmd.comm_toggle, op); - up(&cmd->event_sem); - return -EAGAIN; - } - - spin_lock(&cmd->context_lock); - BUG_ON(cmd->free_head < 0); - context = &cmd->context[cmd->free_head]; - context->token += cmd->token_mask + 1; - cmd->free_head = context->next; - spin_unlock(&cmd->context_lock); - - init_completion(&context->done); - - mlx4_comm_cmd_post(dev, op, param); - - /* In slave, wait unconditionally for completion */ - wait_for_completion(&context->done); - - err = context->result; - if (err && context->fw_status != CMD_STAT_MULTI_FUNC_REQ) { - mlx4_err(dev, "command 0x%x failed: fw status = 0x%x\n", - op, context->fw_status); - goto out; - } - -out: - /* wait for comm channel ready - * this is necessary for prevention the race - * when switching between event to polling mode - */ - end = msecs_to_jiffies(timeout) + jiffies; - while (comm_pending(dev) && time_before(jiffies, end)) - cond_resched(); - - spin_lock(&cmd->context_lock); - context->next = cmd->free_head; - cmd->free_head = context - cmd->context; - spin_unlock(&cmd->context_lock); - - up(&cmd->event_sem); - return err; -} - -int mlx4_comm_cmd(struct mlx4_dev *dev, u8 cmd, u16 param, - unsigned long timeout) -{ - if (mlx4_priv(dev)->cmd.use_events) - return mlx4_comm_cmd_wait(dev, cmd, param, timeout); - return mlx4_comm_cmd_poll(dev, cmd, param, timeout); -} - -static int cmd_pending(struct mlx4_dev *dev) -{ - u32 status; - - if (pci_channel_offline(dev->pdev)) - return -EIO; - - status = readl(mlx4_priv(dev)->cmd.hcr + HCR_STATUS_OFFSET); - - return (status & swab32(1 << HCR_GO_BIT)) || - (mlx4_priv(dev)->cmd.toggle == - !!(status & swab32(1 << HCR_T_BIT))); -} - -static int get_status(struct mlx4_dev *dev, u32 *status, int *go_bit, - int *t_bit) -{ - if (pci_channel_offline(dev->pdev)) - return -EIO; - - *status = readl(mlx4_priv(dev)->cmd.hcr + HCR_STATUS_OFFSET); - *t_bit = !!(*status & swab32(1 << HCR_T_BIT)); - *go_bit = !!(*status & swab32(1 << HCR_GO_BIT)); - - return 0; -} - -static int mlx4_cmd_post(struct mlx4_dev *dev, struct timespec *ts1, - u64 in_param, u64 out_param, u32 in_modifier, - u8 op_modifier, u16 op, u16 token, int event) -{ - struct mlx4_cmd *cmd = &mlx4_priv(dev)->cmd; - u32 __iomem *hcr = cmd->hcr; - int ret = -EAGAIN; - unsigned long end; - int err, go_bit = 0, t_bit = 0; - u32 status = 0; - - mutex_lock(&cmd->hcr_mutex); - - if (pci_channel_offline(dev->pdev)) { - /* - * Device is going through error recovery - * and cannot accept commands. - */ - ret = -EIO; - goto out; - } - - end = jiffies; - if (event) - end += msecs_to_jiffies(GO_BIT_TIMEOUT_MSECS); - - while (cmd_pending(dev)) { - if (pci_channel_offline(dev->pdev)) { - /* - * Device is going through error recovery - * and cannot accept commands. - */ - ret = -EIO; - goto out; - } - - if (time_after_eq(jiffies, end)) { - mlx4_err(dev, "%s:cmd_pending failed\n", __func__); - goto out; - } - cond_resched(); - } - - /* - * We use writel (instead of something like memcpy_toio) - * because writes of less than 32 bits to the HCR don't work - * (and some architectures such as ia64 implement memcpy_toio - * in terms of writeb). - */ - __raw_writel((__force u32) cpu_to_be32(in_param >> 32), hcr + 0); - __raw_writel((__force u32) cpu_to_be32(in_param & 0xfffffffful), hcr + 1); - __raw_writel((__force u32) cpu_to_be32(in_modifier), hcr + 2); - __raw_writel((__force u32) cpu_to_be32(out_param >> 32), hcr + 3); - __raw_writel((__force u32) cpu_to_be32(out_param & 0xfffffffful), hcr + 4); - __raw_writel((__force u32) cpu_to_be32(token << 16), hcr + 5); - - if (ts1) - ktime_get_ts(ts1); - - /* __raw_writel may not order writes. */ - wmb(); - - __raw_writel((__force u32) cpu_to_be32((1 << HCR_GO_BIT) | - (cmd->toggle << HCR_T_BIT) | - (event ? (1 << HCR_E_BIT) : 0) | - (op_modifier << HCR_OPMOD_SHIFT) | - op), hcr + 6); - - /* - * Make sure that our HCR writes don't get mixed in with - * writes from another CPU starting a FW command. - */ - mmiowb(); - - cmd->toggle = cmd->toggle ^ 1; - - ret = 0; - -out: - if (ret) { - err = get_status(dev, &status, &go_bit, &t_bit); - mlx4_warn(dev, "Could not post command %s (0x%x): ret=%d, " - "in_param=0x%llx, in_mod=0x%x, op_mod=0x%x, " - "get_status err=%d, status_reg=0x%x, go_bit=%d, " - "t_bit=%d, toggle=0x%x\n", cmd_to_str(op), op, ret, - (unsigned long long) in_param, in_modifier, op_modifier, err, status, - go_bit, t_bit, cmd->toggle); - } - mutex_unlock(&cmd->hcr_mutex); - return ret; -} - -static int mlx4_slave_cmd(struct mlx4_dev *dev, u64 in_param, u64 *out_param, - int out_is_imm, u32 in_modifier, u8 op_modifier, - u16 op, unsigned long timeout) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_vhcr_cmd *vhcr = priv->mfunc.vhcr; - int ret; - - mutex_lock(&priv->cmd.slave_cmd_mutex); - - vhcr->in_param = cpu_to_be64(in_param); - vhcr->out_param = out_param ? cpu_to_be64(*out_param) : 0; - vhcr->in_modifier = cpu_to_be32(in_modifier); - vhcr->opcode = cpu_to_be16((((u16) op_modifier) << 12) | (op & 0xfff)); - vhcr->token = cpu_to_be16(CMD_POLL_TOKEN); - vhcr->status = 0; - vhcr->flags = !!(priv->cmd.use_events) << 6; - - if (mlx4_is_master(dev)) { - ret = mlx4_master_process_vhcr(dev, dev->caps.function, vhcr); - if (!ret) { - if (out_is_imm) { - if (out_param) - *out_param = - be64_to_cpu(vhcr->out_param); - else { - mlx4_err(dev, "response expected while" - "output mailbox is NULL for " - "command 0x%x\n", op); - vhcr->status = CMD_STAT_BAD_PARAM; - } - } - ret = mlx4_status_to_errno(vhcr->status); - } - } else { - ret = mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR_POST, 0, - MLX4_COMM_TIME + timeout); - if (!ret) { - if (out_is_imm) { - if (out_param) - *out_param = - be64_to_cpu(vhcr->out_param); - else { - mlx4_err(dev, "response expected while" - "output mailbox is NULL for " - "command 0x%x\n", op); - vhcr->status = CMD_STAT_BAD_PARAM; - } - } - ret = mlx4_status_to_errno(vhcr->status); - } else - mlx4_err(dev, "failed execution of VHCR_POST command" - "opcode %s (0x%x)\n", cmd_to_str(op), op); - } - - mutex_unlock(&priv->cmd.slave_cmd_mutex); - return ret; -} - -static int mlx4_cmd_poll(struct mlx4_dev *dev, u64 in_param, u64 *out_param, - int out_is_imm, u32 in_modifier, u8 op_modifier, - u16 op, unsigned long timeout) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - void __iomem *hcr = priv->cmd.hcr; - int err = 0; - unsigned long end; - u32 stat; - - down(&priv->cmd.poll_sem); - - if (pci_channel_offline(dev->pdev)) { - /* - * Device is going through error recovery - * and cannot accept commands. - */ - err = -EIO; - goto out; - } - - err = mlx4_cmd_post(dev, NULL, in_param, out_param ? *out_param : 0, - in_modifier, op_modifier, op, CMD_POLL_TOKEN, 0); - if (err) - goto out; - - end = msecs_to_jiffies(timeout) + jiffies; - while (cmd_pending(dev) && time_before(jiffies, end)) { - if (pci_channel_offline(dev->pdev)) { - /* - * Device is going through error recovery - * and cannot accept commands. - */ - err = -EIO; - goto out; - } - - cond_resched(); - } - - if (cmd_pending(dev)) { - mlx4_warn(dev, "command %s (0x%x) timed out (go bit not cleared)\n", - cmd_to_str(op), op); - err = -ETIMEDOUT; - goto out; - } - - if (out_is_imm) - *out_param = - (u64) be32_to_cpu((__force __be32) - __raw_readl(hcr + HCR_OUT_PARAM_OFFSET)) << 32 | - (u64) be32_to_cpu((__force __be32) - __raw_readl(hcr + HCR_OUT_PARAM_OFFSET + 4)); - stat = be32_to_cpu((__force __be32) - __raw_readl(hcr + HCR_STATUS_OFFSET)) >> 24; - err = mlx4_status_to_errno(stat); - if (err) - mlx4_err(dev, "command %s (0x%x) failed: fw status = 0x%x\n", - cmd_to_str(op), op, stat); - -out: - up(&priv->cmd.poll_sem); - return err; -} - -void mlx4_cmd_event(struct mlx4_dev *dev, u16 token, u8 status, u64 out_param) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_cmd_context *context = - &priv->cmd.context[token & priv->cmd.token_mask]; - - /* previously timed out command completing at long last */ - if (token != context->token) - return; - - context->fw_status = status; - context->result = mlx4_status_to_errno(status); - context->out_param = out_param; - - complete(&context->done); -} - -static int mlx4_cmd_wait(struct mlx4_dev *dev, u64 in_param, u64 *out_param, - int out_is_imm, u32 in_modifier, u8 op_modifier, - u16 op, unsigned long timeout) -{ - struct mlx4_cmd *cmd = &mlx4_priv(dev)->cmd; - struct mlx4_cmd_context *context; - int err = 0; - int go_bit = 0, t_bit = 0, stat_err; - u32 status = 0; - struct timespec ts1, ts2; - ktime_t t1, t2, delta; - s64 ds; - - if (out_is_imm && !out_param) - return -EINVAL; - - down(&cmd->event_sem); - - spin_lock(&cmd->context_lock); - BUG_ON(cmd->free_head < 0); - context = &cmd->context[cmd->free_head]; - context->token += cmd->token_mask + 1; - cmd->free_head = context->next; - spin_unlock(&cmd->context_lock); - - init_completion(&context->done); - - err = mlx4_cmd_post(dev, &ts1, in_param, out_param ? *out_param : 0, - in_modifier, op_modifier, op, context->token, 1); - if (err) - goto out; - - if (!wait_for_completion_timeout(&context->done, - msecs_to_jiffies(timeout))) { - stat_err = get_status(dev, &status, &go_bit, &t_bit); - mlx4_warn(dev, "command %s (0x%x) timed out: in_param=0x%llx, " - "in_mod=0x%x, op_mod=0x%x, get_status err=%d, " - "status_reg=0x%x, go_bit=%d, t_bit=%d, toggle=0x%x\n" - , cmd_to_str(op), op, (unsigned long long) in_param, in_modifier, - op_modifier, stat_err, status, go_bit, t_bit, - mlx4_priv(dev)->cmd.toggle); - err = -EBUSY; - goto out; - } - if (mlx4_debug_level & MLX4_DEBUG_MASK_CMD_TIME) { - ktime_get_ts(&ts2); - t1 = timespec_to_ktime(ts1); - t2 = timespec_to_ktime(ts2); - delta = ktime_sub(t2, t1); - ds = ktime_to_ns(delta); - pr_info("mlx4: fw exec time for %s is %lld nsec\n", cmd_to_str(op), (long long) ds); - } - - err = context->result; - if (err) { - mlx4_err(dev, "command %s (0x%x) failed: in_param=0x%llx, " - "in_mod=0x%x, op_mod=0x%x, fw status = 0x%x\n", - cmd_to_str(op), op, (unsigned long long) in_param, in_modifier, - op_modifier, context->fw_status); - - switch(context->fw_status) { - case CMD_STAT_BAD_PARAM: - mlx4_err(dev, "Parameter is not supported, " - "parameter is out of range\n"); - break; - case CMD_STAT_EXCEED_LIM: - mlx4_err(dev, "Required capability exceeded " - "device limits\n"); - break; - default: - break; - } - goto out; - } - - if (out_is_imm) - *out_param = context->out_param; - -out: - spin_lock(&cmd->context_lock); - context->next = cmd->free_head; - cmd->free_head = context - cmd->context; - spin_unlock(&cmd->context_lock); - - up(&cmd->event_sem); - return err; -} - -int __mlx4_cmd(struct mlx4_dev *dev, u64 in_param, u64 *out_param, - int out_is_imm, u32 in_modifier, u8 op_modifier, - u16 op, unsigned long timeout, int native) -{ - if (pci_channel_offline(dev->pdev)) - return -EIO; - - if (!mlx4_is_mfunc(dev) || (native && mlx4_is_master(dev))) { - if (mlx4_priv(dev)->cmd.use_events) - return mlx4_cmd_wait(dev, in_param, out_param, - out_is_imm, in_modifier, - op_modifier, op, timeout); - else - return mlx4_cmd_poll(dev, in_param, out_param, - out_is_imm, in_modifier, - op_modifier, op, timeout); - } - return mlx4_slave_cmd(dev, in_param, out_param, out_is_imm, - in_modifier, op_modifier, op, timeout); -} -EXPORT_SYMBOL_GPL(__mlx4_cmd); - - -static int mlx4_ARM_COMM_CHANNEL(struct mlx4_dev *dev) -{ - return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_ARM_COMM_CHANNEL, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); -} - -static int mlx4_ACCESS_MEM(struct mlx4_dev *dev, u64 master_addr, - int slave, u64 slave_addr, - int size, int is_read) -{ - u64 in_param; - u64 out_param; - - if ((slave_addr & 0xfff) | (master_addr & 0xfff) | - (slave & ~0x7f) | (size & 0xff)) { - mlx4_err(dev, "Bad access mem params - slave_addr:0x%llx " - "master_addr:0x%llx slave_id:%d size:%d\n", - (unsigned long long) slave_addr, (unsigned long long) master_addr, slave, size); - return -EINVAL; - } - - if (is_read) { - in_param = (u64) slave | slave_addr; - out_param = (u64) dev->caps.function | master_addr; - } else { - in_param = (u64) dev->caps.function | master_addr; - out_param = (u64) slave | slave_addr; - } - - return mlx4_cmd_imm(dev, in_param, &out_param, size, 0, - MLX4_CMD_ACCESS_MEM, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); -} - -static int query_pkey_block(struct mlx4_dev *dev, u8 port, u16 index, u16 *pkey, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox) -{ - struct ib_smp *in_mad = (struct ib_smp *)(inbox->buf); - struct ib_smp *out_mad = (struct ib_smp *)(outbox->buf); - int err; - int i; - - if (index & 0x1f) - return -EINVAL; - - in_mad->attr_mod = cpu_to_be32(index / 32); - - err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, port, 3, - MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C, - MLX4_CMD_NATIVE); - if (err) - return err; - - for (i = 0; i < 32; ++i) - pkey[i] = be16_to_cpu(((__be16 *) out_mad->data)[i]); - - return err; -} - -static int get_full_pkey_table(struct mlx4_dev *dev, u8 port, u16 *table, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox) -{ - int i; - int err; - - for (i = 0; i < dev->caps.pkey_table_len[port]; i += 32) { - err = query_pkey_block(dev, port, i, table + i, inbox, outbox); - if (err) - return err; - } - - return 0; -} -#define PORT_CAPABILITY_LOCATION_IN_SMP 20 -#define PORT_STATE_OFFSET 32 - -static enum ib_port_state vf_port_state(struct mlx4_dev *dev, int port, int vf) -{ - if (mlx4_get_slave_port_state(dev, vf, port) == SLAVE_PORT_UP) - return IB_PORT_ACTIVE; - else - return IB_PORT_DOWN; -} - -static int mlx4_MAD_IFC_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - struct ib_smp *smp = inbox->buf; - u32 index; - u8 port; - u16 *table; - int err; - int vidx, pidx; - struct mlx4_priv *priv = mlx4_priv(dev); - struct ib_smp *outsmp = outbox->buf; - __be16 *outtab = (__be16 *)(outsmp->data); - __be32 slave_cap_mask; - __be64 slave_node_guid; - port = vhcr->in_modifier; - - if (smp->base_version == 1 && - smp->mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED && - smp->class_version == 1) { - if (smp->method == IB_MGMT_METHOD_GET) { - if (smp->attr_id == IB_SMP_ATTR_PKEY_TABLE) { - index = be32_to_cpu(smp->attr_mod); - if (port < 1 || port > dev->caps.num_ports) - return -EINVAL; - table = kcalloc(dev->caps.pkey_table_len[port], sizeof *table, GFP_KERNEL); - if (!table) - return -ENOMEM; - /* need to get the full pkey table because the paravirtualized - * pkeys may be scattered among several pkey blocks. - */ - err = get_full_pkey_table(dev, port, table, inbox, outbox); - if (!err) { - for (vidx = index * 32; vidx < (index + 1) * 32; ++vidx) { - pidx = priv->virt2phys_pkey[slave][port - 1][vidx]; - outtab[vidx % 32] = cpu_to_be16(table[pidx]); - } - } - kfree(table); - return err; - } - if (smp->attr_id == IB_SMP_ATTR_PORT_INFO) { - /*get the slave specific caps:*/ - /*do the command */ - err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, - vhcr->in_modifier, vhcr->op_modifier, - vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); - /* modify the response for slaves */ - if (!err && slave != mlx4_master_func_num(dev)) { - u8 *state = outsmp->data + PORT_STATE_OFFSET; - - *state = (*state & 0xf0) | vf_port_state(dev, port, slave); - slave_cap_mask = priv->mfunc.master.slave_state[slave].ib_cap_mask[port]; - memcpy(outsmp->data + PORT_CAPABILITY_LOCATION_IN_SMP, &slave_cap_mask, 4); - } - return err; - } - if (smp->attr_id == IB_SMP_ATTR_GUID_INFO) { - /* compute slave's gid block */ - smp->attr_mod = cpu_to_be32(slave / 8); - /* execute cmd */ - err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, - vhcr->in_modifier, vhcr->op_modifier, - vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); - if (!err) { - /* if needed, move slave gid to index 0 */ - if (slave % 8) - memcpy(outsmp->data, - outsmp->data + (slave % 8) * 8, 8); - /* delete all other gids */ - memset(outsmp->data + 8, 0, 56); - } - return err; - } - if (smp->attr_id == IB_SMP_ATTR_NODE_INFO) { - err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, - vhcr->in_modifier, vhcr->op_modifier, - vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); - if (!err) { - slave_node_guid = mlx4_get_slave_node_guid(dev, slave); - memcpy(outsmp->data + 12, &slave_node_guid, 8); - } - return err; - } - } - } - if (slave != mlx4_master_func_num(dev) && - ((smp->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) || - (smp->mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED && - smp->method == IB_MGMT_METHOD_SET))) { - mlx4_err(dev, "slave %d is trying to execute a Subnet MGMT MAD, " - "class 0x%x, method 0x%x for attr 0x%x. Rejecting\n", - slave, smp->method, smp->mgmt_class, - be16_to_cpu(smp->attr_id)); - return -EPERM; - } - /*default:*/ - return mlx4_cmd_box(dev, inbox->dma, outbox->dma, - vhcr->in_modifier, vhcr->op_modifier, - vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); -} - -static int MLX4_CMD_DIAG_RPRT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - return -EPERM; -} - -static int MLX4_CMD_UPDATE_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - return -EPERM; -} - -int mlx4_DMA_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - u64 in_param; - u64 out_param; - int err; - - in_param = cmd->has_inbox ? (u64) inbox->dma : vhcr->in_param; - out_param = cmd->has_outbox ? (u64) outbox->dma : vhcr->out_param; - if (cmd->encode_slave_id) { - in_param &= 0xffffffffffffff00ll; - in_param |= slave; - } - - err = __mlx4_cmd(dev, in_param, &out_param, cmd->out_is_imm, - vhcr->in_modifier, vhcr->op_modifier, vhcr->op, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); - - if (cmd->out_is_imm) - vhcr->out_param = out_param; - - return err; -} - -static struct mlx4_cmd_info cmd_info[] = { - { - .opcode = MLX4_CMD_QUERY_FW, - .has_inbox = false, - .has_outbox = true, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_QUERY_FW_wrapper - }, - { - .opcode = MLX4_CMD_QUERY_HCA, - .has_inbox = false, - .has_outbox = true, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = NULL - }, - { - .opcode = MLX4_CMD_QUERY_DEV_CAP, - .has_inbox = false, - .has_outbox = true, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_QUERY_DEV_CAP_wrapper - }, - { - .opcode = MLX4_CMD_QUERY_FUNC_CAP, - .has_inbox = false, - .has_outbox = true, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_QUERY_FUNC_CAP_wrapper - }, - { - .opcode = MLX4_CMD_QUERY_ADAPTER, - .has_inbox = false, - .has_outbox = true, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = NULL - }, - { - .opcode = MLX4_CMD_INIT_PORT, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_INIT_PORT_wrapper - }, - { - .opcode = MLX4_CMD_CLOSE_PORT, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_CLOSE_PORT_wrapper - }, - { - .opcode = MLX4_CMD_QUERY_PORT, - .has_inbox = false, - .has_outbox = true, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_QUERY_PORT_wrapper - }, - { - .opcode = MLX4_CMD_SET_PORT, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_SET_PORT_wrapper - }, - { - .opcode = MLX4_CMD_MAP_EQ, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_MAP_EQ_wrapper - }, - { - .opcode = MLX4_CMD_SW2HW_EQ, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = true, - .verify = NULL, - .wrapper = mlx4_SW2HW_EQ_wrapper - }, - { - .opcode = MLX4_CMD_HW_HEALTH_CHECK, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = NULL - }, - { - .opcode = MLX4_CMD_DIAG_RPRT, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .skip_err_print = true, - .verify = NULL, - .wrapper = MLX4_CMD_DIAG_RPRT_wrapper - }, - { - .opcode = MLX4_CMD_NOP, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = NULL - }, - { - .opcode = MLX4_CMD_ALLOC_RES, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = true, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_ALLOC_RES_wrapper - }, - { - .opcode = MLX4_CMD_FREE_RES, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_FREE_RES_wrapper - }, - { - .opcode = MLX4_CMD_SW2HW_MPT, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = true, - .verify = NULL, - .wrapper = mlx4_SW2HW_MPT_wrapper - }, - { - .opcode = MLX4_CMD_QUERY_MPT, - .has_inbox = false, - .has_outbox = true, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_QUERY_MPT_wrapper - }, - { - .opcode = MLX4_CMD_HW2SW_MPT, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_HW2SW_MPT_wrapper - }, - { - .opcode = MLX4_CMD_READ_MTT, - .has_inbox = false, - .has_outbox = true, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = NULL - }, - { - .opcode = MLX4_CMD_WRITE_MTT, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_WRITE_MTT_wrapper - }, - { - .opcode = MLX4_CMD_SYNC_TPT, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = NULL - }, - { - .opcode = MLX4_CMD_HW2SW_EQ, - .has_inbox = false, - .has_outbox = true, - .out_is_imm = false, - .encode_slave_id = true, - .verify = NULL, - .wrapper = mlx4_HW2SW_EQ_wrapper - }, - { - .opcode = MLX4_CMD_QUERY_EQ, - .has_inbox = false, - .has_outbox = true, - .out_is_imm = false, - .encode_slave_id = true, - .verify = NULL, - .wrapper = mlx4_QUERY_EQ_wrapper - }, - { - .opcode = MLX4_CMD_SW2HW_CQ, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = true, - .verify = NULL, - .wrapper = mlx4_SW2HW_CQ_wrapper - }, - { - .opcode = MLX4_CMD_HW2SW_CQ, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_HW2SW_CQ_wrapper - }, - { - .opcode = MLX4_CMD_QUERY_CQ, - .has_inbox = false, - .has_outbox = true, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_QUERY_CQ_wrapper - }, - { - .opcode = MLX4_CMD_MODIFY_CQ, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = true, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_MODIFY_CQ_wrapper - }, - { - .opcode = MLX4_CMD_SW2HW_SRQ, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = true, - .verify = NULL, - .wrapper = mlx4_SW2HW_SRQ_wrapper - }, - { - .opcode = MLX4_CMD_HW2SW_SRQ, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_HW2SW_SRQ_wrapper - }, - { - .opcode = MLX4_CMD_QUERY_SRQ, - .has_inbox = false, - .has_outbox = true, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_QUERY_SRQ_wrapper - }, - { - .opcode = MLX4_CMD_ARM_SRQ, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_ARM_SRQ_wrapper - }, - { - .opcode = MLX4_CMD_RST2INIT_QP, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = true, - .verify = NULL, - .wrapper = mlx4_RST2INIT_QP_wrapper - }, - { - .opcode = MLX4_CMD_INIT2INIT_QP, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_INIT2INIT_QP_wrapper - }, - { - .opcode = MLX4_CMD_INIT2RTR_QP, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_INIT2RTR_QP_wrapper - }, - { - .opcode = MLX4_CMD_RTR2RTS_QP, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_RTR2RTS_QP_wrapper - }, - { - .opcode = MLX4_CMD_RTS2RTS_QP, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_RTS2RTS_QP_wrapper - }, - { - .opcode = MLX4_CMD_SQERR2RTS_QP, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_SQERR2RTS_QP_wrapper - }, - { - .opcode = MLX4_CMD_2ERR_QP, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_GEN_QP_wrapper - }, - { - .opcode = MLX4_CMD_RTS2SQD_QP, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_GEN_QP_wrapper - }, - { - .opcode = MLX4_CMD_SQD2SQD_QP, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_SQD2SQD_QP_wrapper - }, - { - .opcode = MLX4_CMD_SQD2RTS_QP, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_SQD2RTS_QP_wrapper - }, - { - .opcode = MLX4_CMD_2RST_QP, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_2RST_QP_wrapper - }, - { - .opcode = MLX4_CMD_QUERY_QP, - .has_inbox = false, - .has_outbox = true, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_GEN_QP_wrapper - }, - { - .opcode = MLX4_CMD_SUSPEND_QP, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_GEN_QP_wrapper - }, - { - .opcode = MLX4_CMD_UNSUSPEND_QP, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_GEN_QP_wrapper - }, - { - .opcode = MLX4_CMD_UPDATE_QP, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .skip_err_print = true, - .verify = NULL, - .wrapper = MLX4_CMD_UPDATE_QP_wrapper - }, - { - .opcode = MLX4_CMD_CONF_SPECIAL_QP, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, /* XXX verify: only demux can do this */ - .wrapper = NULL - }, - { - .opcode = MLX4_CMD_MAD_IFC, - .has_inbox = true, - .has_outbox = true, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_MAD_IFC_wrapper - }, - { - .opcode = MLX4_CMD_QUERY_IF_STAT, - .has_inbox = false, - .has_outbox = true, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_QUERY_IF_STAT_wrapper - }, - /* Native multicast commands are not available for guests */ - { - .opcode = MLX4_CMD_QP_ATTACH, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_QP_ATTACH_wrapper - }, - { - .opcode = MLX4_CMD_PROMISC, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_PROMISC_wrapper - }, - /* Ethernet specific commands */ - { - .opcode = MLX4_CMD_SET_VLAN_FLTR, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_SET_VLAN_FLTR_wrapper - }, - { - .opcode = MLX4_CMD_SET_MCAST_FLTR, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_SET_MCAST_FLTR_wrapper - }, - { - .opcode = MLX4_CMD_DUMP_ETH_STATS, - .has_inbox = false, - .has_outbox = true, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_DUMP_ETH_STATS_wrapper - }, - { - .opcode = MLX4_CMD_INFORM_FLR_DONE, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = NULL - }, - /* flow steering commands */ - { - .opcode = MLX4_QP_FLOW_STEERING_ATTACH, - .has_inbox = true, - .has_outbox = false, - .out_is_imm = true, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_QP_FLOW_STEERING_ATTACH_wrapper - }, - { - .opcode = MLX4_QP_FLOW_STEERING_DETACH, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .verify = NULL, - .wrapper = mlx4_QP_FLOW_STEERING_DETACH_wrapper - }, - /* wol commands */ - { - .opcode = MLX4_CMD_MOD_STAT_CFG, - .has_inbox = false, - .has_outbox = false, - .out_is_imm = false, - .encode_slave_id = false, - .skip_err_print = true, - .verify = NULL, - .wrapper = mlx4_MOD_STAT_CFG_wrapper - }, -}; - -static int mlx4_master_process_vhcr(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr_cmd *in_vhcr) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_cmd_info *cmd = NULL; - struct mlx4_vhcr_cmd *vhcr_cmd = in_vhcr ? in_vhcr : priv->mfunc.vhcr; - struct mlx4_vhcr *vhcr; - struct mlx4_cmd_mailbox *inbox = NULL; - struct mlx4_cmd_mailbox *outbox = NULL; - u64 in_param; - u64 out_param; - int ret = 0; - int i; - int err = 0; - - /* Create sw representation of Virtual HCR */ - vhcr = kzalloc(sizeof(struct mlx4_vhcr), GFP_KERNEL); - if (!vhcr) - return -ENOMEM; - - /* DMA in the vHCR */ - if (!in_vhcr) { - ret = mlx4_ACCESS_MEM(dev, priv->mfunc.vhcr_dma, slave, - priv->mfunc.master.slave_state[slave].vhcr_dma, - ALIGN(sizeof(struct mlx4_vhcr_cmd), - MLX4_ACCESS_MEM_ALIGN), 1); - if (ret) { - mlx4_err(dev, "%s:Failed reading vhcr" - "ret: 0x%x\n", __func__, ret); - kfree(vhcr); - return ret; - } - } - - /* Fill SW VHCR fields */ - vhcr->in_param = be64_to_cpu(vhcr_cmd->in_param); - vhcr->out_param = be64_to_cpu(vhcr_cmd->out_param); - vhcr->in_modifier = be32_to_cpu(vhcr_cmd->in_modifier); - vhcr->token = be16_to_cpu(vhcr_cmd->token); - vhcr->op = be16_to_cpu(vhcr_cmd->opcode) & 0xfff; - vhcr->op_modifier = (u8) (be16_to_cpu(vhcr_cmd->opcode) >> 12); - vhcr->e_bit = vhcr_cmd->flags & (1 << 6); - - /* Lookup command */ - for (i = 0; i < ARRAY_SIZE(cmd_info); ++i) { - if (vhcr->op == cmd_info[i].opcode) { - cmd = &cmd_info[i]; - break; - } - } - if (!cmd) { - mlx4_err(dev, "unparavirt command: %s (0x%x) accepted from slave:%d\n", - cmd_to_str(vhcr->op), vhcr->op, slave); - vhcr_cmd->status = CMD_STAT_BAD_PARAM; - goto out_status; - } - - /* Read inbox */ - if (cmd->has_inbox) { - vhcr->in_param &= INBOX_MASK; - inbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(inbox)) { - vhcr_cmd->status = CMD_STAT_BAD_SIZE; - inbox = NULL; - goto out_status; - } - - if (mlx4_ACCESS_MEM(dev, inbox->dma, slave, - vhcr->in_param, - MLX4_MAILBOX_SIZE, 1)) { - mlx4_err(dev, "%s: Failed reading inbox for cmd %s (0x%x)\n", - __func__, cmd_to_str(cmd->opcode), cmd->opcode); - vhcr_cmd->status = CMD_STAT_INTERNAL_ERR; - goto out_status; - } - } - - /* Apply permission and bound checks if applicable */ - if (cmd->verify && cmd->verify(dev, slave, vhcr, inbox)) { - mlx4_warn(dev, "Command %s (0x%x) from slave: %d failed protection " - "checks for resource_id: %d\n", cmd_to_str(vhcr->op), - vhcr->op, slave, vhcr->in_modifier); - vhcr_cmd->status = CMD_STAT_BAD_OP; - goto out_status; - } - - /* Allocate outbox */ - if (cmd->has_outbox) { - outbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(outbox)) { - vhcr_cmd->status = CMD_STAT_BAD_SIZE; - outbox = NULL; - goto out_status; - } - } - - /* Execute the command! */ - if (cmd->wrapper) { - err = cmd->wrapper(dev, slave, vhcr, inbox, outbox, - cmd); - if (cmd->out_is_imm) - vhcr_cmd->out_param = cpu_to_be64(vhcr->out_param); - } else { - in_param = cmd->has_inbox ? (u64) inbox->dma : - vhcr->in_param; - out_param = cmd->has_outbox ? (u64) outbox->dma : - vhcr->out_param; - err = __mlx4_cmd(dev, in_param, &out_param, - cmd->out_is_imm, vhcr->in_modifier, - vhcr->op_modifier, vhcr->op, - MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - - if (cmd->out_is_imm) { - vhcr->out_param = out_param; - vhcr_cmd->out_param = cpu_to_be64(vhcr->out_param); - } - } - - if (err) { - if (!cmd->skip_err_print) - mlx4_warn(dev, "vhcr command %s (0x%x) slave:%d " - "in_param 0x%llx in_mod=0x%x, op_mod=0x%x " - "failed with error:%d, status %d\n", - cmd_to_str(vhcr->op), vhcr->op, slave, - (unsigned long long) vhcr->in_param, vhcr->in_modifier, - vhcr->op_modifier, vhcr->errno, err); - vhcr_cmd->status = mlx4_errno_to_status(err); - goto out_status; - } - - - /* Write outbox if command completed successfully */ - if (cmd->has_outbox && !vhcr_cmd->status) { - ret = mlx4_ACCESS_MEM(dev, outbox->dma, slave, - vhcr->out_param, - MLX4_MAILBOX_SIZE, MLX4_CMD_WRAPPED); - if (ret) { - /* If we failed to write back the outbox after the - *command was successfully executed, we must fail this - * slave, as it is now in undefined state */ - mlx4_err(dev, "%s: Failed writing outbox\n", __func__); - goto out; - } - } - -out_status: - /* DMA back vhcr result */ - if (!in_vhcr) { - ret = mlx4_ACCESS_MEM(dev, priv->mfunc.vhcr_dma, slave, - priv->mfunc.master.slave_state[slave].vhcr_dma, - ALIGN(sizeof(struct mlx4_vhcr), - MLX4_ACCESS_MEM_ALIGN), - MLX4_CMD_WRAPPED); - if (ret) - mlx4_err(dev, "%s:Failed writing vhcr result\n", - __func__); - else if (vhcr->e_bit && - mlx4_GEN_EQE(dev, slave, &priv->mfunc.master.cmd_eqe)) - mlx4_warn(dev, "Failed to generate command completion " - "eqe for slave %d\n", slave); - } - -out: - kfree(vhcr); - mlx4_free_cmd_mailbox(dev, inbox); - mlx4_free_cmd_mailbox(dev, outbox); - return ret; -} - -static int mlx4_master_immediate_activate_vlan_qos(struct mlx4_priv *priv, - int slave, int port) -{ - struct mlx4_vport_oper_state *vp_oper; - struct mlx4_vport_state *vp_admin; - struct mlx4_vf_immed_vlan_work *work; - int err; - int admin_vlan_ix = NO_INDX; - - vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; - vp_admin = &priv->mfunc.master.vf_admin[slave].vport[port]; - - if (vp_oper->state.default_vlan == vp_admin->default_vlan && - vp_oper->state.default_qos == vp_admin->default_qos) - return 0; - - work = kzalloc(sizeof(*work), GFP_KERNEL); - if (!work) - return -ENOMEM; - - if (vp_oper->state.default_vlan != vp_admin->default_vlan) { - if (MLX4_VGT != vp_admin->default_vlan) { - err = __mlx4_register_vlan(&priv->dev, port, - vp_admin->default_vlan, - &admin_vlan_ix); - if (err) { - mlx4_warn((&priv->dev), - "No vlan resources slave %d, port %d\n", - slave, port); - kfree(work); - return err; - } - } else { - admin_vlan_ix = NO_INDX; - } - work->flags |= MLX4_VF_IMMED_VLAN_FLAG_VLAN; - mlx4_dbg((&(priv->dev)), - "alloc vlan %d idx %d slave %d port %d\n", - (int)(vp_admin->default_vlan), - admin_vlan_ix, slave, port); - } - - /* save original vlan ix and vlan id */ - work->orig_vlan_id = vp_oper->state.default_vlan; - work->orig_vlan_ix = vp_oper->vlan_idx; - - /* handle new qos */ - if (vp_oper->state.default_qos != vp_admin->default_qos) - work->flags |= MLX4_VF_IMMED_VLAN_FLAG_QOS; - - if (work->flags & MLX4_VF_IMMED_VLAN_FLAG_VLAN) - vp_oper->vlan_idx = admin_vlan_ix; - - vp_oper->state.default_vlan = vp_admin->default_vlan; - vp_oper->state.default_qos = vp_admin->default_qos; - - /* iterate over QPs owned by this slave, using UPDATE_QP */ - work->port = port; - work->slave = slave; - work->qos = vp_oper->state.default_qos; - work->vlan_id = vp_oper->state.default_vlan; - work->vlan_ix = vp_oper->vlan_idx; - work->priv = priv; - INIT_WORK(&work->work, mlx4_vf_immed_vlan_work_handler); - queue_work(priv->mfunc.master.comm_wq, &work->work); - - return 0; -} - - -static int mlx4_master_activate_admin_state(struct mlx4_priv *priv, int slave) -{ - int port, err; - struct mlx4_vport_state *vp_admin; - struct mlx4_vport_oper_state *vp_oper; - - for (port = 1; port <= MLX4_MAX_PORTS; port++) { - vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; - vp_admin = &priv->mfunc.master.vf_admin[slave].vport[port]; - vp_oper->state = *vp_admin; - if (MLX4_VGT != vp_admin->default_vlan) { - err = __mlx4_register_vlan(&priv->dev, port, - vp_admin->default_vlan, &(vp_oper->vlan_idx)); - if (err) { - vp_oper->vlan_idx = NO_INDX; - mlx4_warn((&priv->dev), - "No vlan resorces slave %d, port %d\n", - slave, port); - return err; - } - mlx4_dbg((&(priv->dev)), "alloc vlan %d idx %d slave %d port %d\n", - (int)(vp_oper->state.default_vlan), - vp_oper->vlan_idx, slave, port); - } - if (vp_admin->spoofchk) { - vp_oper->mac_idx = __mlx4_register_mac(&priv->dev, - port, - vp_admin->mac); - if (0 > vp_oper->mac_idx) { - err = vp_oper->mac_idx; - vp_oper->mac_idx = NO_INDX; - mlx4_warn((&priv->dev), - "No mac resources slave %d, port %d\n", - slave, port); - return err; - } - mlx4_dbg((&(priv->dev)), "alloc mac %llx idx %d slave %d port %d\n", - (unsigned long long) vp_oper->state.mac, vp_oper->mac_idx, slave, port); - } - } - return 0; -} - -static void mlx4_master_deactivate_admin_state(struct mlx4_priv *priv, int slave) -{ - int port; - struct mlx4_vport_oper_state *vp_oper; - - for (port = 1; port <= MLX4_MAX_PORTS; port++) { - vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; - if (NO_INDX != vp_oper->vlan_idx) { - __mlx4_unregister_vlan(&priv->dev, - port, vp_oper->state.default_vlan); - vp_oper->vlan_idx = NO_INDX; - } - if (NO_INDX != vp_oper->mac_idx) { - __mlx4_unregister_mac(&priv->dev, port, vp_oper->state.mac); - vp_oper->mac_idx = NO_INDX; - } - } - return; -} - -static void mlx4_master_do_cmd(struct mlx4_dev *dev, int slave, u8 cmd, - u16 param, u8 toggle) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state; - u32 reply; - u8 is_going_down = 0; - int i; - unsigned long flags; - - slave_state[slave].comm_toggle ^= 1; - reply = (u32) slave_state[slave].comm_toggle << 31; - if (toggle != slave_state[slave].comm_toggle) { - mlx4_warn(dev, "Incorrect toggle %d from slave %d. *** MASTER" - "STATE COMPROMISIED ***\n", toggle, slave); - goto reset_slave; - } - if (cmd == MLX4_COMM_CMD_RESET) { - mlx4_warn(dev, "Received reset from slave:%d\n", slave); - slave_state[slave].active = false; - slave_state[slave].old_vlan_api = false; - mlx4_master_deactivate_admin_state(priv, slave); - for (i = 0; i < MLX4_EVENT_TYPES_NUM; ++i) { - slave_state[slave].event_eq[i].eqn = -1; - slave_state[slave].event_eq[i].token = 0; - } - /*check if we are in the middle of FLR process, - if so return "retry" status to the slave*/ - if (MLX4_COMM_CMD_FLR == slave_state[slave].last_cmd) - goto inform_slave_state; - - mlx4_dispatch_event(dev, MLX4_DEV_EVENT_SLAVE_SHUTDOWN, slave); - - /* write the version in the event field */ - reply |= mlx4_comm_get_version(); - - goto reset_slave; - } - /*command from slave in the middle of FLR*/ - if (cmd != MLX4_COMM_CMD_RESET && - MLX4_COMM_CMD_FLR == slave_state[slave].last_cmd) { - mlx4_warn(dev, "slave:%d is Trying to run cmd (0x%x) " - "in the middle of FLR\n", slave, cmd); - return; - } - - switch (cmd) { - case MLX4_COMM_CMD_VHCR0: - if (slave_state[slave].last_cmd != MLX4_COMM_CMD_RESET) - goto reset_slave; - slave_state[slave].vhcr_dma = ((u64) param) << 48; - priv->mfunc.master.slave_state[slave].cookie = 0; - break; - case MLX4_COMM_CMD_VHCR1: - if (slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR0) - goto reset_slave; - slave_state[slave].vhcr_dma |= ((u64) param) << 32; - break; - case MLX4_COMM_CMD_VHCR2: - if (slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR1) - goto reset_slave; - slave_state[slave].vhcr_dma |= ((u64) param) << 16; - break; - case MLX4_COMM_CMD_VHCR_EN: - if (slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR2) - goto reset_slave; - slave_state[slave].vhcr_dma |= param; - if (mlx4_master_activate_admin_state(priv, slave)) - goto reset_slave; - slave_state[slave].active = true; - mlx4_dispatch_event(dev, MLX4_DEV_EVENT_SLAVE_INIT, slave); - break; - case MLX4_COMM_CMD_VHCR_POST: - if ((slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR_EN) && - (slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR_POST)) - goto reset_slave; - - mutex_lock(&priv->cmd.slave_cmd_mutex); - if (mlx4_master_process_vhcr(dev, slave, NULL)) { - mlx4_err(dev, "Failed processing vhcr for slave: %d," - " resetting slave.\n", slave); - mutex_unlock(&priv->cmd.slave_cmd_mutex); - goto reset_slave; - } - mutex_unlock(&priv->cmd.slave_cmd_mutex); - break; - default: - mlx4_warn(dev, "Bad comm cmd: %d from slave: %d\n", cmd, slave); - goto reset_slave; - } - spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags); - if (!slave_state[slave].is_slave_going_down) - slave_state[slave].last_cmd = cmd; - else - is_going_down = 1; - spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags); - if (is_going_down) { - mlx4_warn(dev, "Slave is going down aborting command (%d)" - " executing from slave: %d\n", - cmd, slave); - return; - } - __raw_writel((__force u32) cpu_to_be32(reply), - &priv->mfunc.comm[slave].slave_read); - mmiowb(); - - return; - -reset_slave: - /* cleanup any slave resources */ - mlx4_delete_all_resources_for_slave(dev, slave); - spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags); - if (!slave_state[slave].is_slave_going_down) - slave_state[slave].last_cmd = MLX4_COMM_CMD_RESET; - spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags); - /*with slave in the middle of flr, no need to clean resources again.*/ -inform_slave_state: - __raw_writel((__force u32) cpu_to_be32(reply), - &priv->mfunc.comm[slave].slave_read); - wmb(); -} - -/* master command processing */ -void mlx4_master_comm_channel(struct work_struct *work) -{ - struct mlx4_mfunc_master_ctx *master = - container_of(work, - struct mlx4_mfunc_master_ctx, - comm_work); - struct mlx4_mfunc *mfunc = - container_of(master, struct mlx4_mfunc, master); - struct mlx4_priv *priv = - container_of(mfunc, struct mlx4_priv, mfunc); - struct mlx4_dev *dev = &priv->dev; - __be32 *bit_vec; - u32 comm_cmd; - u32 vec; - int i, j, slave; - int toggle; - int served = 0; - int reported = 0; - u32 slt; - - bit_vec = master->comm_arm_bit_vector; - for (i = 0; i < COMM_CHANNEL_BIT_ARRAY_SIZE; i++) { - vec = be32_to_cpu(bit_vec[i]); - for (j = 0; j < 32; j++) { - if (!(vec & (1 << j))) - continue; - ++reported; - slave = (i * 32) + j; - comm_cmd = swab32(readl( - &mfunc->comm[slave].slave_write)); - slt = swab32(readl(&mfunc->comm[slave].slave_read)) - >> 31; - toggle = comm_cmd >> 31; - if (toggle != slt) { - if (master->slave_state[slave].comm_toggle - != slt) { - mlx4_info(dev, "slave %d out of sync." - " read toggle %d, state toggle %d. " - "Resynching.\n", slave, slt, - master->slave_state[slave].comm_toggle); - master->slave_state[slave].comm_toggle = - slt; - } - mlx4_master_do_cmd(dev, slave, - comm_cmd >> 16 & 0xff, - comm_cmd & 0xffff, toggle); - ++served; - } else - mlx4_err(dev, "slave %d out of sync." - " read toggle %d, write toggle %d.\n", slave, slt, - toggle); - } - } - - if (reported && reported != served) - mlx4_warn(dev, "Got command event with bitmask from %d slaves" - " but %d were served\n", - reported, served); -} -/* master command processing */ -void mlx4_master_arm_comm_channel(struct work_struct *work) -{ - struct mlx4_mfunc_master_ctx *master = - container_of(work, - struct mlx4_mfunc_master_ctx, - arm_comm_work); - struct mlx4_mfunc *mfunc = - container_of(master, struct mlx4_mfunc, master); - struct mlx4_priv *priv = - container_of(mfunc, struct mlx4_priv, mfunc); - struct mlx4_dev *dev = &priv->dev; - - if (mlx4_ARM_COMM_CHANNEL(dev)) - mlx4_warn(dev, "Failed to arm comm channel events\n"); -} - -static int sync_toggles(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int wr_toggle; - int rd_toggle; - unsigned long end; - - wr_toggle = swab32(readl(&priv->mfunc.comm->slave_write)) >> 31; - end = jiffies + msecs_to_jiffies(5000); - - while (time_before(jiffies, end)) { - rd_toggle = swab32(readl(&priv->mfunc.comm->slave_read)) >> 31; - if (rd_toggle == wr_toggle) { - priv->cmd.comm_toggle = rd_toggle; - return 0; - } - - cond_resched(); - } - - /* - * we could reach here if for example the previous VM using this - * function misbehaved and left the channel with unsynced state. We - * should fix this here and give this VM a chance to use a properly - * synced channel - */ - mlx4_warn(dev, "recovering from previously mis-behaved VM\n"); - __raw_writel((__force u32) 0, &priv->mfunc.comm->slave_read); - __raw_writel((__force u32) 0, &priv->mfunc.comm->slave_write); - priv->cmd.comm_toggle = 0; - - return 0; -} - -int mlx4_multi_func_init(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_slave_state *s_state; - int i, j, err, port; - - if (mlx4_is_master(dev)) - priv->mfunc.comm = - ioremap(pci_resource_start(dev->pdev, priv->fw.comm_bar) + - priv->fw.comm_base, MLX4_COMM_PAGESIZE); - else - priv->mfunc.comm = - ioremap(pci_resource_start(dev->pdev, 2) + - MLX4_SLAVE_COMM_BASE, MLX4_COMM_PAGESIZE); - if (!priv->mfunc.comm) { - mlx4_err(dev, "Couldn't map communication vector.\n"); - goto err_vhcr; - } - - if (mlx4_is_master(dev)) { - priv->mfunc.master.slave_state = - kzalloc(dev->num_slaves * - sizeof(struct mlx4_slave_state), GFP_KERNEL); - if (!priv->mfunc.master.slave_state) - goto err_comm; - - priv->mfunc.master.vf_admin = - kzalloc(dev->num_slaves * - sizeof(struct mlx4_vf_admin_state), GFP_KERNEL); - if (!priv->mfunc.master.vf_admin) - goto err_comm_admin; - - priv->mfunc.master.vf_oper = - kzalloc(dev->num_slaves * - sizeof(struct mlx4_vf_oper_state), GFP_KERNEL); - if (!priv->mfunc.master.vf_oper) - goto err_comm_oper; - - for (i = 0; i < dev->num_slaves; ++i) { - s_state = &priv->mfunc.master.slave_state[i]; - s_state->last_cmd = MLX4_COMM_CMD_RESET; - mutex_init(&priv->mfunc.master.gen_eqe_mutex[i]); - for (j = 0; j < MLX4_EVENT_TYPES_NUM; ++j) - s_state->event_eq[j].eqn = -1; - __raw_writel((__force u32) 0, - &priv->mfunc.comm[i].slave_write); - __raw_writel((__force u32) 0, - &priv->mfunc.comm[i].slave_read); - mmiowb(); - for (port = 1; port <= MLX4_MAX_PORTS; port++) { - s_state->vlan_filter[port] = - kzalloc(sizeof(struct mlx4_vlan_fltr), - GFP_KERNEL); - if (!s_state->vlan_filter[port]) { - if (--port) - kfree(s_state->vlan_filter[port]); - goto err_slaves; - } - INIT_LIST_HEAD(&s_state->mcast_filters[port]); - priv->mfunc.master.vf_admin[i].vport[port].default_vlan = MLX4_VGT; - priv->mfunc.master.vf_oper[i].vport[port].state.default_vlan = MLX4_VGT; - priv->mfunc.master.vf_oper[i].vport[port].vlan_idx = NO_INDX; - priv->mfunc.master.vf_oper[i].vport[port].mac_idx = NO_INDX; - } - spin_lock_init(&s_state->lock); - } - - memset(&priv->mfunc.master.cmd_eqe, 0, dev->caps.eqe_size); - priv->mfunc.master.cmd_eqe.type = MLX4_EVENT_TYPE_CMD; - INIT_WORK(&priv->mfunc.master.comm_work, - mlx4_master_comm_channel); - INIT_WORK(&priv->mfunc.master.arm_comm_work, - mlx4_master_arm_comm_channel); - INIT_WORK(&priv->mfunc.master.slave_event_work, - mlx4_gen_slave_eqe); - INIT_WORK(&priv->mfunc.master.slave_flr_event_work, - mlx4_master_handle_slave_flr); - spin_lock_init(&priv->mfunc.master.slave_state_lock); - spin_lock_init(&priv->mfunc.master.slave_eq.event_lock); - priv->mfunc.master.comm_wq = - create_singlethread_workqueue("mlx4_comm"); - if (!priv->mfunc.master.comm_wq) - goto err_slaves; - - if (mlx4_init_resource_tracker(dev)) - goto err_thread; - - err = mlx4_ARM_COMM_CHANNEL(dev); - if (err) { - mlx4_err(dev, " Failed to arm comm channel eq: %x\n", - err); - goto err_resource; - } - - } else { - err = sync_toggles(dev); - if (err) { - mlx4_err(dev, "Couldn't sync toggles\n"); - goto err_comm; - } - } - return 0; - -err_resource: - mlx4_free_resource_tracker(dev, RES_TR_FREE_ALL); -err_thread: - flush_workqueue(priv->mfunc.master.comm_wq); - destroy_workqueue(priv->mfunc.master.comm_wq); -err_slaves: - while (--i) { - for (port = 1; port <= MLX4_MAX_PORTS; port++) - kfree(priv->mfunc.master.slave_state[i].vlan_filter[port]); - } - kfree(priv->mfunc.master.vf_oper); -err_comm_oper: - kfree(priv->mfunc.master.vf_admin); -err_comm_admin: - kfree(priv->mfunc.master.slave_state); -err_comm: - iounmap(priv->mfunc.comm); -err_vhcr: - dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, - priv->mfunc.vhcr, - priv->mfunc.vhcr_dma); - priv->mfunc.vhcr = NULL; - return -ENOMEM; -} - -int mlx4_cmd_init(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - mutex_init(&priv->cmd.hcr_mutex); - mutex_init(&priv->cmd.slave_cmd_mutex); - sema_init(&priv->cmd.poll_sem, 1); - priv->cmd.use_events = 0; - priv->cmd.toggle = 1; - - priv->cmd.hcr = NULL; - priv->mfunc.vhcr = NULL; - - if (!mlx4_is_slave(dev)) { - priv->cmd.hcr = ioremap(pci_resource_start(dev->pdev, 0) + - MLX4_HCR_BASE, MLX4_HCR_SIZE); - if (!priv->cmd.hcr) { - mlx4_err(dev, "Couldn't map command register.\n"); - return -ENOMEM; - } - } - - if (mlx4_is_mfunc(dev)) { - priv->mfunc.vhcr = dma_alloc_coherent(&(dev->pdev->dev), PAGE_SIZE, - &priv->mfunc.vhcr_dma, - GFP_KERNEL); - if (!priv->mfunc.vhcr) { - mlx4_err(dev, "Couldn't allocate VHCR.\n"); - goto err_hcr; - } - } - - priv->cmd.pool = pci_pool_create("mlx4_cmd", dev->pdev, - MLX4_MAILBOX_SIZE, - MLX4_MAILBOX_SIZE, 0); - if (!priv->cmd.pool) - goto err_vhcr; - - return 0; - -err_vhcr: - if (mlx4_is_mfunc(dev)) - dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, - priv->mfunc.vhcr, priv->mfunc.vhcr_dma); - priv->mfunc.vhcr = NULL; - -err_hcr: - if (!mlx4_is_slave(dev)) - iounmap(priv->cmd.hcr); - return -ENOMEM; -} - -void mlx4_multi_func_cleanup(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int i, port; - - if (mlx4_is_master(dev)) { - flush_workqueue(priv->mfunc.master.comm_wq); - destroy_workqueue(priv->mfunc.master.comm_wq); - for (i = 0; i < dev->num_slaves; i++) { - for (port = 1; port <= MLX4_MAX_PORTS; port++) - kfree(priv->mfunc.master.slave_state[i].vlan_filter[port]); - } - kfree(priv->mfunc.master.slave_state); - kfree(priv->mfunc.master.vf_admin); - kfree(priv->mfunc.master.vf_oper); - } - - iounmap(priv->mfunc.comm); -} - -void mlx4_cmd_cleanup(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - pci_pool_destroy(priv->cmd.pool); - - if (!mlx4_is_slave(dev)) - iounmap(priv->cmd.hcr); - if (mlx4_is_mfunc(dev)) - dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, - priv->mfunc.vhcr, priv->mfunc.vhcr_dma); - priv->mfunc.vhcr = NULL; -} - -/* - * Switch to using events to issue FW commands (can only be called - * after event queue for command events has been initialized). - */ -int mlx4_cmd_use_events(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int i; - int err = 0; - - priv->cmd.context = kmalloc(priv->cmd.max_cmds * - sizeof (struct mlx4_cmd_context), - GFP_KERNEL); - if (!priv->cmd.context) - return -ENOMEM; - - for (i = 0; i < priv->cmd.max_cmds; ++i) { - priv->cmd.context[i].token = i; - priv->cmd.context[i].next = i + 1; - } - - priv->cmd.context[priv->cmd.max_cmds - 1].next = -1; - priv->cmd.free_head = 0; - - sema_init(&priv->cmd.event_sem, priv->cmd.max_cmds); - spin_lock_init(&priv->cmd.context_lock); - - for (priv->cmd.token_mask = 1; - priv->cmd.token_mask < priv->cmd.max_cmds; - priv->cmd.token_mask <<= 1) - ; /* nothing */ - --priv->cmd.token_mask; - - down(&priv->cmd.poll_sem); - priv->cmd.use_events = 1; - - return err; -} - -/* - * Switch back to polling (used when shutting down the device) - */ -void mlx4_cmd_use_polling(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int i; - - priv->cmd.use_events = 0; - - for (i = 0; i < priv->cmd.max_cmds; ++i) - down(&priv->cmd.event_sem); - - kfree(priv->cmd.context); - - up(&priv->cmd.poll_sem); -} - -struct mlx4_cmd_mailbox *mlx4_alloc_cmd_mailbox(struct mlx4_dev *dev) -{ - struct mlx4_cmd_mailbox *mailbox; - - mailbox = kmalloc(sizeof *mailbox, GFP_KERNEL); - if (!mailbox) - return ERR_PTR(-ENOMEM); - - mailbox->buf = pci_pool_alloc(mlx4_priv(dev)->cmd.pool, GFP_KERNEL, - &mailbox->dma); - if (!mailbox->buf) { - kfree(mailbox); - return ERR_PTR(-ENOMEM); - } - - memset(mailbox->buf, 0, MLX4_MAILBOX_SIZE); - - return mailbox; -} -EXPORT_SYMBOL_GPL(mlx4_alloc_cmd_mailbox); - -void mlx4_free_cmd_mailbox(struct mlx4_dev *dev, - struct mlx4_cmd_mailbox *mailbox) -{ - if (!mailbox) - return; - - pci_pool_free(mlx4_priv(dev)->cmd.pool, mailbox->buf, mailbox->dma); - kfree(mailbox); -} -EXPORT_SYMBOL_GPL(mlx4_free_cmd_mailbox); - -u32 mlx4_comm_get_version(void) -{ - return ((u32) CMD_CHAN_IF_REV << 8) | (u32) CMD_CHAN_VER; -} - -static int mlx4_get_slave_indx(struct mlx4_dev *dev, int vf) -{ - if ((vf < 0) || (vf >= dev->num_vfs)) { - mlx4_err(dev, "Bad vf number:%d (number of activated vf: %d)\n", vf, dev->num_vfs); - return -EINVAL; - } - return (vf+1); -} - -int mlx4_set_vf_mac(struct mlx4_dev *dev, int port, int vf, u8 *mac) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_vport_state *s_info; - int slave; - - if (!mlx4_is_master(dev)) - return -EPROTONOSUPPORT; - - slave = mlx4_get_slave_indx(dev, vf); - if (slave < 0) - return -EINVAL; - - s_info = &priv->mfunc.master.vf_admin[slave].vport[port]; - s_info->mac = mlx4_mac_to_u64(mac); - mlx4_info(dev, "default mac on vf %d port %d to %llX will take afect only after vf restart\n", - vf, port, (unsigned long long) s_info->mac); - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_set_vf_mac); - -int mlx4_set_vf_vlan(struct mlx4_dev *dev, int port, int vf, u16 vlan, u8 qos) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_vport_oper_state *vf_oper; - struct mlx4_vport_state *vf_admin; - int slave; - - if ((!mlx4_is_master(dev)) || - !(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_VLAN_CONTROL)) - return -EPROTONOSUPPORT; - - if ((vlan > 4095) || (qos > 7)) - return -EINVAL; - - slave = mlx4_get_slave_indx(dev, vf); - if (slave < 0) - return -EINVAL; - - vf_admin = &priv->mfunc.master.vf_admin[slave].vport[port]; - vf_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; - - if ((0 == vlan) && (0 == qos)) - vf_admin->default_vlan = MLX4_VGT; - else - vf_admin->default_vlan = vlan; - vf_admin->default_qos = qos; - - if (priv->mfunc.master.slave_state[slave].active && - dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_UPDATE_QP) { - mlx4_info(dev, "updating vf %d port %d config params immediately\n", - vf, port); - mlx4_master_immediate_activate_vlan_qos(priv, slave, port); - } - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_set_vf_vlan); - - /* mlx4_get_slave_default_vlan - - * retrun true if VST ( default vlan) - * if VST will fill vlan & qos (if not NULL) */ -bool mlx4_get_slave_default_vlan(struct mlx4_dev *dev, int port, int slave, u16 *vlan, u8 *qos) -{ - struct mlx4_vport_oper_state *vp_oper; - struct mlx4_priv *priv; - - priv = mlx4_priv(dev); - vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; - - if (MLX4_VGT != vp_oper->state.default_vlan) { - if (vlan) - *vlan = vp_oper->state.default_vlan; - if (qos) - *qos = vp_oper->state.default_qos; - return true; - } - return false; -} -EXPORT_SYMBOL_GPL(mlx4_get_slave_default_vlan); - -int mlx4_set_vf_spoofchk(struct mlx4_dev *dev, int port, int vf, bool setting) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_vport_state *s_info; - int slave; - - if ((!mlx4_is_master(dev)) || - !(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FSM)) - return -EPROTONOSUPPORT; - - slave = mlx4_get_slave_indx(dev, vf); - if (slave < 0) - return -EINVAL; - - s_info = &priv->mfunc.master.vf_admin[slave].vport[port]; - s_info->spoofchk = setting; - - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_set_vf_spoofchk); - -int mlx4_set_vf_link_state(struct mlx4_dev *dev, int port, int vf, int link_state) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_vport_state *s_info; - struct mlx4_vport_oper_state *vp_oper; - int slave; - u8 link_stat_event; - - slave = mlx4_get_slave_indx(dev, vf); - if (slave < 0) - return -EINVAL; - - switch (link_state) { - case IFLA_VF_LINK_STATE_AUTO: - /* get link curent state */ - if (!priv->sense.do_sense_port[port]) - link_stat_event = MLX4_PORT_CHANGE_SUBTYPE_ACTIVE; - else - link_stat_event = MLX4_PORT_CHANGE_SUBTYPE_DOWN; - break; - - case IFLA_VF_LINK_STATE_ENABLE: - link_stat_event = MLX4_PORT_CHANGE_SUBTYPE_ACTIVE; - break; - - case IFLA_VF_LINK_STATE_DISABLE: - link_stat_event = MLX4_PORT_CHANGE_SUBTYPE_DOWN; - break; - - default: - mlx4_warn(dev, "unknown value for link_state %02x on slave %d port %d\n", - link_state, slave, port); - return -EINVAL; - } - /* update the admin & oper state on the link state */ - s_info = &priv->mfunc.master.vf_admin[slave].vport[port]; - vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; - s_info->link_state = link_state; - vp_oper->state.link_state = link_state; - - /* send event */ - mlx4_gen_port_state_change_eqe(dev, slave, port, link_stat_event); - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_set_vf_link_state); - -int mlx4_get_vf_link_state(struct mlx4_dev *dev, int port, int vf) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_vport_state *s_info; - int slave; - - if (!mlx4_is_master(dev)) - return -EPROTONOSUPPORT; - - slave = mlx4_get_slave_indx(dev, vf); - if (slave < 0) - return -EINVAL; - - s_info = &priv->mfunc.master.vf_admin[slave].vport[port]; - - return s_info->link_state; -} -EXPORT_SYMBOL_GPL(mlx4_get_vf_link_state); - Property changes on: stable/11/sys/ofed/drivers/net/mlx4/cmd.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/en_tx.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/en_tx.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/en_tx.c (nonexistent) @@ -1,1102 +0,0 @@ -/* - * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#define LINUXKPI_PARAM_PREFIX mlx4_ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "mlx4_en.h" - -enum { - MAX_INLINE = 104, /* 128 - 16 - 4 - 4 */ - MAX_BF = 256, - MIN_PKT_LEN = 17, -}; - -static int inline_thold __read_mostly = MAX_INLINE; - -module_param_named(inline_thold, inline_thold, uint, 0444); -MODULE_PARM_DESC(inline_thold, "threshold for using inline data"); - -int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring **pring, u32 size, - u16 stride, int node, int queue_idx) -{ - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_tx_ring *ring; - uint32_t x; - int tmp; - int err; - - ring = kzalloc_node(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL, node); - if (!ring) { - ring = kzalloc(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL); - if (!ring) { - en_err(priv, "Failed allocating TX ring\n"); - return -ENOMEM; - } - } - - /* Create DMA descriptor TAG */ - if ((err = -bus_dma_tag_create( - bus_get_dma_tag(mdev->pdev->dev.bsddev), - 1, /* any alignment */ - 0, /* no boundary */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MLX4_EN_TX_MAX_PAYLOAD_SIZE, /* maxsize */ - MLX4_EN_TX_MAX_MBUF_FRAGS, /* nsegments */ - MLX4_EN_TX_MAX_MBUF_SIZE, /* maxsegsize */ - 0, /* flags */ - NULL, NULL, /* lockfunc, lockfuncarg */ - &ring->dma_tag))) - goto done; - - ring->size = size; - ring->size_mask = size - 1; - ring->stride = stride; - ring->inline_thold = MAX(MIN_PKT_LEN, MIN(inline_thold, MAX_INLINE)); - mtx_init(&ring->tx_lock.m, "mlx4 tx", NULL, MTX_DEF); - mtx_init(&ring->comp_lock.m, "mlx4 comp", NULL, MTX_DEF); - - /* Allocate the buf ring */ - ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF, - M_WAITOK, &ring->tx_lock.m); - if (ring->br == NULL) { - en_err(priv, "Failed allocating tx_info ring\n"); - err = -ENOMEM; - goto err_free_dma_tag; - } - - tmp = size * sizeof(struct mlx4_en_tx_info); - ring->tx_info = kzalloc_node(tmp, GFP_KERNEL, node); - if (!ring->tx_info) { - ring->tx_info = kzalloc(tmp, GFP_KERNEL); - if (!ring->tx_info) { - err = -ENOMEM; - goto err_ring; - } - } - - /* Create DMA descriptor MAPs */ - for (x = 0; x != size; x++) { - err = -bus_dmamap_create(ring->dma_tag, 0, - &ring->tx_info[x].dma_map); - if (err != 0) { - while (x--) { - bus_dmamap_destroy(ring->dma_tag, - ring->tx_info[x].dma_map); - } - goto err_info; - } - } - - en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n", - ring->tx_info, tmp); - - ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE); - - /* Allocate HW buffers on provided NUMA node */ - err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size, - 2 * PAGE_SIZE); - if (err) { - en_err(priv, "Failed allocating hwq resources\n"); - goto err_dma_map; - } - - err = mlx4_en_map_buffer(&ring->wqres.buf); - if (err) { - en_err(priv, "Failed to map TX buffer\n"); - goto err_hwq_res; - } - - ring->buf = ring->wqres.buf.direct.buf; - - en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d " - "buf_size:%d dma:%llx\n", ring, ring->buf, ring->size, - ring->buf_size, (unsigned long long) ring->wqres.buf.direct.map); - - err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn, - MLX4_RESERVE_BF_QP); - if (err) { - en_err(priv, "failed reserving qp for TX ring\n"); - goto err_map; - } - - err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp); - if (err) { - en_err(priv, "Failed allocating qp %d\n", ring->qpn); - goto err_reserve; - } - ring->qp.event = mlx4_en_sqp_event; - - err = mlx4_bf_alloc(mdev->dev, &ring->bf, node); - if (err) { - en_dbg(DRV, priv, "working without blueflame (%d)", err); - ring->bf.uar = &mdev->priv_uar; - ring->bf.uar->map = mdev->uar_map; - ring->bf_enabled = false; - } else - ring->bf_enabled = true; - ring->queue_index = queue_idx; - if (queue_idx < priv->num_tx_rings_p_up ) - CPU_SET(queue_idx, &ring->affinity_mask); - - *pring = ring; - return 0; - -err_reserve: - mlx4_qp_release_range(mdev->dev, ring->qpn, 1); -err_map: - mlx4_en_unmap_buffer(&ring->wqres.buf); -err_hwq_res: - mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); -err_dma_map: - for (x = 0; x != size; x++) - bus_dmamap_destroy(ring->dma_tag, ring->tx_info[x].dma_map); -err_info: - vfree(ring->tx_info); -err_ring: - buf_ring_free(ring->br, M_DEVBUF); -err_free_dma_tag: - bus_dma_tag_destroy(ring->dma_tag); -done: - kfree(ring); - return err; -} - -void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring **pring) -{ - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_tx_ring *ring = *pring; - uint32_t x; - en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn); - - buf_ring_free(ring->br, M_DEVBUF); - if (ring->bf_enabled) - mlx4_bf_free(mdev->dev, &ring->bf); - mlx4_qp_remove(mdev->dev, &ring->qp); - mlx4_qp_free(mdev->dev, &ring->qp); - mlx4_qp_release_range(priv->mdev->dev, ring->qpn, 1); - mlx4_en_unmap_buffer(&ring->wqres.buf); - mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); - for (x = 0; x != ring->size; x++) - bus_dmamap_destroy(ring->dma_tag, ring->tx_info[x].dma_map); - vfree(ring->tx_info); - mtx_destroy(&ring->tx_lock.m); - mtx_destroy(&ring->comp_lock.m); - bus_dma_tag_destroy(ring->dma_tag); - kfree(ring); - *pring = NULL; -} - -int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring *ring, - int cq, int user_prio) -{ - struct mlx4_en_dev *mdev = priv->mdev; - int err; - - ring->cqn = cq; - ring->prod = 0; - ring->cons = 0xffffffff; - ring->last_nr_txbb = 1; - ring->poll_cnt = 0; - ring->blocked = 0; - memset(ring->buf, 0, ring->buf_size); - - ring->qp_state = MLX4_QP_STATE_RST; - ring->doorbell_qpn = ring->qp.qpn << 8; - - mlx4_en_fill_qp_context(priv, ring->size, ring->stride, 1, 0, ring->qpn, - ring->cqn, user_prio, &ring->context); - if (ring->bf_enabled) - ring->context.usr_page = cpu_to_be32(ring->bf.uar->index); - - err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, &ring->context, - &ring->qp, &ring->qp_state); - return err; -} - -void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring *ring) -{ - struct mlx4_en_dev *mdev = priv->mdev; - - mlx4_qp_modify(mdev->dev, NULL, ring->qp_state, - MLX4_QP_STATE_RST, NULL, 0, 0, &ring->qp); -} - -static volatile struct mlx4_wqe_data_seg * -mlx4_en_store_inline_lso_data(volatile struct mlx4_wqe_data_seg *dseg, - struct mbuf *mb, int len, __be32 owner_bit) -{ - uint8_t *inl = __DEVOLATILE(uint8_t *, dseg); - - /* copy data into place */ - m_copydata(mb, 0, len, inl + 4); - dseg += DIV_ROUND_UP(4 + len, DS_SIZE_ALIGNMENT); - return (dseg); -} - -static void -mlx4_en_store_inline_lso_header(volatile struct mlx4_wqe_data_seg *dseg, - int len, __be32 owner_bit) -{ -} - -static void -mlx4_en_stamp_wqe(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring *ring, u32 index, u8 owner) -{ - struct mlx4_en_tx_info *tx_info = &ring->tx_info[index]; - struct mlx4_en_tx_desc *tx_desc = (struct mlx4_en_tx_desc *) - (ring->buf + (index * TXBB_SIZE)); - volatile __be32 *ptr = (__be32 *)tx_desc; - const __be32 stamp = cpu_to_be32(STAMP_VAL | - ((u32)owner << STAMP_SHIFT)); - u32 i; - - /* Stamp the freed descriptor */ - for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) { - *ptr = stamp; - ptr += STAMP_DWORDS; - } -} - -static u32 -mlx4_en_free_tx_desc(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring *ring, u32 index) -{ - struct mlx4_en_tx_info *tx_info; - struct mbuf *mb; - - tx_info = &ring->tx_info[index]; - mb = tx_info->mb; - - if (mb == NULL) - goto done; - - bus_dmamap_sync(ring->dma_tag, tx_info->dma_map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(ring->dma_tag, tx_info->dma_map); - - m_freem(mb); -done: - return (tx_info->nr_txbb); -} - -int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - int cnt = 0; - - /* Skip last polled descriptor */ - ring->cons += ring->last_nr_txbb; - en_dbg(DRV, priv, "Freeing Tx buf - cons:0x%x prod:0x%x\n", - ring->cons, ring->prod); - - if ((u32) (ring->prod - ring->cons) > ring->size) { - en_warn(priv, "Tx consumer passed producer!\n"); - return 0; - } - - while (ring->cons != ring->prod) { - ring->last_nr_txbb = mlx4_en_free_tx_desc(priv, ring, - ring->cons & ring->size_mask); - ring->cons += ring->last_nr_txbb; - cnt++; - } - - if (cnt) - en_dbg(DRV, priv, "Freed %d uncompleted tx descriptors\n", cnt); - - return cnt; -} - -static bool -mlx4_en_tx_ring_is_full(struct mlx4_en_tx_ring *ring) -{ - int wqs; - wqs = ring->size - (ring->prod - ring->cons); - return (wqs < (HEADROOM + (2 * MLX4_EN_TX_WQE_MAX_WQEBBS))); -} - -static int mlx4_en_process_tx_cq(struct net_device *dev, - struct mlx4_en_cq *cq) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_cq *mcq = &cq->mcq; - struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring]; - struct mlx4_cqe *cqe; - u16 index; - u16 new_index, ring_index, stamp_index; - u32 txbbs_skipped = 0; - u32 txbbs_stamp = 0; - u32 cons_index = mcq->cons_index; - int size = cq->size; - u32 size_mask = ring->size_mask; - struct mlx4_cqe *buf = cq->buf; - int factor = priv->cqe_factor; - - if (!priv->port_up) - return 0; - - index = cons_index & size_mask; - cqe = &buf[(index << factor) + factor]; - ring_index = ring->cons & size_mask; - stamp_index = ring_index; - - /* Process all completed CQEs */ - while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, - cons_index & size)) { - /* - * make sure we read the CQE after we read the - * ownership bit - */ - rmb(); - - if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == - MLX4_CQE_OPCODE_ERROR)) { - en_err(priv, "CQE completed in error - vendor syndrom: 0x%x syndrom: 0x%x\n", - ((struct mlx4_err_cqe *)cqe)-> - vendor_err_syndrome, - ((struct mlx4_err_cqe *)cqe)->syndrome); - } - - /* Skip over last polled CQE */ - new_index = be16_to_cpu(cqe->wqe_index) & size_mask; - - do { - txbbs_skipped += ring->last_nr_txbb; - ring_index = (ring_index + ring->last_nr_txbb) & size_mask; - /* free next descriptor */ - ring->last_nr_txbb = mlx4_en_free_tx_desc( - priv, ring, ring_index); - mlx4_en_stamp_wqe(priv, ring, stamp_index, - !!((ring->cons + txbbs_stamp) & - ring->size)); - stamp_index = ring_index; - txbbs_stamp = txbbs_skipped; - } while (ring_index != new_index); - - ++cons_index; - index = cons_index & size_mask; - cqe = &buf[(index << factor) + factor]; - } - - - /* - * To prevent CQ overflow we first update CQ consumer and only then - * the ring consumer. - */ - mcq->cons_index = cons_index; - mlx4_cq_set_ci(mcq); - wmb(); - ring->cons += txbbs_skipped; - - /* Wakeup Tx queue if it was stopped and ring is not full */ - if (unlikely(ring->blocked) && !mlx4_en_tx_ring_is_full(ring)) { - ring->blocked = 0; - if (atomic_fetchadd_int(&priv->blocked, -1) == 1) - atomic_clear_int(&dev->if_drv_flags ,IFF_DRV_OACTIVE); - ring->wake_queue++; - priv->port_stats.wake_queue++; - } - return (0); -} - -void mlx4_en_tx_irq(struct mlx4_cq *mcq) -{ - struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq); - struct mlx4_en_priv *priv = netdev_priv(cq->dev); - struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring]; - - if (priv->port_up == 0 || !spin_trylock(&ring->comp_lock)) - return; - mlx4_en_process_tx_cq(cq->dev, cq); - mod_timer(&cq->timer, jiffies + 1); - spin_unlock(&ring->comp_lock); -} - -void mlx4_en_poll_tx_cq(unsigned long data) -{ - struct mlx4_en_cq *cq = (struct mlx4_en_cq *) data; - struct mlx4_en_priv *priv = netdev_priv(cq->dev); - struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring]; - u32 inflight; - - INC_PERF_COUNTER(priv->pstats.tx_poll); - - if (priv->port_up == 0) - return; - if (!spin_trylock(&ring->comp_lock)) { - mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT); - return; - } - mlx4_en_process_tx_cq(cq->dev, cq); - inflight = (u32) (ring->prod - ring->cons - ring->last_nr_txbb); - - /* If there are still packets in flight and the timer has not already - * been scheduled by the Tx routine then schedule it here to guarantee - * completion processing of these packets */ - if (inflight && priv->port_up) - mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT); - - spin_unlock(&ring->comp_lock); -} - -static inline void mlx4_en_xmit_poll(struct mlx4_en_priv *priv, int tx_ind) -{ - struct mlx4_en_cq *cq = priv->tx_cq[tx_ind]; - struct mlx4_en_tx_ring *ring = priv->tx_ring[tx_ind]; - - if (priv->port_up == 0) - return; - - /* If we don't have a pending timer, set one up to catch our recent - post in case the interface becomes idle */ - if (!timer_pending(&cq->timer)) - mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT); - - /* Poll the CQ every mlx4_en_TX_MODER_POLL packets */ - if ((++ring->poll_cnt & (MLX4_EN_TX_POLL_MODER - 1)) == 0) - if (spin_trylock(&ring->comp_lock)) { - mlx4_en_process_tx_cq(priv->dev, cq); - spin_unlock(&ring->comp_lock); - } -} - -static u16 -mlx4_en_get_inline_hdr_size(struct mlx4_en_tx_ring *ring, struct mbuf *mb) -{ - u16 retval; - - /* only copy from first fragment, if possible */ - retval = MIN(ring->inline_thold, mb->m_len); - - /* check for too little data */ - if (unlikely(retval < MIN_PKT_LEN)) - retval = MIN(ring->inline_thold, mb->m_pkthdr.len); - return (retval); -} - -static int -mlx4_en_get_header_size(struct mbuf *mb) -{ - struct ether_vlan_header *eh; - struct tcphdr *th; - struct ip *ip; - int ip_hlen, tcp_hlen; - struct ip6_hdr *ip6; - uint16_t eth_type; - int eth_hdr_len; - - eh = mtod(mb, struct ether_vlan_header *); - if (mb->m_len < ETHER_HDR_LEN) - return (0); - if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { - eth_type = ntohs(eh->evl_proto); - eth_hdr_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; - } else { - eth_type = ntohs(eh->evl_encap_proto); - eth_hdr_len = ETHER_HDR_LEN; - } - if (mb->m_len < eth_hdr_len) - return (0); - switch (eth_type) { - case ETHERTYPE_IP: - ip = (struct ip *)(mb->m_data + eth_hdr_len); - if (mb->m_len < eth_hdr_len + sizeof(*ip)) - return (0); - if (ip->ip_p != IPPROTO_TCP) - return (0); - ip_hlen = ip->ip_hl << 2; - eth_hdr_len += ip_hlen; - break; - case ETHERTYPE_IPV6: - ip6 = (struct ip6_hdr *)(mb->m_data + eth_hdr_len); - if (mb->m_len < eth_hdr_len + sizeof(*ip6)) - return (0); - if (ip6->ip6_nxt != IPPROTO_TCP) - return (0); - eth_hdr_len += sizeof(*ip6); - break; - default: - return (0); - } - if (mb->m_len < eth_hdr_len + sizeof(*th)) - return (0); - th = (struct tcphdr *)(mb->m_data + eth_hdr_len); - tcp_hlen = th->th_off << 2; - eth_hdr_len += tcp_hlen; - if (mb->m_len < eth_hdr_len) - return (0); - return (eth_hdr_len); -} - -static volatile struct mlx4_wqe_data_seg * -mlx4_en_store_inline_data(volatile struct mlx4_wqe_data_seg *dseg, - struct mbuf *mb, int len, __be32 owner_bit) -{ - uint8_t *inl = __DEVOLATILE(uint8_t *, dseg); - const int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - 4; - - if (unlikely(len < MIN_PKT_LEN)) { - m_copydata(mb, 0, len, inl + 4); - memset(inl + 4 + len, 0, MIN_PKT_LEN - len); - dseg += DIV_ROUND_UP(4 + MIN_PKT_LEN, DS_SIZE_ALIGNMENT); - } else if (len <= spc) { - m_copydata(mb, 0, len, inl + 4); - dseg += DIV_ROUND_UP(4 + len, DS_SIZE_ALIGNMENT); - } else { - m_copydata(mb, 0, spc, inl + 4); - m_copydata(mb, spc, len - spc, inl + 8 + spc); - dseg += DIV_ROUND_UP(8 + len, DS_SIZE_ALIGNMENT); - } - return (dseg); -} - -static void -mlx4_en_store_inline_header(volatile struct mlx4_wqe_data_seg *dseg, - int len, __be32 owner_bit) -{ - uint8_t *inl = __DEVOLATILE(uint8_t *, dseg); - const int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - 4; - - if (unlikely(len < MIN_PKT_LEN)) { - *(volatile uint32_t *)inl = - SET_BYTE_COUNT((1 << 31) | MIN_PKT_LEN); - } else if (len <= spc) { - *(volatile uint32_t *)inl = - SET_BYTE_COUNT((1 << 31) | len); - } else { - *(volatile uint32_t *)(inl + 4 + spc) = - SET_BYTE_COUNT((1 << 31) | (len - spc)); - wmb(); - *(volatile uint32_t *)inl = - SET_BYTE_COUNT((1 << 31) | spc); - } -} - -static uint32_t hashrandom; -static void hashrandom_init(void *arg) -{ - /* - * It is assumed that the random subsystem has been - * initialized when this function is called: - */ - hashrandom = m_ether_tcpip_hash_init(); -} -SYSINIT(hashrandom_init, SI_SUB_RANDOM, SI_ORDER_ANY, &hashrandom_init, NULL); - -u16 mlx4_en_select_queue(struct net_device *dev, struct mbuf *mb) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - u32 rings_p_up = priv->num_tx_rings_p_up; - u32 up = 0; - u32 queue_index; - -#if (MLX4_EN_NUM_UP > 1) - /* Obtain VLAN information if present */ - if (mb->m_flags & M_VLANTAG) { - u32 vlan_tag = mb->m_pkthdr.ether_vtag; - up = (vlan_tag >> 13) % MLX4_EN_NUM_UP; - } -#endif - queue_index = m_ether_tcpip_hash(MBUF_HASHFLAG_L3 | MBUF_HASHFLAG_L4, mb, hashrandom); - - return ((queue_index % rings_p_up) + (up * rings_p_up)); -} - -static void mlx4_bf_copy(void __iomem *dst, volatile unsigned long *src, unsigned bytecnt) -{ - __iowrite64_copy(dst, __DEVOLATILE(void *, src), bytecnt / 8); -} - -static int mlx4_en_xmit(struct mlx4_en_priv *priv, int tx_ind, struct mbuf **mbp) -{ - enum { - DS_FACT = TXBB_SIZE / DS_SIZE_ALIGNMENT, - CTRL_FLAGS = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE | - MLX4_WQE_CTRL_SOLICITED), - }; - bus_dma_segment_t segs[MLX4_EN_TX_MAX_MBUF_FRAGS]; - volatile struct mlx4_wqe_data_seg *dseg; - volatile struct mlx4_wqe_data_seg *dseg_inline; - volatile struct mlx4_en_tx_desc *tx_desc; - struct mlx4_en_tx_ring *ring = priv->tx_ring[tx_ind]; - struct ifnet *ifp = priv->dev; - struct mlx4_en_tx_info *tx_info; - struct mbuf *mb = *mbp; - struct mbuf *m; - __be32 owner_bit; - int nr_segs; - int pad; - int err; - u32 bf_size; - u32 bf_prod; - u32 opcode; - u16 index; - u16 ds_cnt; - u16 ihs; - - if (unlikely(!priv->port_up)) { - err = EINVAL; - goto tx_drop; - } - - /* check if TX ring is full */ - if (unlikely(mlx4_en_tx_ring_is_full(ring))) { - /* every full native Tx ring stops queue */ - if (ring->blocked == 0) - atomic_add_int(&priv->blocked, 1); - /* Set HW-queue-is-full flag */ - atomic_set_int(&ifp->if_drv_flags, IFF_DRV_OACTIVE); - priv->port_stats.queue_stopped++; - ring->blocked = 1; - ring->queue_stopped++; - - /* Use interrupts to find out when queue opened */ - mlx4_en_arm_cq(priv, priv->tx_cq[tx_ind]); - return (ENOBUFS); - } - - /* sanity check we are not wrapping around */ - KASSERT(((~ring->prod) & ring->size_mask) >= - (MLX4_EN_TX_WQE_MAX_WQEBBS - 1), ("Wrapping around TX ring")); - - /* Track current inflight packets for performance analysis */ - AVG_PERF_COUNTER(priv->pstats.inflight_avg, - (u32) (ring->prod - ring->cons - 1)); - - /* Track current mbuf packet header length */ - AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, mb->m_pkthdr.len); - - /* Grab an index and try to transmit packet */ - owner_bit = (ring->prod & ring->size) ? - cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0; - index = ring->prod & ring->size_mask; - tx_desc = (volatile struct mlx4_en_tx_desc *) - (ring->buf + index * TXBB_SIZE); - tx_info = &ring->tx_info[index]; - dseg = &tx_desc->data; - - /* send a copy of the frame to the BPF listener, if any */ - if (ifp != NULL && ifp->if_bpf != NULL) - ETHER_BPF_MTAP(ifp, mb); - - /* get default flags */ - tx_desc->ctrl.srcrb_flags = CTRL_FLAGS; - - if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) - tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM); - - if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | - CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) - tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_TCP_UDP_CSUM); - - /* do statistics */ - if (likely(tx_desc->ctrl.srcrb_flags != CTRL_FLAGS)) { - priv->port_stats.tx_chksum_offload++; - ring->tx_csum++; - } - - /* check for VLAN tag */ - if (mb->m_flags & M_VLANTAG) { - tx_desc->ctrl.vlan_tag = cpu_to_be16(mb->m_pkthdr.ether_vtag); - tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_VLAN; - } else { - tx_desc->ctrl.vlan_tag = 0; - tx_desc->ctrl.ins_vlan = 0; - } - - if (unlikely(mlx4_is_mfunc(priv->mdev->dev) || priv->validate_loopback)) { - /* - * Copy destination MAC address to WQE. This allows - * loopback in eSwitch, so that VFs and PF can - * communicate with each other: - */ - m_copydata(mb, 0, 2, __DEVOLATILE(void *, &tx_desc->ctrl.srcrb_flags16[0])); - m_copydata(mb, 2, 4, __DEVOLATILE(void *, &tx_desc->ctrl.imm)); - } else { - /* clear immediate field */ - tx_desc->ctrl.imm = 0; - } - - /* Handle LSO (TSO) packets */ - if (mb->m_pkthdr.csum_flags & CSUM_TSO) { - u32 payload_len; - u32 mss = mb->m_pkthdr.tso_segsz; - u32 num_pkts; - - opcode = cpu_to_be32(MLX4_OPCODE_LSO | MLX4_WQE_CTRL_RR) | - owner_bit; - ihs = mlx4_en_get_header_size(mb); - if (unlikely(ihs > MAX_INLINE)) { - ring->oversized_packets++; - err = EINVAL; - goto tx_drop; - } - tx_desc->lso.mss_hdr_size = cpu_to_be32((mss << 16) | ihs); - payload_len = mb->m_pkthdr.len - ihs; - if (unlikely(payload_len == 0)) - num_pkts = 1; - else - num_pkts = DIV_ROUND_UP(payload_len, mss); - ring->bytes += payload_len + (num_pkts * ihs); - ring->packets += num_pkts; - ring->tso_packets++; - /* store pointer to inline header */ - dseg_inline = dseg; - /* copy data inline */ - dseg = mlx4_en_store_inline_lso_data(dseg, - mb, ihs, owner_bit); - } else { - opcode = cpu_to_be32(MLX4_OPCODE_SEND) | - owner_bit; - ihs = mlx4_en_get_inline_hdr_size(ring, mb); - ring->bytes += max_t (unsigned int, - mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN); - ring->packets++; - /* store pointer to inline header */ - dseg_inline = dseg; - /* copy data inline */ - dseg = mlx4_en_store_inline_data(dseg, - mb, ihs, owner_bit); - } - m_adj(mb, ihs); - - err = bus_dmamap_load_mbuf_sg(ring->dma_tag, tx_info->dma_map, - mb, segs, &nr_segs, BUS_DMA_NOWAIT); - if (unlikely(err == EFBIG)) { - /* Too many mbuf fragments */ - ring->defrag_attempts++; - m = m_defrag(mb, M_NOWAIT); - if (m == NULL) { - ring->oversized_packets++; - goto tx_drop; - } - mb = m; - /* Try again */ - err = bus_dmamap_load_mbuf_sg(ring->dma_tag, tx_info->dma_map, - mb, segs, &nr_segs, BUS_DMA_NOWAIT); - } - /* catch errors */ - if (unlikely(err != 0)) { - ring->oversized_packets++; - goto tx_drop; - } - /* If there were no errors and we didn't load anything, don't sync. */ - if (nr_segs != 0) { - /* make sure all mbuf data is written to RAM */ - bus_dmamap_sync(ring->dma_tag, tx_info->dma_map, - BUS_DMASYNC_PREWRITE); - } else { - /* All data was inlined, free the mbuf. */ - bus_dmamap_unload(ring->dma_tag, tx_info->dma_map); - m_freem(mb); - mb = NULL; - } - - /* compute number of DS needed */ - ds_cnt = (dseg - ((volatile struct mlx4_wqe_data_seg *)tx_desc)) + nr_segs; - - /* - * Check if the next request can wrap around and fill the end - * of the current request with zero immediate data: - */ - pad = DIV_ROUND_UP(ds_cnt, DS_FACT); - pad = (~(ring->prod + pad)) & ring->size_mask; - - if (unlikely(pad < (MLX4_EN_TX_WQE_MAX_WQEBBS - 1))) { - /* - * Compute the least number of DS blocks we need to - * pad in order to achieve a TX ring wraparound: - */ - pad = (DS_FACT * (pad + 1)); - } else { - /* - * The hardware will automatically jump to the next - * TXBB. No need for padding. - */ - pad = 0; - } - - /* compute total number of DS blocks */ - ds_cnt += pad; - /* - * When modifying this code, please ensure that the following - * computation is always less than or equal to 0x3F: - * - * ((MLX4_EN_TX_WQE_MAX_WQEBBS - 1) * DS_FACT) + - * (MLX4_EN_TX_WQE_MAX_WQEBBS * DS_FACT) - * - * Else the "ds_cnt" variable can become too big. - */ - tx_desc->ctrl.fence_size = (ds_cnt & 0x3f); - - /* store pointer to mbuf */ - tx_info->mb = mb; - tx_info->nr_txbb = DIV_ROUND_UP(ds_cnt, DS_FACT); - bf_size = ds_cnt * DS_SIZE_ALIGNMENT; - bf_prod = ring->prod; - - /* compute end of "dseg" array */ - dseg += nr_segs + pad; - - /* pad using zero immediate dseg */ - while (pad--) { - dseg--; - dseg->addr = 0; - dseg->lkey = 0; - wmb(); - dseg->byte_count = SET_BYTE_COUNT((1 << 31)|0); - } - - /* fill segment list */ - while (nr_segs--) { - if (unlikely(segs[nr_segs].ds_len == 0)) { - dseg--; - dseg->addr = 0; - dseg->lkey = 0; - wmb(); - dseg->byte_count = SET_BYTE_COUNT((1 << 31)|0); - } else { - dseg--; - dseg->addr = cpu_to_be64((uint64_t)segs[nr_segs].ds_addr); - dseg->lkey = cpu_to_be32(priv->mdev->mr.key); - wmb(); - dseg->byte_count = SET_BYTE_COUNT((uint32_t)segs[nr_segs].ds_len); - } - } - - wmb(); - - /* write owner bits in reverse order */ - if ((opcode & cpu_to_be32(0x1F)) == cpu_to_be32(MLX4_OPCODE_LSO)) - mlx4_en_store_inline_lso_header(dseg_inline, ihs, owner_bit); - else - mlx4_en_store_inline_header(dseg_inline, ihs, owner_bit); - - /* update producer counter */ - ring->prod += tx_info->nr_txbb; - - if (ring->bf_enabled && bf_size <= MAX_BF && - (tx_desc->ctrl.ins_vlan != MLX4_WQE_CTRL_INS_VLAN)) { - - /* store doorbell number */ - *(volatile __be32 *) (&tx_desc->ctrl.vlan_tag) |= cpu_to_be32(ring->doorbell_qpn); - - /* or in producer number for this WQE */ - opcode |= cpu_to_be32((bf_prod & 0xffff) << 8); - - /* - * Ensure the new descriptor hits memory before - * setting ownership of this descriptor to HW: - */ - wmb(); - tx_desc->ctrl.owner_opcode = opcode; - wmb(); - mlx4_bf_copy(((u8 *)ring->bf.reg) + ring->bf.offset, - (volatile unsigned long *) &tx_desc->ctrl, bf_size); - wmb(); - ring->bf.offset ^= ring->bf.buf_size; - } else { - /* - * Ensure the new descriptor hits memory before - * setting ownership of this descriptor to HW: - */ - wmb(); - tx_desc->ctrl.owner_opcode = opcode; - wmb(); - writel(cpu_to_be32(ring->doorbell_qpn), - ((u8 *)ring->bf.uar->map) + MLX4_SEND_DOORBELL); - } - - return (0); -tx_drop: - *mbp = NULL; - m_freem(mb); - return (err); -} - -static int -mlx4_en_transmit_locked(struct ifnet *dev, int tx_ind, struct mbuf *m) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_tx_ring *ring; - struct mbuf *next; - int enqueued, err = 0; - - ring = priv->tx_ring[tx_ind]; - if ((dev->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != - IFF_DRV_RUNNING || priv->port_up == 0) { - if (m != NULL) - err = drbr_enqueue(dev, ring->br, m); - return (err); - } - - enqueued = 0; - if (m != NULL) - /* - * If we can't insert mbuf into drbr, try to xmit anyway. - * We keep the error we got so we could return that after xmit. - */ - err = drbr_enqueue(dev, ring->br, m); - - /* Process the queue */ - while ((next = drbr_peek(dev, ring->br)) != NULL) { - if (mlx4_en_xmit(priv, tx_ind, &next) != 0) { - if (next == NULL) { - drbr_advance(dev, ring->br); - } else { - drbr_putback(dev, ring->br, next); - } - break; - } - drbr_advance(dev, ring->br); - enqueued++; - if ((dev->if_drv_flags & IFF_DRV_RUNNING) == 0) - break; - } - - if (enqueued > 0) - ring->watchdog_time = ticks; - - return (err); -} - -void -mlx4_en_tx_que(void *context, int pending) -{ - struct mlx4_en_tx_ring *ring; - struct mlx4_en_priv *priv; - struct net_device *dev; - struct mlx4_en_cq *cq; - int tx_ind; - cq = context; - dev = cq->dev; - priv = dev->if_softc; - tx_ind = cq->ring; - ring = priv->tx_ring[tx_ind]; - - if (priv->port_up != 0 && - (dev->if_drv_flags & IFF_DRV_RUNNING) != 0) { - mlx4_en_xmit_poll(priv, tx_ind); - spin_lock(&ring->tx_lock); - if (!drbr_empty(dev, ring->br)) - mlx4_en_transmit_locked(dev, tx_ind, NULL); - spin_unlock(&ring->tx_lock); - } -} - -int -mlx4_en_transmit(struct ifnet *dev, struct mbuf *m) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_tx_ring *ring; - struct mlx4_en_cq *cq; - int i, err = 0; - - if (priv->port_up == 0) { - m_freem(m); - return (ENETDOWN); - } - - /* Compute which queue to use */ - if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { - i = (m->m_pkthdr.flowid % 128) % priv->tx_ring_num; - } - else { - i = mlx4_en_select_queue(dev, m); - } - - ring = priv->tx_ring[i]; - if (spin_trylock(&ring->tx_lock)) { - err = mlx4_en_transmit_locked(dev, i, m); - spin_unlock(&ring->tx_lock); - /* Poll CQ here */ - mlx4_en_xmit_poll(priv, i); - } else { - err = drbr_enqueue(dev, ring->br, m); - cq = priv->tx_cq[i]; - taskqueue_enqueue(cq->tq, &cq->cq_task); - } - -#if __FreeBSD_version >= 1100000 - if (unlikely(err != 0)) - if_inc_counter(dev, IFCOUNTER_IQDROPS, 1); -#endif - return (err); -} - -/* - * Flush ring buffers. - */ -void -mlx4_en_qflush(struct ifnet *dev) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_tx_ring *ring; - struct mbuf *m; - - if (priv->port_up == 0) - return; - - for (int i = 0; i < priv->tx_ring_num; i++) { - ring = priv->tx_ring[i]; - spin_lock(&ring->tx_lock); - while ((m = buf_ring_dequeue_sc(ring->br)) != NULL) - m_freem(m); - spin_unlock(&ring->tx_lock); - } - if_qflush(dev); -} Property changes on: stable/11/sys/ofed/drivers/net/mlx4/en_tx.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/en_rx.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/en_rx.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/en_rx.c (nonexistent) @@ -1,928 +0,0 @@ -/* - * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ -#include "opt_inet.h" -#include -#include -#include -#include -#include -#include -#include -#ifdef CONFIG_NET_RX_BUSY_POLL -#include -#endif - -#include "mlx4_en.h" - - -static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring, - int index) -{ - struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) - (ring->buf + (ring->stride * index)); - int possible_frags; - int i; - - /* Set size and memtype fields */ - rx_desc->data[0].byte_count = cpu_to_be32(priv->rx_mb_size - MLX4_NET_IP_ALIGN); - rx_desc->data[0].lkey = cpu_to_be32(priv->mdev->mr.key); - - /* - * If the number of used fragments does not fill up the ring - * stride, remaining (unused) fragments must be padded with - * null address/size and a special memory key: - */ - possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE; - for (i = 1; i < possible_frags; i++) { - rx_desc->data[i].byte_count = 0; - rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); - rx_desc->data[i].addr = 0; - } -} - -static int -mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, - __be64 *pdma, struct mlx4_en_rx_mbuf *mb_list) -{ - bus_dma_segment_t segs[1]; - bus_dmamap_t map; - struct mbuf *mb; - int nsegs; - int err; - - /* try to allocate a new spare mbuf */ - if (unlikely(ring->spare.mbuf == NULL)) { - mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); - if (unlikely(mb == NULL)) - return (-ENOMEM); - /* setup correct length */ - mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size; - - /* make sure IP header gets aligned */ - m_adj(mb, MLX4_NET_IP_ALIGN); - - /* load spare mbuf into BUSDMA */ - err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, ring->spare.dma_map, - mb, segs, &nsegs, BUS_DMA_NOWAIT); - if (unlikely(err != 0)) { - m_freem(mb); - return (err); - } - - /* store spare info */ - ring->spare.mbuf = mb; - ring->spare.paddr_be = cpu_to_be64(segs[0].ds_addr); - - bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, - BUS_DMASYNC_PREREAD); - } - - /* synchronize and unload the current mbuf, if any */ - if (likely(mb_list->mbuf != NULL)) { - bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(ring->dma_tag, mb_list->dma_map); - } - - mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); - if (unlikely(mb == NULL)) - goto use_spare; - - /* setup correct length */ - mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size; - - /* make sure IP header gets aligned */ - m_adj(mb, MLX4_NET_IP_ALIGN); - - err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, mb_list->dma_map, - mb, segs, &nsegs, BUS_DMA_NOWAIT); - if (unlikely(err != 0)) { - m_freem(mb); - goto use_spare; - } - - *pdma = cpu_to_be64(segs[0].ds_addr); - mb_list->mbuf = mb; - - bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, BUS_DMASYNC_PREREAD); - return (0); - -use_spare: - /* swap DMA maps */ - map = mb_list->dma_map; - mb_list->dma_map = ring->spare.dma_map; - ring->spare.dma_map = map; - - /* swap MBUFs */ - mb_list->mbuf = ring->spare.mbuf; - ring->spare.mbuf = NULL; - - /* store physical address */ - *pdma = ring->spare.paddr_be; - return (0); -} - -static void -mlx4_en_free_buf(struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_mbuf *mb_list) -{ - bus_dmamap_t map = mb_list->dma_map; - bus_dmamap_sync(ring->dma_tag, map, BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(ring->dma_tag, map); - m_freem(mb_list->mbuf); - mb_list->mbuf = NULL; /* safety clearing */ -} - -static int -mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring, int index) -{ - struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) - (ring->buf + (index * ring->stride)); - struct mlx4_en_rx_mbuf *mb_list = ring->mbuf + index; - - mb_list->mbuf = NULL; - - if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) { - priv->port_stats.rx_alloc_failed++; - return (-ENOMEM); - } - return (0); -} - -static inline void -mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring) -{ - *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff); -} - -static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv) -{ - struct mlx4_en_rx_ring *ring; - int ring_ind; - int buf_ind; - int new_size; - int err; - - for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) { - for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { - ring = priv->rx_ring[ring_ind]; - - err = mlx4_en_prepare_rx_desc(priv, ring, - ring->actual_size); - if (err) { - if (ring->actual_size == 0) { - en_err(priv, "Failed to allocate " - "enough rx buffers\n"); - return -ENOMEM; - } else { - new_size = - rounddown_pow_of_two(ring->actual_size); - en_warn(priv, "Only %d buffers allocated " - "reducing ring size to %d\n", - ring->actual_size, new_size); - goto reduce_rings; - } - } - ring->actual_size++; - ring->prod++; - } - } - return 0; - -reduce_rings: - for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { - ring = priv->rx_ring[ring_ind]; - while (ring->actual_size > new_size) { - ring->actual_size--; - ring->prod--; - mlx4_en_free_buf(ring, - ring->mbuf + ring->actual_size); - } - } - - return 0; -} - -static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring) -{ - int index; - - en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n", - ring->cons, ring->prod); - - /* Unmap and free Rx buffers */ - BUG_ON((u32) (ring->prod - ring->cons) > ring->actual_size); - while (ring->cons != ring->prod) { - index = ring->cons & ring->size_mask; - en_dbg(DRV, priv, "Processing descriptor:%d\n", index); - mlx4_en_free_buf(ring, ring->mbuf + index); - ++ring->cons; - } -} - -void mlx4_en_calc_rx_buf(struct net_device *dev) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - int eff_mtu = dev->if_mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN + - MLX4_NET_IP_ALIGN; - - if (eff_mtu > MJUM16BYTES) { - en_err(priv, "MTU(%d) is too big\n", dev->if_mtu); - eff_mtu = MJUM16BYTES; - } else if (eff_mtu > MJUM9BYTES) { - eff_mtu = MJUM16BYTES; - } else if (eff_mtu > MJUMPAGESIZE) { - eff_mtu = MJUM9BYTES; - } else if (eff_mtu > MCLBYTES) { - eff_mtu = MJUMPAGESIZE; - } else { - eff_mtu = MCLBYTES; - } - - priv->rx_mb_size = eff_mtu; - - en_dbg(DRV, priv, "Effective RX MTU: %d bytes\n", eff_mtu); -} - -int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring **pring, - u32 size, int node) -{ - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_rx_ring *ring; - int err; - int tmp; - uint32_t x; - - ring = kzalloc(sizeof(struct mlx4_en_rx_ring), GFP_KERNEL); - if (!ring) { - en_err(priv, "Failed to allocate RX ring structure\n"); - return -ENOMEM; - } - - /* Create DMA descriptor TAG */ - if ((err = -bus_dma_tag_create( - bus_get_dma_tag(mdev->pdev->dev.bsddev), - 1, /* any alignment */ - 0, /* no boundary */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MJUM16BYTES, /* maxsize */ - 1, /* nsegments */ - MJUM16BYTES, /* maxsegsize */ - 0, /* flags */ - NULL, NULL, /* lockfunc, lockfuncarg */ - &ring->dma_tag))) { - en_err(priv, "Failed to create DMA tag\n"); - goto err_ring; - } - - ring->prod = 0; - ring->cons = 0; - ring->size = size; - ring->size_mask = size - 1; - ring->stride = roundup_pow_of_two( - sizeof(struct mlx4_en_rx_desc) + DS_SIZE); - ring->log_stride = ffs(ring->stride) - 1; - ring->buf_size = ring->size * ring->stride + TXBB_SIZE; - - tmp = size * sizeof(struct mlx4_en_rx_mbuf); - - ring->mbuf = kzalloc(tmp, GFP_KERNEL); - if (ring->mbuf == NULL) { - err = -ENOMEM; - goto err_dma_tag; - } - - err = -bus_dmamap_create(ring->dma_tag, 0, &ring->spare.dma_map); - if (err != 0) - goto err_info; - - for (x = 0; x != size; x++) { - err = -bus_dmamap_create(ring->dma_tag, 0, - &ring->mbuf[x].dma_map); - if (err != 0) { - while (x--) - bus_dmamap_destroy(ring->dma_tag, - ring->mbuf[x].dma_map); - goto err_info; - } - } - en_dbg(DRV, priv, "Allocated MBUF ring at addr:%p size:%d\n", - ring->mbuf, tmp); - - err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, - ring->buf_size, 2 * PAGE_SIZE); - if (err) - goto err_dma_map; - - err = mlx4_en_map_buffer(&ring->wqres.buf); - if (err) { - en_err(priv, "Failed to map RX buffer\n"); - goto err_hwq; - } - ring->buf = ring->wqres.buf.direct.buf; - *pring = ring; - return 0; - -err_hwq: - mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); -err_dma_map: - for (x = 0; x != size; x++) { - bus_dmamap_destroy(ring->dma_tag, - ring->mbuf[x].dma_map); - } - bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map); -err_info: - vfree(ring->mbuf); -err_dma_tag: - bus_dma_tag_destroy(ring->dma_tag); -err_ring: - kfree(ring); - return (err); -} - -int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv) -{ - struct mlx4_en_rx_ring *ring; - int i; - int ring_ind; - int err; - int stride = roundup_pow_of_two( - sizeof(struct mlx4_en_rx_desc) + DS_SIZE); - - for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { - ring = priv->rx_ring[ring_ind]; - - ring->prod = 0; - ring->cons = 0; - ring->actual_size = 0; - ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn; - ring->rx_mb_size = priv->rx_mb_size; - - ring->stride = stride; - if (ring->stride <= TXBB_SIZE) { - /* Stamp first unused send wqe */ - __be32 *ptr = (__be32 *)ring->buf; - __be32 stamp = cpu_to_be32(1 << STAMP_SHIFT); - *ptr = stamp; - /* Move pointer to start of rx section */ - ring->buf += TXBB_SIZE; - } - - ring->log_stride = ffs(ring->stride) - 1; - ring->buf_size = ring->size * ring->stride; - - memset(ring->buf, 0, ring->buf_size); - mlx4_en_update_rx_prod_db(ring); - - /* Initialize all descriptors */ - for (i = 0; i < ring->size; i++) - mlx4_en_init_rx_desc(priv, ring, i); - -#ifdef INET - /* Configure lro mngr */ - if (priv->dev->if_capenable & IFCAP_LRO) { - if (tcp_lro_init(&ring->lro)) - priv->dev->if_capenable &= ~IFCAP_LRO; - else - ring->lro.ifp = priv->dev; - } -#endif - } - - - err = mlx4_en_fill_rx_buffers(priv); - if (err) - goto err_buffers; - - for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { - ring = priv->rx_ring[ring_ind]; - - ring->size_mask = ring->actual_size - 1; - mlx4_en_update_rx_prod_db(ring); - } - - return 0; - -err_buffers: - for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) - mlx4_en_free_rx_buf(priv, priv->rx_ring[ring_ind]); - - ring_ind = priv->rx_ring_num - 1; - - while (ring_ind >= 0) { - ring = priv->rx_ring[ring_ind]; - if (ring->stride <= TXBB_SIZE) - ring->buf -= TXBB_SIZE; - ring_ind--; - } - - return err; -} - - -void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring **pring, - u32 size, u16 stride) -{ - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_rx_ring *ring = *pring; - uint32_t x; - - mlx4_en_unmap_buffer(&ring->wqres.buf); - mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE); - for (x = 0; x != size; x++) - bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map); - /* free spare mbuf, if any */ - if (ring->spare.mbuf != NULL) { - bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(ring->dma_tag, ring->spare.dma_map); - m_freem(ring->spare.mbuf); - } - bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map); - vfree(ring->mbuf); - bus_dma_tag_destroy(ring->dma_tag); - kfree(ring); - *pring = NULL; -#ifdef CONFIG_RFS_ACCEL - mlx4_en_cleanup_filters(priv, ring); -#endif -} - -void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring) -{ -#ifdef INET - tcp_lro_free(&ring->lro); -#endif - mlx4_en_free_rx_buf(priv, ring); - if (ring->stride <= TXBB_SIZE) - ring->buf -= TXBB_SIZE; -} - - -static void validate_loopback(struct mlx4_en_priv *priv, struct mbuf *mb) -{ - int i; - int offset = ETHER_HDR_LEN; - - for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++, offset++) { - if (*(mb->m_data + offset) != (unsigned char) (i & 0xff)) - goto out_loopback; - } - /* Loopback found */ - priv->loopback_ok = 1; - -out_loopback: - m_freem(mb); -} - - -static inline int invalid_cqe(struct mlx4_en_priv *priv, - struct mlx4_cqe *cqe) -{ - /* Drop packet on bad receive or bad checksum */ - if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == - MLX4_CQE_OPCODE_ERROR)) { - en_err(priv, "CQE completed in error - vendor syndrom:%d syndrom:%d\n", - ((struct mlx4_err_cqe *)cqe)->vendor_err_syndrome, - ((struct mlx4_err_cqe *)cqe)->syndrome); - return 1; - } - if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) { - en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n"); - return 1; - } - - return 0; -} - -static struct mbuf * -mlx4_en_rx_mb(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, - struct mlx4_en_rx_desc *rx_desc, struct mlx4_en_rx_mbuf *mb_list, - int length) -{ - struct mbuf *mb; - - /* get mbuf */ - mb = mb_list->mbuf; - - /* collect used fragment while atomically replacing it */ - if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) - return (NULL); - - /* range check hardware computed value */ - if (unlikely(length > mb->m_len)) - length = mb->m_len; - - /* update total packet length in packet header */ - mb->m_len = mb->m_pkthdr.len = length; - return (mb); -} - -/* For cpu arch with cache line of 64B the performance is better when cqe size==64B - * To enlarge cqe size from 32B to 64B --> 32B of garbage (i.e. 0xccccccc) - * was added in the beginning of each cqe (the real data is in the corresponding 32B). - * The following calc ensures that when factor==1, it means we are aligned to 64B - * and we get the real cqe data*/ -#define CQE_FACTOR_INDEX(index, factor) ((index << factor) + factor) -int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_cqe *cqe; - struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring]; - struct mlx4_en_rx_mbuf *mb_list; - struct mlx4_en_rx_desc *rx_desc; - struct mbuf *mb; - struct mlx4_cq *mcq = &cq->mcq; - struct mlx4_cqe *buf = cq->buf; - int index; - unsigned int length; - int polled = 0; - u32 cons_index = mcq->cons_index; - u32 size_mask = ring->size_mask; - int size = cq->size; - int factor = priv->cqe_factor; - - if (!priv->port_up) - return 0; - - /* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx - * descriptor offset can be deducted from the CQE index instead of - * reading 'cqe->index' */ - index = cons_index & size_mask; - cqe = &buf[CQE_FACTOR_INDEX(index, factor)]; - - /* Process all completed CQEs */ - while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, - cons_index & size)) { - mb_list = ring->mbuf + index; - rx_desc = (struct mlx4_en_rx_desc *) - (ring->buf + (index << ring->log_stride)); - - /* - * make sure we read the CQE after we read the ownership bit - */ - rmb(); - - if (invalid_cqe(priv, cqe)) { - goto next; - } - /* - * Packet is OK - process it. - */ - length = be32_to_cpu(cqe->byte_cnt); - length -= ring->fcs_del; - - mb = mlx4_en_rx_mb(priv, ring, rx_desc, mb_list, length); - if (unlikely(!mb)) { - ring->errors++; - goto next; - } - - ring->bytes += length; - ring->packets++; - - if (unlikely(priv->validate_loopback)) { - validate_loopback(priv, mb); - goto next; - } - - /* forward Toeplitz compatible hash value */ - mb->m_pkthdr.flowid = be32_to_cpu(cqe->immed_rss_invalid); - M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH); - mb->m_pkthdr.rcvif = dev; - if (be32_to_cpu(cqe->vlan_my_qpn) & - MLX4_CQE_VLAN_PRESENT_MASK) { - mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->sl_vid); - mb->m_flags |= M_VLANTAG; - } - if (likely(dev->if_capenable & - (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) && - (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) && - (cqe->checksum == cpu_to_be16(0xffff))) { - priv->port_stats.rx_chksum_good++; - mb->m_pkthdr.csum_flags = - CSUM_IP_CHECKED | CSUM_IP_VALID | - CSUM_DATA_VALID | CSUM_PSEUDO_HDR; - mb->m_pkthdr.csum_data = htons(0xffff); - /* This packet is eligible for LRO if it is: - * - DIX Ethernet (type interpretation) - * - TCP/IP (v4) - * - without IP options - * - not an IP fragment - */ -#ifdef INET - if (mlx4_en_can_lro(cqe->status) && - (dev->if_capenable & IFCAP_LRO)) { - if (ring->lro.lro_cnt != 0 && - tcp_lro_rx(&ring->lro, mb, 0) == 0) - goto next; - } - -#endif - /* LRO not possible, complete processing here */ - INC_PERF_COUNTER(priv->pstats.lro_misses); - } else { - mb->m_pkthdr.csum_flags = 0; - priv->port_stats.rx_chksum_none++; - } - - /* Push it up the stack */ - dev->if_input(dev, mb); - -next: - ++cons_index; - index = cons_index & size_mask; - cqe = &buf[CQE_FACTOR_INDEX(index, factor)]; - if (++polled == budget) - goto out; - } - /* Flush all pending IP reassembly sessions */ -out: -#ifdef INET - tcp_lro_flush_all(&ring->lro); -#endif - AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled); - mcq->cons_index = cons_index; - mlx4_cq_set_ci(mcq); - wmb(); /* ensure HW sees CQ consumer before we post new buffers */ - ring->cons = mcq->cons_index; - ring->prod += polled; /* Polled descriptors were realocated in place */ - mlx4_en_update_rx_prod_db(ring); - return polled; - -} - -/* Rx CQ polling - called by NAPI */ -static int mlx4_en_poll_rx_cq(struct mlx4_en_cq *cq, int budget) -{ - struct net_device *dev = cq->dev; - int done; - - done = mlx4_en_process_rx_cq(dev, cq, budget); - cq->tot_rx += done; - - return done; - -} -void mlx4_en_rx_irq(struct mlx4_cq *mcq) -{ - struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq); - struct mlx4_en_priv *priv = netdev_priv(cq->dev); - int done; - - // Shoot one within the irq context - // Because there is no NAPI in freeBSD - done = mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET); - if (priv->port_up && (done == MLX4_EN_RX_BUDGET) ) { - cq->curr_poll_rx_cpu_id = curcpu; - taskqueue_enqueue(cq->tq, &cq->cq_task); - } - else { - mlx4_en_arm_cq(priv, cq); - } -} - -void mlx4_en_rx_que(void *context, int pending) -{ - struct mlx4_en_cq *cq; - struct thread *td; - - cq = context; - td = curthread; - - thread_lock(td); - sched_bind(td, cq->curr_poll_rx_cpu_id); - thread_unlock(td); - - while (mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET) - == MLX4_EN_RX_BUDGET); - mlx4_en_arm_cq(cq->dev->if_softc, cq); -} - - -/* RSS related functions */ - -static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn, - struct mlx4_en_rx_ring *ring, - enum mlx4_qp_state *state, - struct mlx4_qp *qp) -{ - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_qp_context *context; - int err = 0; - - context = kmalloc(sizeof *context , GFP_KERNEL); - if (!context) { - en_err(priv, "Failed to allocate qp context\n"); - return -ENOMEM; - } - - err = mlx4_qp_alloc(mdev->dev, qpn, qp); - if (err) { - en_err(priv, "Failed to allocate qp #%x\n", qpn); - goto out; - } - qp->event = mlx4_en_sqp_event; - - memset(context, 0, sizeof *context); - mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0, - qpn, ring->cqn, -1, context); - context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma); - - /* Cancel FCS removal if FW allows */ - if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) { - context->param3 |= cpu_to_be32(1 << 29); - ring->fcs_del = ETH_FCS_LEN; - } else - ring->fcs_del = 0; - - err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state); - if (err) { - mlx4_qp_remove(mdev->dev, qp); - mlx4_qp_free(mdev->dev, qp); - } - mlx4_en_update_rx_prod_db(ring); -out: - kfree(context); - return err; -} - -int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv) -{ - int err; - u32 qpn; - - err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn, 0); - if (err) { - en_err(priv, "Failed reserving drop qpn\n"); - return err; - } - err = mlx4_qp_alloc(priv->mdev->dev, qpn, &priv->drop_qp); - if (err) { - en_err(priv, "Failed allocating drop qp\n"); - mlx4_qp_release_range(priv->mdev->dev, qpn, 1); - return err; - } - - return 0; -} - -void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv) -{ - u32 qpn; - - qpn = priv->drop_qp.qpn; - mlx4_qp_remove(priv->mdev->dev, &priv->drop_qp); - mlx4_qp_free(priv->mdev->dev, &priv->drop_qp); - mlx4_qp_release_range(priv->mdev->dev, qpn, 1); -} - -/* Allocate rx qp's and configure them according to rss map */ -int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv) -{ - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_rss_map *rss_map = &priv->rss_map; - struct mlx4_qp_context context; - struct mlx4_rss_context *rss_context; - int rss_rings; - void *ptr; - u8 rss_mask = (MLX4_RSS_IPV4 | MLX4_RSS_TCP_IPV4 | MLX4_RSS_IPV6 | - MLX4_RSS_TCP_IPV6); - int i; - int err = 0; - int good_qps = 0; - static const u32 rsskey[10] = { 0xD181C62C, 0xF7F4DB5B, 0x1983A2FC, - 0x943E1ADB, 0xD9389E6B, 0xD1039C2C, 0xA74499AD, - 0x593D56D9, 0xF3253C06, 0x2ADC1FFC}; - - en_dbg(DRV, priv, "Configuring rss steering\n"); - err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num, - priv->rx_ring_num, - &rss_map->base_qpn, 0); - if (err) { - en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num); - return err; - } - - for (i = 0; i < priv->rx_ring_num; i++) { - priv->rx_ring[i]->qpn = rss_map->base_qpn + i; - err = mlx4_en_config_rss_qp(priv, priv->rx_ring[i]->qpn, - priv->rx_ring[i], - &rss_map->state[i], - &rss_map->qps[i]); - if (err) - goto rss_err; - - ++good_qps; - } - - /* Configure RSS indirection qp */ - err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp); - if (err) { - en_err(priv, "Failed to allocate RSS indirection QP\n"); - goto rss_err; - } - rss_map->indir_qp.event = mlx4_en_sqp_event; - mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn, - priv->rx_ring[0]->cqn, -1, &context); - - if (!priv->prof->rss_rings || priv->prof->rss_rings > priv->rx_ring_num) - rss_rings = priv->rx_ring_num; - else - rss_rings = priv->prof->rss_rings; - - ptr = ((u8 *)&context) + offsetof(struct mlx4_qp_context, pri_path) + - MLX4_RSS_OFFSET_IN_QPC_PRI_PATH; - rss_context = ptr; - rss_context->base_qpn = cpu_to_be32(ilog2(rss_rings) << 24 | - (rss_map->base_qpn)); - rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn); - if (priv->mdev->profile.udp_rss) { - rss_mask |= MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6; - rss_context->base_qpn_udp = rss_context->default_qpn; - } - rss_context->flags = rss_mask; - rss_context->hash_fn = MLX4_RSS_HASH_TOP; - for (i = 0; i < 10; i++) - rss_context->rss_key[i] = cpu_to_be32(rsskey[i]); - - err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context, - &rss_map->indir_qp, &rss_map->indir_state); - if (err) - goto indir_err; - - return 0; - -indir_err: - mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state, - MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp); - mlx4_qp_remove(mdev->dev, &rss_map->indir_qp); - mlx4_qp_free(mdev->dev, &rss_map->indir_qp); -rss_err: - for (i = 0; i < good_qps; i++) { - mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i], - MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]); - mlx4_qp_remove(mdev->dev, &rss_map->qps[i]); - mlx4_qp_free(mdev->dev, &rss_map->qps[i]); - } - mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num); - return err; -} - -void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv) -{ - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_rss_map *rss_map = &priv->rss_map; - int i; - - mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state, - MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp); - mlx4_qp_remove(mdev->dev, &rss_map->indir_qp); - mlx4_qp_free(mdev->dev, &rss_map->indir_qp); - - for (i = 0; i < priv->rx_ring_num; i++) { - mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i], - MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]); - mlx4_qp_remove(mdev->dev, &rss_map->qps[i]); - mlx4_qp_free(mdev->dev, &rss_map->qps[i]); - } - mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num); -} - Property changes on: stable/11/sys/ofed/drivers/net/mlx4/en_rx.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/Makefile =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/Makefile (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/Makefile (nonexistent) @@ -1,33 +0,0 @@ -# $FreeBSD$ -#.PATH: ${.CURDIR}/../../ofed/drivers/net/mlx4:${.CURDIR}/../../ofed/include/linux -.PATH: ${.CURDIR}/../../../../../include/linux - -.include - - -KMOD = mlx4 -SRCS = device_if.h bus_if.h pci_if.h vnode_if.h -SRCS+= alloc.c catas.c cmd.c cq.c eq.c fw.c icm.c intf.c main.c mcg.c mr.c linux_compat.c linux_radix.c -SRCS+= pd.c port.c profile.c qp.c reset.c sense.c srq.c resource_tracker.c sys_tune.c -SRCS+= opt_inet.h opt_inet6.h - - -#CFLAGS+= -I${.CURDIR}/../../ofed/drivers/net/mlx4 -#CFLAGS+= -I${.CURDIR}/../../ofed/include/ -CFLAGS+= -I${.CURDIR}/../../../../../include - -.if !defined(KERNBUILDDIR) -.if ${MK_INET_SUPPORT} != "no" -opt_inet.h: - @echo "#define INET 1" > ${.TARGET} -.endif - -.if ${MK_INET6_SUPPORT} != "no" -opt_inet6.h: - @echo "#define INET6 1" > ${.TARGET} -.endif -.endif - -.include - -CFLAGS+= -Wno-cast-qual -Wno-pointer-arith Property changes on: stable/11/sys/ofed/drivers/net/mlx4/Makefile ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/sense.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/sense.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/sense.c (nonexistent) @@ -1,143 +0,0 @@ -/* - * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include -#include - -#include - -#include "mlx4.h" - -int mlx4_SENSE_PORT(struct mlx4_dev *dev, int port, - enum mlx4_port_type *type) -{ - u64 out_param; - int err = 0; - - err = mlx4_cmd_imm(dev, 0, &out_param, port, 0, - MLX4_CMD_SENSE_PORT, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_WRAPPED); - if (err) { - mlx4_err(dev, "Sense command failed for port: %d\n", port); - return err; - } - - if (out_param > 2) { - mlx4_err(dev, "Sense returned illegal value: 0x%llx\n", (unsigned long long)out_param); - return -EINVAL; - } - - *type = out_param; - return 0; -} - -void mlx4_do_sense_ports(struct mlx4_dev *dev, - enum mlx4_port_type *stype, - enum mlx4_port_type *defaults) -{ - struct mlx4_sense *sense = &mlx4_priv(dev)->sense; - int err; - int i; - - for (i = 1; i <= dev->caps.num_ports; i++) { - stype[i - 1] = 0; - if (sense->do_sense_port[i] && sense->sense_allowed[i] && - dev->caps.possible_type[i] == MLX4_PORT_TYPE_AUTO) { - err = mlx4_SENSE_PORT(dev, i, &stype[i - 1]); - if (err) - stype[i - 1] = defaults[i - 1]; - } else - stype[i - 1] = defaults[i - 1]; - } - - /* - * If sensed nothing, remain in current configuration. - */ - for (i = 0; i < dev->caps.num_ports; i++) - stype[i] = stype[i] ? stype[i] : defaults[i]; - -} - -static void mlx4_sense_port(struct work_struct *work) -{ - struct delayed_work *delay = to_delayed_work(work); - struct mlx4_sense *sense = container_of(delay, struct mlx4_sense, - sense_poll); - struct mlx4_dev *dev = sense->dev; - struct mlx4_priv *priv = mlx4_priv(dev); - enum mlx4_port_type stype[MLX4_MAX_PORTS]; - - mutex_lock(&priv->port_mutex); - mlx4_do_sense_ports(dev, stype, &dev->caps.port_type[1]); - - if (mlx4_check_port_params(dev, stype)) - goto sense_again; - - if (mlx4_change_port_types(dev, stype)) - mlx4_err(dev, "Failed to change port_types\n"); - -sense_again: - mutex_unlock(&priv->port_mutex); - queue_delayed_work(mlx4_wq , &sense->sense_poll, - round_jiffies_relative(MLX4_SENSE_RANGE)); -} - -void mlx4_start_sense(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_sense *sense = &priv->sense; - - if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) - return; - - queue_delayed_work(mlx4_wq , &sense->sense_poll, - round_jiffies_relative(MLX4_SENSE_RANGE)); -} - -void mlx4_stop_sense(struct mlx4_dev *dev) -{ - cancel_delayed_work_sync(&mlx4_priv(dev)->sense.sense_poll); -} - -void mlx4_sense_init(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_sense *sense = &priv->sense; - int port; - - sense->dev = dev; - for (port = 1; port <= dev->caps.num_ports; port++) - sense->do_sense_port[port] = 1; - - INIT_DEFERRABLE_WORK(&sense->sense_poll, mlx4_sense_port); -} Property changes on: stable/11/sys/ofed/drivers/net/mlx4/sense.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/icm.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/icm.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/icm.c (nonexistent) @@ -1,473 +0,0 @@ -/* - * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include -#include - -#include - -#include "mlx4.h" -#include "icm.h" -#include "fw.h" - -/* - * We allocate in as big chunks as we can, up to a maximum of 256 KB - * per chunk. - */ -enum { - MLX4_ICM_ALLOC_SIZE = 1 << 18, - MLX4_TABLE_CHUNK_SIZE = 1 << 18 -}; - -static void mlx4_free_icm_pages(struct mlx4_dev *dev, struct mlx4_icm_chunk *chunk) -{ - int i; - - if (chunk->nsg > 0) - pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages, - PCI_DMA_BIDIRECTIONAL); - - for (i = 0; i < chunk->npages; ++i) - __free_pages(sg_page(&chunk->mem[i]), - get_order(chunk->mem[i].length)); -} - -static void mlx4_free_icm_coherent(struct mlx4_dev *dev, struct mlx4_icm_chunk *chunk) -{ - int i; - - for (i = 0; i < chunk->npages; ++i) - dma_free_coherent(&dev->pdev->dev, chunk->mem[i].length, - lowmem_page_address(sg_page(&chunk->mem[i])), - sg_dma_address(&chunk->mem[i])); -} - -void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm, int coherent) -{ - struct mlx4_icm_chunk *chunk, *tmp; - - if (!icm) - return; - - list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) { - if (coherent) - mlx4_free_icm_coherent(dev, chunk); - else - mlx4_free_icm_pages(dev, chunk); - - kfree(chunk); - } - - kfree(icm); -} - -static int mlx4_alloc_icm_pages(struct scatterlist *mem, int order, - gfp_t gfp_mask, int node) -{ - struct page *page; - - page = alloc_pages_node(node, gfp_mask, order); - if (!page) { - page = alloc_pages(gfp_mask, order); - if (!page) - return -ENOMEM; - } - - sg_set_page(mem, page, PAGE_SIZE << order, 0); - return 0; -} - -static int mlx4_alloc_icm_coherent(struct device *dev, struct scatterlist *mem, - int order, gfp_t gfp_mask) -{ - void *buf = dma_alloc_coherent(dev, PAGE_SIZE << order, - &sg_dma_address(mem), gfp_mask); - if (!buf) - return -ENOMEM; - - sg_set_buf(mem, buf, PAGE_SIZE << order); - BUG_ON(mem->offset); - sg_dma_len(mem) = PAGE_SIZE << order; - return 0; -} - -struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages, - gfp_t gfp_mask, int coherent) -{ - struct mlx4_icm *icm; - struct mlx4_icm_chunk *chunk = NULL; - int cur_order; - int ret; - - /* We use sg_set_buf for coherent allocs, which assumes low memory */ - BUG_ON(coherent && (gfp_mask & __GFP_HIGHMEM)); - - icm = kmalloc_node(sizeof *icm, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN), - dev->numa_node); - if (!icm) { - icm = kmalloc(sizeof *icm, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); - if (!icm) - return NULL; - } - - icm->refcount = 0; - INIT_LIST_HEAD(&icm->chunk_list); - - cur_order = get_order(MLX4_ICM_ALLOC_SIZE); - - while (npages > 0) { - if (!chunk) { - chunk = kmalloc_node(sizeof *chunk, - gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN), - dev->numa_node); - if (!chunk) { - chunk = kmalloc(sizeof *chunk, - gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); - if (!chunk) - goto fail; - } - - sg_init_table(chunk->mem, MLX4_ICM_CHUNK_LEN); - chunk->npages = 0; - chunk->nsg = 0; - list_add_tail(&chunk->list, &icm->chunk_list); - } - - while (1 << cur_order > npages) - --cur_order; - - if (coherent) - ret = mlx4_alloc_icm_coherent(&dev->pdev->dev, - &chunk->mem[chunk->npages], - cur_order, gfp_mask); - else - ret = mlx4_alloc_icm_pages(&chunk->mem[chunk->npages], - cur_order, gfp_mask, - dev->numa_node); - - if (ret) { - if (--cur_order < 0) - goto fail; - else - continue; - } - - ++chunk->npages; - - if (coherent) - ++chunk->nsg; - else if (chunk->npages == MLX4_ICM_CHUNK_LEN) { - chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, - chunk->npages, - PCI_DMA_BIDIRECTIONAL); - - if (chunk->nsg <= 0) - goto fail; - } - - if (chunk->npages == MLX4_ICM_CHUNK_LEN) - chunk = NULL; - - npages -= 1 << cur_order; - } - - if (!coherent && chunk) { - chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, - chunk->npages, - PCI_DMA_BIDIRECTIONAL); - - if (chunk->nsg <= 0) - goto fail; - } - - return icm; - -fail: - mlx4_free_icm(dev, icm, coherent); - return NULL; -} - -static int mlx4_MAP_ICM(struct mlx4_dev *dev, struct mlx4_icm *icm, u64 virt) -{ - return mlx4_map_cmd(dev, MLX4_CMD_MAP_ICM, icm, virt); -} - -static int mlx4_UNMAP_ICM(struct mlx4_dev *dev, u64 virt, u32 page_count) -{ - return mlx4_cmd(dev, virt, page_count, 0, MLX4_CMD_UNMAP_ICM, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); -} - -int mlx4_MAP_ICM_AUX(struct mlx4_dev *dev, struct mlx4_icm *icm) -{ - return mlx4_map_cmd(dev, MLX4_CMD_MAP_ICM_AUX, icm, -1); -} - -int mlx4_UNMAP_ICM_AUX(struct mlx4_dev *dev) -{ - return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_UNMAP_ICM_AUX, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); -} - -int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj) -{ - u32 i = (obj & (table->num_obj - 1)) / - (MLX4_TABLE_CHUNK_SIZE / table->obj_size); - int ret = 0; - - mutex_lock(&table->mutex); - - if (table->icm[i]) { - ++table->icm[i]->refcount; - goto out; - } - - table->icm[i] = mlx4_alloc_icm(dev, MLX4_TABLE_CHUNK_SIZE >> PAGE_SHIFT, - (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | - __GFP_NOWARN, table->coherent); - if (!table->icm[i]) { - ret = -ENOMEM; - goto out; - } - - if (mlx4_MAP_ICM(dev, table->icm[i], table->virt + - (u64) i * MLX4_TABLE_CHUNK_SIZE)) { - mlx4_free_icm(dev, table->icm[i], table->coherent); - table->icm[i] = NULL; - ret = -ENOMEM; - goto out; - } - - ++table->icm[i]->refcount; - -out: - mutex_unlock(&table->mutex); - return ret; -} - -void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj) -{ - u32 i; - u64 offset; - - i = (obj & (table->num_obj - 1)) / (MLX4_TABLE_CHUNK_SIZE / table->obj_size); - - mutex_lock(&table->mutex); - - if (--table->icm[i]->refcount == 0) { - offset = (u64) i * MLX4_TABLE_CHUNK_SIZE; - - if (!mlx4_UNMAP_ICM(dev, table->virt + offset, - MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE)) { - mlx4_free_icm(dev, table->icm[i], table->coherent); - table->icm[i] = NULL; - } else { - pr_warn("mlx4_core: mlx4_UNMAP_ICM failed.\n"); - } - } - - mutex_unlock(&table->mutex); -} - -void *mlx4_table_find(struct mlx4_icm_table *table, u32 obj, - dma_addr_t *dma_handle) -{ - int offset, dma_offset, i; - u64 idx; - struct mlx4_icm_chunk *chunk; - struct mlx4_icm *icm; - struct page *page = NULL; - - if (!table->lowmem) - return NULL; - - mutex_lock(&table->mutex); - - idx = (u64) (obj & (table->num_obj - 1)) * table->obj_size; - icm = table->icm[idx / MLX4_TABLE_CHUNK_SIZE]; - dma_offset = offset = idx % MLX4_TABLE_CHUNK_SIZE; - - if (!icm) - goto out; - - list_for_each_entry(chunk, &icm->chunk_list, list) { - for (i = 0; i < chunk->npages; ++i) { - if (dma_handle && dma_offset >= 0) { - if (sg_dma_len(&chunk->mem[i]) > dma_offset) - *dma_handle = sg_dma_address(&chunk->mem[i]) + - dma_offset; - dma_offset -= sg_dma_len(&chunk->mem[i]); - } - /* - * DMA mapping can merge pages but not split them, - * so if we found the page, dma_handle has already - * been assigned to. - */ - if (chunk->mem[i].length > offset) { - page = sg_page(&chunk->mem[i]); - goto out; - } - offset -= chunk->mem[i].length; - } - } - -out: - mutex_unlock(&table->mutex); - return page ? lowmem_page_address(page) + offset : NULL; -} - -int mlx4_table_get_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, - u32 start, u32 end) -{ - int inc = MLX4_TABLE_CHUNK_SIZE / table->obj_size; - int err; - u32 i; - - for (i = start; i <= end; i += inc) { - err = mlx4_table_get(dev, table, i); - if (err) - goto fail; - } - - return 0; - -fail: - while (i > start) { - i -= inc; - mlx4_table_put(dev, table, i); - } - - return err; -} - -void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, - u32 start, u32 end) -{ - u32 i; - - for (i = start; i <= end; i += MLX4_TABLE_CHUNK_SIZE / table->obj_size) - mlx4_table_put(dev, table, i); -} - -int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table, - u64 virt, int obj_size, u64 nobj, int reserved, - int use_lowmem, int use_coherent) -{ - int obj_per_chunk; - int num_icm; - unsigned chunk_size; - int i; - u64 size; - - obj_per_chunk = MLX4_TABLE_CHUNK_SIZE / obj_size; - num_icm = div_u64((nobj + obj_per_chunk - 1), obj_per_chunk); - - table->icm = kcalloc(num_icm, sizeof *table->icm, GFP_KERNEL); - if (!table->icm) - return -ENOMEM; - table->virt = virt; - table->num_icm = num_icm; - table->num_obj = nobj; - table->obj_size = obj_size; - table->lowmem = use_lowmem; - table->coherent = use_coherent; - mutex_init(&table->mutex); - - size = (u64) nobj * obj_size; - for (i = 0; i * MLX4_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) { - chunk_size = MLX4_TABLE_CHUNK_SIZE; - if ((i + 1) * MLX4_TABLE_CHUNK_SIZE > size) - chunk_size = PAGE_ALIGN(size - - i * MLX4_TABLE_CHUNK_SIZE); - - table->icm[i] = mlx4_alloc_icm(dev, chunk_size >> PAGE_SHIFT, - (use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) | - __GFP_NOWARN, use_coherent); - if (!table->icm[i]) - goto err; - if (mlx4_MAP_ICM(dev, table->icm[i], virt + i * MLX4_TABLE_CHUNK_SIZE)) { - mlx4_free_icm(dev, table->icm[i], use_coherent); - table->icm[i] = NULL; - goto err; - } - - /* - * Add a reference to this ICM chunk so that it never - * gets freed (since it contains reserved firmware objects). - */ - ++table->icm[i]->refcount; - } - - return 0; - -err: - for (i = 0; i < num_icm; ++i) - if (table->icm[i]) { - if (!mlx4_UNMAP_ICM(dev, - virt + i * MLX4_TABLE_CHUNK_SIZE, - MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE)) { - mlx4_free_icm(dev, table->icm[i], use_coherent); - } else { - pr_warn("mlx4_core: mlx4_UNMAP_ICM failed.\n"); - return -ENOMEM; - } - } - kfree(table->icm); - - return -ENOMEM; -} - -void mlx4_cleanup_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table) -{ - int i, err = 0; - - for (i = 0; i < table->num_icm; ++i) - if (table->icm[i]) { - err = mlx4_UNMAP_ICM(dev, - table->virt + i * MLX4_TABLE_CHUNK_SIZE, - MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE); - if (!err) { - mlx4_free_icm(dev, table->icm[i], - table->coherent); - } else { - pr_warn("mlx4_core: mlx4_UNMAP_ICM failed.\n"); - break; - } - } - - if (!err) - kfree(table->icm); -} Property changes on: stable/11/sys/ofed/drivers/net/mlx4/icm.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/en_resources.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/en_resources.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/en_resources.c (nonexistent) @@ -1,121 +0,0 @@ -/* - * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include -#include -#include - -#include "mlx4_en.h" - - -void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride, - int is_tx, int rss, int qpn, int cqn, - int user_prio, struct mlx4_qp_context *context) -{ - struct mlx4_en_dev *mdev = priv->mdev; - struct net_device *dev = priv->dev; - - memset(context, 0, sizeof *context); - context->flags = cpu_to_be32(7 << 16 | rss << MLX4_RSS_QPC_FLAG_OFFSET); - context->pd = cpu_to_be32(mdev->priv_pdn); - context->mtu_msgmax = 0xff; - if (!is_tx && !rss) - context->rq_size_stride = ilog2(size) << 3 | (ilog2(stride) - 4); - if (is_tx) - context->sq_size_stride = ilog2(size) << 3 | (ilog2(stride) - 4); - else - context->sq_size_stride = ilog2(TXBB_SIZE) - 4; - context->usr_page = cpu_to_be32(mdev->priv_uar.index); - context->local_qpn = cpu_to_be32(qpn); - context->pri_path.ackto = 1 & 0x07; - context->pri_path.sched_queue = 0x83 | (priv->port - 1) << 6; - if (user_prio >= 0) { - context->pri_path.sched_queue |= user_prio << 3; - context->pri_path.feup = 1 << 6; - } - context->pri_path.counter_index = (u8)(priv->counter_index); - if (!rss && - (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_LB_SRC_CHK) && - context->pri_path.counter_index != 0xFF) { - /* disable multicast loopback to qp with same counter */ - context->pri_path.fl |= MLX4_FL_ETH_SRC_CHECK_MC_LB; - context->pri_path.vlan_control |= - MLX4_VLAN_CTRL_ETH_SRC_CHECK_IF_COUNTER; - } - - context->cqn_send = cpu_to_be32(cqn); - context->cqn_recv = cpu_to_be32(cqn); - context->db_rec_addr = cpu_to_be64(priv->res.db.dma << 2); - if (!(dev->if_capabilities & IFCAP_VLAN_HWCSUM)) - context->param3 |= cpu_to_be32(1 << 30); -} - - -int mlx4_en_map_buffer(struct mlx4_buf *buf) -{ - struct page **pages; - int i; - - // if nbufs == 1 - there is no need to vmap - // if buf->direct.buf is not NULL it means that vmap was already done by mlx4_alloc_buff - if (buf->direct.buf != NULL || buf->nbufs == 1) - return 0; - - pages = kmalloc(sizeof *pages * buf->nbufs, GFP_KERNEL); - if (!pages) - return -ENOMEM; - - for (i = 0; i < buf->nbufs; ++i) - pages[i] = virt_to_page(buf->page_list[i].buf); - - buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL); - kfree(pages); - if (!buf->direct.buf) - return -ENOMEM; - - return 0; -} - -void mlx4_en_unmap_buffer(struct mlx4_buf *buf) -{ - if (BITS_PER_LONG == 64 || buf->nbufs == 1) - return; - - vunmap(buf->direct.buf); -} - -void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event) -{ - return; -} - Property changes on: stable/11/sys/ofed/drivers/net/mlx4/en_resources.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/profile.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/profile.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/profile.c (nonexistent) @@ -1,256 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005, 2014 Mellanox Technologies. All rights reserved. - * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include - -#include "mlx4.h" -#include "fw.h" - -enum { - MLX4_RES_QP, - MLX4_RES_RDMARC, - MLX4_RES_ALTC, - MLX4_RES_AUXC, - MLX4_RES_SRQ, - MLX4_RES_CQ, - MLX4_RES_EQ, - MLX4_RES_DMPT, - MLX4_RES_CMPT, - MLX4_RES_MTT, - MLX4_RES_MCG, - MLX4_RES_NUM -}; - -static const char *res_name[] = { - [MLX4_RES_QP] = "QP", - [MLX4_RES_RDMARC] = "RDMARC", - [MLX4_RES_ALTC] = "ALTC", - [MLX4_RES_AUXC] = "AUXC", - [MLX4_RES_SRQ] = "SRQ", - [MLX4_RES_CQ] = "CQ", - [MLX4_RES_EQ] = "EQ", - [MLX4_RES_DMPT] = "DMPT", - [MLX4_RES_CMPT] = "CMPT", - [MLX4_RES_MTT] = "MTT", - [MLX4_RES_MCG] = "MCG", -}; - -u64 mlx4_make_profile(struct mlx4_dev *dev, - struct mlx4_profile *request, - struct mlx4_dev_cap *dev_cap, - struct mlx4_init_hca_param *init_hca) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_resource { - u64 size; - u64 start; - int type; - u64 num; - int log_num; - }; - - u64 total_size = 0; - struct mlx4_resource *profile; - struct mlx4_resource tmp; - int i, j; - - profile = kcalloc(MLX4_RES_NUM, sizeof(*profile), GFP_KERNEL); - if (!profile) - return -ENOMEM; - - profile[MLX4_RES_QP].size = dev_cap->qpc_entry_sz; - profile[MLX4_RES_RDMARC].size = dev_cap->rdmarc_entry_sz; - profile[MLX4_RES_ALTC].size = dev_cap->altc_entry_sz; - profile[MLX4_RES_AUXC].size = dev_cap->aux_entry_sz; - profile[MLX4_RES_SRQ].size = dev_cap->srq_entry_sz; - profile[MLX4_RES_CQ].size = dev_cap->cqc_entry_sz; - profile[MLX4_RES_EQ].size = dev_cap->eqc_entry_sz; - profile[MLX4_RES_DMPT].size = dev_cap->dmpt_entry_sz; - profile[MLX4_RES_CMPT].size = dev_cap->cmpt_entry_sz; - profile[MLX4_RES_MTT].size = dev_cap->mtt_entry_sz; - profile[MLX4_RES_MCG].size = mlx4_get_mgm_entry_size(dev); - - profile[MLX4_RES_QP].num = request->num_qp; - profile[MLX4_RES_RDMARC].num = request->num_qp * request->rdmarc_per_qp; - profile[MLX4_RES_ALTC].num = request->num_qp; - profile[MLX4_RES_AUXC].num = request->num_qp; - profile[MLX4_RES_SRQ].num = request->num_srq; - profile[MLX4_RES_CQ].num = request->num_cq; - profile[MLX4_RES_EQ].num = mlx4_is_mfunc(dev) ? - dev->phys_caps.num_phys_eqs : - min_t(unsigned, dev_cap->max_eqs, MAX_MSIX); - profile[MLX4_RES_DMPT].num = request->num_mpt; - profile[MLX4_RES_CMPT].num = MLX4_NUM_CMPTS; - profile[MLX4_RES_MTT].num = ((u64)request->num_mtt_segs) * - (1 << log_mtts_per_seg); - profile[MLX4_RES_MCG].num = request->num_mcg; - - for (i = 0; i < MLX4_RES_NUM; ++i) { - profile[i].type = i; - profile[i].num = roundup_pow_of_two(profile[i].num); - profile[i].log_num = ilog2(profile[i].num); - profile[i].size *= profile[i].num; - profile[i].size = max(profile[i].size, (u64) PAGE_SIZE); - } - - /* - * Sort the resources in decreasing order of size. Since they - * all have sizes that are powers of 2, we'll be able to keep - * resources aligned to their size and pack them without gaps - * using the sorted order. - */ - for (i = MLX4_RES_NUM; i > 0; --i) - for (j = 1; j < i; ++j) { - if (profile[j].size > profile[j - 1].size) { - tmp = profile[j]; - profile[j] = profile[j - 1]; - profile[j - 1] = tmp; - } - } - - for (i = 0; i < MLX4_RES_NUM; ++i) { - if (profile[i].size) { - profile[i].start = total_size; - total_size += profile[i].size; - } - - if (total_size > dev_cap->max_icm_sz) { - mlx4_err(dev, "Profile requires 0x%llx bytes; " - "won't fit in 0x%llx bytes of context memory.\n", - (unsigned long long) total_size, - (unsigned long long) dev_cap->max_icm_sz); - kfree(profile); - return -ENOMEM; - } - - if (profile[i].size) - mlx4_dbg(dev, " profile[%2d] (%6s): 2^%02d entries @ 0x%10llx, " - "size 0x%10llx\n", - i, res_name[profile[i].type], profile[i].log_num, - (unsigned long long) profile[i].start, - (unsigned long long) profile[i].size); - } - - mlx4_dbg(dev, "HCA context memory: reserving %d KB\n", - (int) (total_size >> 10)); - - for (i = 0; i < MLX4_RES_NUM; ++i) { - switch (profile[i].type) { - case MLX4_RES_QP: - dev->caps.num_qps = profile[i].num; - init_hca->qpc_base = profile[i].start; - init_hca->log_num_qps = profile[i].log_num; - break; - case MLX4_RES_RDMARC: - for (priv->qp_table.rdmarc_shift = 0; - request->num_qp << priv->qp_table.rdmarc_shift < profile[i].num; - ++priv->qp_table.rdmarc_shift) - ; /* nothing */ - dev->caps.max_qp_dest_rdma = 1 << priv->qp_table.rdmarc_shift; - priv->qp_table.rdmarc_base = (u32) profile[i].start; - init_hca->rdmarc_base = profile[i].start; - init_hca->log_rd_per_qp = priv->qp_table.rdmarc_shift; - break; - case MLX4_RES_ALTC: - init_hca->altc_base = profile[i].start; - break; - case MLX4_RES_AUXC: - init_hca->auxc_base = profile[i].start; - break; - case MLX4_RES_SRQ: - dev->caps.num_srqs = profile[i].num; - init_hca->srqc_base = profile[i].start; - init_hca->log_num_srqs = profile[i].log_num; - break; - case MLX4_RES_CQ: - dev->caps.num_cqs = profile[i].num; - init_hca->cqc_base = profile[i].start; - init_hca->log_num_cqs = profile[i].log_num; - break; - case MLX4_RES_EQ: - if (dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) { - init_hca->log_num_eqs = 0x1f; - init_hca->eqc_base = profile[i].start; - init_hca->num_sys_eqs = dev_cap->num_sys_eqs; - } else { - dev->caps.num_eqs = roundup_pow_of_two( - min_t(unsigned, - dev_cap->max_eqs, MAX_MSIX)); - init_hca->eqc_base = profile[i].start; - init_hca->log_num_eqs = ilog2(dev->caps.num_eqs); - } - break; - case MLX4_RES_DMPT: - dev->caps.num_mpts = profile[i].num; - priv->mr_table.mpt_base = profile[i].start; - init_hca->dmpt_base = profile[i].start; - init_hca->log_mpt_sz = profile[i].log_num; - break; - case MLX4_RES_CMPT: - init_hca->cmpt_base = profile[i].start; - break; - case MLX4_RES_MTT: - dev->caps.num_mtts = profile[i].num; - priv->mr_table.mtt_base = profile[i].start; - init_hca->mtt_base = profile[i].start; - break; - case MLX4_RES_MCG: - init_hca->mc_base = profile[i].start; - init_hca->log_mc_entry_sz = - ilog2(mlx4_get_mgm_entry_size(dev)); - init_hca->log_mc_table_sz = profile[i].log_num; - if (dev->caps.steering_mode == - MLX4_STEERING_MODE_DEVICE_MANAGED) { - dev->caps.num_mgms = profile[i].num; - } else { - init_hca->log_mc_hash_sz = - profile[i].log_num - 1; - dev->caps.num_mgms = profile[i].num >> 1; - dev->caps.num_amgms = profile[i].num >> 1; - } - break; - default: - break; - } - } - - /* - * PDs don't take any HCA memory, but we assign them as part - * of the HCA profile anyway. - */ - dev->caps.num_pds = MLX4_NUM_PDS; - - kfree(profile); - return total_size; -} Property changes on: stable/11/sys/ofed/drivers/net/mlx4/profile.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/icm.h =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/icm.h (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/icm.h (nonexistent) @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MLX4_ICM_H -#define MLX4_ICM_H - -#include -#include -#include -#include - -#define MLX4_ICM_CHUNK_LEN \ - ((256 - sizeof (struct list_head) - 2 * sizeof (int)) / \ - (sizeof (struct scatterlist))) - -enum { - MLX4_ICM_PAGE_SHIFT = 12, - MLX4_ICM_PAGE_SIZE = 1 << MLX4_ICM_PAGE_SHIFT, -}; - -struct mlx4_icm_chunk { - struct list_head list; - int npages; - int nsg; - struct scatterlist mem[MLX4_ICM_CHUNK_LEN]; -}; - -struct mlx4_icm { - struct list_head chunk_list; - int refcount; -}; - -struct mlx4_icm_iter { - struct mlx4_icm *icm; - struct mlx4_icm_chunk *chunk; - int page_idx; -}; - -struct mlx4_dev; - -struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages, - gfp_t gfp_mask, int coherent); -void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm, int coherent); - -int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj); -void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj); -int mlx4_table_get_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, - u32 start, u32 end); -void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, - u32 start, u32 end); -int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table, - u64 virt, int obj_size, u64 nobj, int reserved, - int use_lowmem, int use_coherent); -void mlx4_cleanup_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table); -void *mlx4_table_find(struct mlx4_icm_table *table, u32 obj, dma_addr_t *dma_handle); - -static inline void mlx4_icm_first(struct mlx4_icm *icm, - struct mlx4_icm_iter *iter) -{ - iter->icm = icm; - iter->chunk = list_empty(&icm->chunk_list) ? - NULL : list_entry(icm->chunk_list.next, - struct mlx4_icm_chunk, list); - iter->page_idx = 0; -} - -static inline int mlx4_icm_last(struct mlx4_icm_iter *iter) -{ - return !iter->chunk; -} - -static inline void mlx4_icm_next(struct mlx4_icm_iter *iter) -{ - if (++iter->page_idx >= iter->chunk->nsg) { - if (iter->chunk->list.next == &iter->icm->chunk_list) { - iter->chunk = NULL; - return; - } - - iter->chunk = list_entry(iter->chunk->list.next, - struct mlx4_icm_chunk, list); - iter->page_idx = 0; - } -} - -static inline dma_addr_t mlx4_icm_addr(struct mlx4_icm_iter *iter) -{ - return sg_dma_address(&iter->chunk->mem[iter->page_idx]); -} - -static inline unsigned long mlx4_icm_size(struct mlx4_icm_iter *iter) -{ - return sg_dma_len(&iter->chunk->mem[iter->page_idx]); -} - -int mlx4_MAP_ICM_AUX(struct mlx4_dev *dev, struct mlx4_icm *icm); -int mlx4_UNMAP_ICM_AUX(struct mlx4_dev *dev); - -#endif /* MLX4_ICM_H */ Property changes on: stable/11/sys/ofed/drivers/net/mlx4/icm.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/sys_tune.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/sys_tune.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/sys_tune.c (nonexistent) @@ -1,323 +0,0 @@ -/* - * Copyright (c) 2010, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include -#include -#include - -#include "mlx4.h" - -#if defined(CONFIG_X86) && defined(CONFIG_APM_MODULE) - -/* Each CPU is put into a group. In most cases, the group number is - * equal to the CPU number of one of the CPUs in the group. The - * exception is group NR_CPUS which is the default group. This is - * protected by sys_tune_startup_mutex. */ -DEFINE_PER_CPU(int, idle_cpu_group) = NR_CPUS; - -/* For each group, a count of the number of CPUs in the group which - * are known to be busy. A busy CPU might be running the busy loop - * below or general kernel code. The count is decremented on entry to - * the old pm_idle handler and incremented on exit. The aim is to - * avoid the count going to zero or negative. This situation can - * occur temporarily during module unload or CPU hot-plug but - * normality will be restored when the affected CPUs next exit the - * idle loop. */ -static atomic_t busy_cpu_count[NR_CPUS+1]; - -/* A workqueue item to be executed to cause the CPU to exit from the - * idle loop. */ -DEFINE_PER_CPU(struct work_struct, sys_tune_cpu_work); - -#define sys_tune_set_state(CPU,STATE) \ - do { } while(0) - - -/* A mutex to protect most of the module datastructures. */ -static DEFINE_MUTEX(sys_tune_startup_mutex); - -/* The old pm_idle handler. */ -static void (*old_pm_idle)(void) = NULL; - -static void sys_tune_pm_idle(void) -{ - atomic_t *busy_cpus_ptr; - int busy_cpus; - int cpu = smp_processor_id(); - - busy_cpus_ptr = &(busy_cpu_count[per_cpu(idle_cpu_group, cpu)]); - - sys_tune_set_state(cpu, 2); - - local_irq_enable(); - while (!need_resched()) { - busy_cpus = atomic_read(busy_cpus_ptr); - - /* If other CPUs in this group are busy then let this - * CPU go idle. We mustn't let the number of busy - * CPUs drop below 1. */ - if ( busy_cpus > 1 && - old_pm_idle != NULL && - ( atomic_cmpxchg(busy_cpus_ptr, busy_cpus, - busy_cpus-1) == busy_cpus ) ) { - local_irq_disable(); - sys_tune_set_state(cpu, 3); - /* This check might not be necessary, but it - * seems safest to include it because there - * might be a kernel version which requires - * it. */ - if (need_resched()) - local_irq_enable(); - else - old_pm_idle(); - /* This CPU is busy again. */ - sys_tune_set_state(cpu, 1); - atomic_add(1, busy_cpus_ptr); - return; - } - - cpu_relax(); - } - sys_tune_set_state(cpu, 0); -} - - -void sys_tune_work_func(struct work_struct *work) -{ - /* Do nothing. Since this function is running in process - * context, the idle thread isn't running on this CPU. */ -} - - -#ifdef CONFIG_SMP -static void sys_tune_smp_call(void *info) -{ - schedule_work(&get_cpu_var(sys_tune_cpu_work)); - put_cpu_var(sys_tune_cpu_work); -} -#endif - - -#ifdef CONFIG_SMP -static void sys_tune_refresh(void) -{ -#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) - on_each_cpu(&sys_tune_smp_call, NULL, 0, 1); -#else - on_each_cpu(&sys_tune_smp_call, NULL, 1); -#endif -} -#else -static void sys_tune_refresh(void) -{ - /* The current thread is executing on the one and only CPU so - * the idle thread isn't running. */ -} -#endif - - - -static int sys_tune_cpu_group(int cpu) -{ -#ifdef CONFIG_SMP - const cpumask_t *mask; - int other_cpu; - int group; - -#if defined(topology_thread_cpumask) && defined(ST_HAVE_EXPORTED_CPU_SIBLING_MAP) - /* Keep one hyperthread busy per core. */ - mask = topology_thread_cpumask(cpu); -#else - return cpu; -#endif - for_each_cpu_mask(cpu, *(mask)) { - group = per_cpu(idle_cpu_group, other_cpu); - if (group != NR_CPUS) - return group; - } -#endif - - return cpu; -} - - -static void sys_tune_add_cpu(int cpu) -{ - int group; - - /* Do nothing if this CPU has already been added. */ - if (per_cpu(idle_cpu_group, cpu) != NR_CPUS) - return; - - group = sys_tune_cpu_group(cpu); - per_cpu(idle_cpu_group, cpu) = group; - atomic_inc(&(busy_cpu_count[group])); - -} - -static void sys_tune_del_cpu(int cpu) -{ - - int group; - - if (per_cpu(idle_cpu_group, cpu) == NR_CPUS) - return; - - group = per_cpu(idle_cpu_group, cpu); - /* If the CPU was busy, this can cause the count to drop to - * zero. To rectify this, we need to cause one of the other - * CPUs in the group to exit the idle loop. If the CPU was - * not busy then this causes the contribution for this CPU to - * go to -1 which can cause the overall count to drop to zero - * or go negative. To rectify this situation we need to cause - * this CPU to exit the idle loop. */ - atomic_dec(&(busy_cpu_count[group])); - per_cpu(idle_cpu_group, cpu) = NR_CPUS; - -} - - -static int sys_tune_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) -{ - int cpu = (long)hcpu; - - switch(action) { -#ifdef CPU_ONLINE_FROZEN - case CPU_ONLINE_FROZEN: -#endif - case CPU_ONLINE: - mutex_lock(&sys_tune_startup_mutex); - sys_tune_add_cpu(cpu); - mutex_unlock(&sys_tune_startup_mutex); - /* The CPU might have already entered the idle loop in - * the wrong group. Make sure it exits the idle loop - * so that it picks up the correct group. */ - sys_tune_refresh(); - break; - -#ifdef CPU_DEAD_FROZEN - case CPU_DEAD_FROZEN: -#endif - case CPU_DEAD: - mutex_lock(&sys_tune_startup_mutex); - sys_tune_del_cpu(cpu); - mutex_unlock(&sys_tune_startup_mutex); - /* The deleted CPU may have been the only busy CPU in - * the group. Make sure one of the other CPUs in the - * group exits the idle loop. */ - sys_tune_refresh(); - break; - } - return NOTIFY_OK; -} - - -static struct notifier_block sys_tune_cpu_nb = { - .notifier_call = sys_tune_cpu_notify, -}; - - -static void sys_tune_ensure_init(void) -{ - BUG_ON (old_pm_idle != NULL); - - /* Atomically update pm_idle to &sys_tune_pm_idle. The old value - * is stored in old_pm_idle before installing the new - * handler. */ - do { - old_pm_idle = pm_idle; - } while (cmpxchg(&pm_idle, old_pm_idle, &sys_tune_pm_idle) != - old_pm_idle); -} -#endif - -void sys_tune_fini(void) -{ -#if defined(CONFIG_X86) && defined(CONFIG_APM_MODULE) - void (*old)(void); - int cpu; - - unregister_cpu_notifier(&sys_tune_cpu_nb); - - mutex_lock(&sys_tune_startup_mutex); - - - old = cmpxchg(&pm_idle, &sys_tune_pm_idle, old_pm_idle); - - for_each_online_cpu(cpu) - sys_tune_del_cpu(cpu); - - mutex_unlock(&sys_tune_startup_mutex); - - /* Our handler may still be executing on other CPUs. - * Schedule this thread on all CPUs to make sure all - * idle threads get interrupted. */ - sys_tune_refresh(); - - /* Make sure the work item has finished executing on all CPUs. - * This in turn ensures that all idle threads have been - * interrupted. */ - flush_scheduled_work(); -#endif /* CONFIG_X86 */ -} - -void sys_tune_init(void) -{ -#if defined(CONFIG_X86) && defined(CONFIG_APM_MODULE) - int cpu; - - for_each_possible_cpu(cpu) { - INIT_WORK(&per_cpu(sys_tune_cpu_work, cpu), - sys_tune_work_func); - } - - /* Start by registering the handler to ensure we don't miss - * any updates. */ - register_cpu_notifier(&sys_tune_cpu_nb); - - mutex_lock(&sys_tune_startup_mutex); - - for_each_online_cpu(cpu) - sys_tune_add_cpu(cpu); - - sys_tune_ensure_init(); - - - mutex_unlock(&sys_tune_startup_mutex); - - /* Ensure our idle handler starts to run. */ - sys_tune_refresh(); -#endif -} - Property changes on: stable/11/sys/ofed/drivers/net/mlx4/sys_tune.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/alloc.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/alloc.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/alloc.c (nonexistent) @@ -1,455 +0,0 @@ -/* - * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include - -#include "mlx4.h" - -u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap) -{ - u32 obj; - - spin_lock(&bitmap->lock); - - obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last); - if (obj >= bitmap->max) { - bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) - & bitmap->mask; - obj = find_first_zero_bit(bitmap->table, bitmap->max); - } - - if (obj < bitmap->max) { - set_bit(obj, bitmap->table); - bitmap->last = (obj + 1); - if (bitmap->last == bitmap->max) - bitmap->last = 0; - obj |= bitmap->top; - } else - obj = -1; - - if (obj != -1) - --bitmap->avail; - - spin_unlock(&bitmap->lock); - - return obj; -} - -void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr) -{ - mlx4_bitmap_free_range(bitmap, obj, 1, use_rr); -} - -static unsigned long find_aligned_range(unsigned long *bitmap, - u32 start, u32 nbits, - int len, int align, u32 skip_mask) -{ - unsigned long end, i; - -again: - start = ALIGN(start, align); - - while ((start < nbits) && (test_bit(start, bitmap) || - (start & skip_mask))) - start += align; - - if (start >= nbits) - return -1; - - end = start+len; - if (end > nbits) - return -1; - - for (i = start + 1; i < end; i++) { - if (test_bit(i, bitmap) || ((u32)i & skip_mask)) { - start = i + 1; - goto again; - } - } - - return start; -} - -u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, - int align, u32 skip_mask) -{ - u32 obj; - - if (likely(cnt == 1 && align == 1 && !skip_mask)) - return mlx4_bitmap_alloc(bitmap); - - spin_lock(&bitmap->lock); - - obj = find_aligned_range(bitmap->table, bitmap->last, - bitmap->max, cnt, align, skip_mask); - if (obj >= bitmap->max) { - bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) - & bitmap->mask; - obj = find_aligned_range(bitmap->table, 0, bitmap->max, - cnt, align, skip_mask); - } - - if (obj < bitmap->max) { - bitmap_set(bitmap->table, obj, cnt); - if (obj == bitmap->last) { - bitmap->last = (obj + cnt); - if (bitmap->last >= bitmap->max) - bitmap->last = 0; - } - obj |= bitmap->top; - } else - obj = -1; - - if (obj != -1) - bitmap->avail -= cnt; - - spin_unlock(&bitmap->lock); - - return obj; -} - -u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap) -{ - return bitmap->avail; -} - -void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt, - int use_rr) -{ - obj &= bitmap->max + bitmap->reserved_top - 1; - - spin_lock(&bitmap->lock); - if (!use_rr) { - bitmap->last = min(bitmap->last, obj); - bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) - & bitmap->mask; - } - bitmap_clear(bitmap->table, obj, cnt); - bitmap->avail += cnt; - spin_unlock(&bitmap->lock); -} - -int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, - u32 reserved_bot, u32 reserved_top) -{ - /* sanity check */ - if (num <= (u64)reserved_top + reserved_bot) - return -EINVAL; - - /* num must be a power of 2 */ - if (num != roundup_pow_of_two(num)) - return -EINVAL; - - if (reserved_bot + reserved_top >= num) - return -EINVAL; - - bitmap->last = 0; - bitmap->top = 0; - bitmap->max = num - reserved_top; - bitmap->mask = mask; - bitmap->reserved_top = reserved_top; - bitmap->avail = num - reserved_top - reserved_bot; - spin_lock_init(&bitmap->lock); - bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) * - sizeof (long), GFP_KERNEL); - if (!bitmap->table) - return -ENOMEM; - - bitmap_set(bitmap->table, 0, reserved_bot); - - return 0; -} - -void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap) -{ - kfree(bitmap->table); -} - -/* - * Handling for queue buffers -- we allocate a bunch of memory and - * register it in a memory region at HCA virtual address 0. If the - * requested size is > max_direct, we split the allocation into - * multiple pages, so we don't require too much contiguous memory. - */ - -int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, - struct mlx4_buf *buf) -{ - dma_addr_t t; - - if (size <= max_direct) { - buf->nbufs = 1; - buf->npages = 1; - buf->page_shift = get_order(size) + PAGE_SHIFT; - buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev, - size, &t, GFP_KERNEL); - if (!buf->direct.buf) - return -ENOMEM; - - buf->direct.map = t; - - while (t & ((1 << buf->page_shift) - 1)) { - --buf->page_shift; - buf->npages *= 2; - } - - memset(buf->direct.buf, 0, size); - } else { - int i; - - buf->direct.buf = NULL; - buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE; - buf->npages = buf->nbufs; - buf->page_shift = PAGE_SHIFT; - buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), - GFP_KERNEL); - if (!buf->page_list) - return -ENOMEM; - - for (i = 0; i < buf->nbufs; ++i) { - buf->page_list[i].buf = - dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, - &t, GFP_KERNEL); - if (!buf->page_list[i].buf) - goto err_free; - - buf->page_list[i].map = t; - - memset(buf->page_list[i].buf, 0, PAGE_SIZE); - } - - if (BITS_PER_LONG == 64) { - struct page **pages; - pages = kmalloc(sizeof *pages * buf->nbufs, GFP_KERNEL); - if (!pages) - goto err_free; - for (i = 0; i < buf->nbufs; ++i) - pages[i] = virt_to_page(buf->page_list[i].buf); - buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL); - kfree(pages); - if (!buf->direct.buf) - goto err_free; - } - } - - return 0; - -err_free: - mlx4_buf_free(dev, size, buf); - - return -ENOMEM; -} -EXPORT_SYMBOL_GPL(mlx4_buf_alloc); - -void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf) -{ - int i; - - if (buf->nbufs == 1) - dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, - buf->direct.map); - else { - if (BITS_PER_LONG == 64 && buf->direct.buf) - vunmap(buf->direct.buf); - - for (i = 0; i < buf->nbufs; ++i) - if (buf->page_list[i].buf) - dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, - buf->page_list[i].buf, - buf->page_list[i].map); - kfree(buf->page_list); - } -} -EXPORT_SYMBOL_GPL(mlx4_buf_free); - -static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device) -{ - struct mlx4_db_pgdir *pgdir; - - pgdir = kzalloc(sizeof *pgdir, GFP_KERNEL); - if (!pgdir) - return NULL; - - bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2); - pgdir->bits[0] = pgdir->order0; - pgdir->bits[1] = pgdir->order1; - pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE, - &pgdir->db_dma, GFP_KERNEL); - if (!pgdir->db_page) { - kfree(pgdir); - return NULL; - } - - return pgdir; -} - -static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, - struct mlx4_db *db, int order) -{ - int o; - int i; - - for (o = order; o <= 1; ++o) { - i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o); - if (i < MLX4_DB_PER_PAGE >> o) - goto found; - } - - return -ENOMEM; - -found: - clear_bit(i, pgdir->bits[o]); - - i <<= o; - - if (o > order) - set_bit(i ^ 1, pgdir->bits[order]); - - db->u.pgdir = pgdir; - db->index = i; - db->db = pgdir->db_page + db->index; - db->dma = pgdir->db_dma + db->index * 4; - db->order = order; - - return 0; -} - -int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_db_pgdir *pgdir; - int ret = 0; - - mutex_lock(&priv->pgdir_mutex); - - list_for_each_entry(pgdir, &priv->pgdir_list, list) - if (!mlx4_alloc_db_from_pgdir(pgdir, db, order)) - goto out; - - pgdir = mlx4_alloc_db_pgdir(&(dev->pdev->dev)); - if (!pgdir) { - ret = -ENOMEM; - goto out; - } - - list_add(&pgdir->list, &priv->pgdir_list); - - /* This should never fail -- we just allocated an empty page: */ - WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order)); - -out: - mutex_unlock(&priv->pgdir_mutex); - - return ret; -} -EXPORT_SYMBOL_GPL(mlx4_db_alloc); - -void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int o; - int i; - - mutex_lock(&priv->pgdir_mutex); - - o = db->order; - i = db->index; - - if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { - clear_bit(i ^ 1, db->u.pgdir->order0); - ++o; - } - i >>= o; - set_bit(i, db->u.pgdir->bits[o]); - - if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) { - dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, - db->u.pgdir->db_page, db->u.pgdir->db_dma); - list_del(&db->u.pgdir->list); - kfree(db->u.pgdir); - } - - mutex_unlock(&priv->pgdir_mutex); -} -EXPORT_SYMBOL_GPL(mlx4_db_free); - -int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, - int size, int max_direct) -{ - int err; - - err = mlx4_db_alloc(dev, &wqres->db, 1); - if (err) - return err; - - *wqres->db.db = 0; - - err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf); - if (err) - goto err_db; - - err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift, - &wqres->mtt); - if (err) - goto err_buf; - - err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf); - if (err) - goto err_mtt; - - return 0; - -err_mtt: - mlx4_mtt_cleanup(dev, &wqres->mtt); -err_buf: - mlx4_buf_free(dev, size, &wqres->buf); -err_db: - mlx4_db_free(dev, &wqres->db); - - return err; -} -EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res); - -void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, - int size) -{ - mlx4_mtt_cleanup(dev, &wqres->mtt); - mlx4_buf_free(dev, size, &wqres->buf); - mlx4_db_free(dev, &wqres->db); -} -EXPORT_SYMBOL_GPL(mlx4_free_hwq_res); Property changes on: stable/11/sys/ofed/drivers/net/mlx4/alloc.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/srq.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/srq.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/srq.c (nonexistent) @@ -1,314 +0,0 @@ -/* - * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include - -#include "mlx4.h" -#include "icm.h" - -void mlx4_srq_event(struct mlx4_dev *dev, u32 srqn, int event_type) -{ - struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; - struct mlx4_srq *srq; - - spin_lock(&srq_table->lock); - - srq = radix_tree_lookup(&srq_table->tree, srqn & (dev->caps.num_srqs - 1)); - if (srq) - atomic_inc(&srq->refcount); - - spin_unlock(&srq_table->lock); - - if (!srq) { - mlx4_warn(dev, "Async event for bogus SRQ %08x\n", srqn); - return; - } - - srq->event(srq, event_type); - - if (atomic_dec_and_test(&srq->refcount)) - complete(&srq->free); -} - -static int mlx4_SW2HW_SRQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, - int srq_num) -{ - return mlx4_cmd(dev, mailbox->dma, srq_num, 0, - MLX4_CMD_SW2HW_SRQ, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_WRAPPED); -} - -static int mlx4_HW2SW_SRQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, - int srq_num) -{ - return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, srq_num, - mailbox ? 0 : 1, MLX4_CMD_HW2SW_SRQ, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); -} - -static int mlx4_ARM_SRQ(struct mlx4_dev *dev, int srq_num, int limit_watermark) -{ - return mlx4_cmd(dev, limit_watermark, srq_num, 0, MLX4_CMD_ARM_SRQ, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); -} - -static int mlx4_QUERY_SRQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, - int srq_num) -{ - return mlx4_cmd_box(dev, 0, mailbox->dma, srq_num, 0, MLX4_CMD_QUERY_SRQ, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); -} - -int __mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn) -{ - struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; - int err; - - - *srqn = mlx4_bitmap_alloc(&srq_table->bitmap); - if (*srqn == -1) - return -ENOMEM; - - err = mlx4_table_get(dev, &srq_table->table, *srqn); - if (err) - goto err_out; - - err = mlx4_table_get(dev, &srq_table->cmpt_table, *srqn); - if (err) - goto err_put; - return 0; - -err_put: - mlx4_table_put(dev, &srq_table->table, *srqn); - -err_out: - mlx4_bitmap_free(&srq_table->bitmap, *srqn, MLX4_NO_RR); - return err; -} - -static int mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn) -{ - u64 out_param; - int err; - - if (mlx4_is_mfunc(dev)) { - err = mlx4_cmd_imm(dev, 0, &out_param, RES_SRQ, - RES_OP_RESERVE_AND_MAP, - MLX4_CMD_ALLOC_RES, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); - if (!err) - *srqn = get_param_l(&out_param); - - return err; - } - return __mlx4_srq_alloc_icm(dev, srqn); -} - -void __mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn) -{ - struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; - - mlx4_table_put(dev, &srq_table->cmpt_table, srqn); - mlx4_table_put(dev, &srq_table->table, srqn); - mlx4_bitmap_free(&srq_table->bitmap, srqn, MLX4_NO_RR); -} - -static void mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn) -{ - u64 in_param = 0; - - if (mlx4_is_mfunc(dev)) { - set_param_l(&in_param, srqn); - if (mlx4_cmd(dev, in_param, RES_SRQ, RES_OP_RESERVE_AND_MAP, - MLX4_CMD_FREE_RES, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) - mlx4_warn(dev, "Failed freeing cq:%d\n", srqn); - return; - } - __mlx4_srq_free_icm(dev, srqn); -} - -int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, u32 cqn, u16 xrcd, - struct mlx4_mtt *mtt, u64 db_rec, struct mlx4_srq *srq) -{ - struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_srq_context *srq_context; - u64 mtt_addr; - int err; - - err = mlx4_srq_alloc_icm(dev, &srq->srqn); - if (err) - return err; - - spin_lock_irq(&srq_table->lock); - err = radix_tree_insert(&srq_table->tree, srq->srqn, srq); - spin_unlock_irq(&srq_table->lock); - if (err) - goto err_icm; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) { - err = PTR_ERR(mailbox); - goto err_radix; - } - - srq_context = mailbox->buf; - memset(srq_context, 0, sizeof *srq_context); - - srq_context->state_logsize_srqn = cpu_to_be32((ilog2(srq->max) << 24) | - srq->srqn); - srq_context->logstride = srq->wqe_shift - 4; - srq_context->xrcd = cpu_to_be16(xrcd); - srq_context->pg_offset_cqn = cpu_to_be32(cqn & 0xffffff); - srq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT; - - mtt_addr = mlx4_mtt_addr(dev, mtt); - srq_context->mtt_base_addr_h = mtt_addr >> 32; - srq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); - srq_context->pd = cpu_to_be32(pdn); - srq_context->db_rec_addr = cpu_to_be64(db_rec); - - err = mlx4_SW2HW_SRQ(dev, mailbox, srq->srqn); - mlx4_free_cmd_mailbox(dev, mailbox); - if (err) - goto err_radix; - - atomic_set(&srq->refcount, 1); - init_completion(&srq->free); - - return 0; - -err_radix: - spin_lock_irq(&srq_table->lock); - radix_tree_delete(&srq_table->tree, srq->srqn); - spin_unlock_irq(&srq_table->lock); - -err_icm: - mlx4_srq_free_icm(dev, srq->srqn); - return err; -} -EXPORT_SYMBOL_GPL(mlx4_srq_alloc); - -void mlx4_srq_free(struct mlx4_dev *dev, struct mlx4_srq *srq) -{ - struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; - int err; - - err = mlx4_HW2SW_SRQ(dev, NULL, srq->srqn); - if (err) - mlx4_warn(dev, "HW2SW_SRQ failed (%d) for SRQN %06x\n", err, srq->srqn); - - spin_lock_irq(&srq_table->lock); - radix_tree_delete(&srq_table->tree, srq->srqn); - spin_unlock_irq(&srq_table->lock); - - if (atomic_dec_and_test(&srq->refcount)) - complete(&srq->free); - wait_for_completion(&srq->free); - - mlx4_srq_free_icm(dev, srq->srqn); -} -EXPORT_SYMBOL_GPL(mlx4_srq_free); - -int mlx4_srq_arm(struct mlx4_dev *dev, struct mlx4_srq *srq, int limit_watermark) -{ - return mlx4_ARM_SRQ(dev, srq->srqn, limit_watermark); -} -EXPORT_SYMBOL_GPL(mlx4_srq_arm); - -int mlx4_srq_query(struct mlx4_dev *dev, struct mlx4_srq *srq, int *limit_watermark) -{ - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_srq_context *srq_context; - int err; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - srq_context = mailbox->buf; - - err = mlx4_QUERY_SRQ(dev, mailbox, srq->srqn); - if (err) - goto err_out; - *limit_watermark = be16_to_cpu(srq_context->limit_watermark); - -err_out: - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} -EXPORT_SYMBOL_GPL(mlx4_srq_query); - -int mlx4_init_srq_table(struct mlx4_dev *dev) -{ - struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; - int err; - - spin_lock_init(&srq_table->lock); - INIT_RADIX_TREE(&srq_table->tree, GFP_ATOMIC); - if (mlx4_is_slave(dev)) - return 0; - - err = mlx4_bitmap_init(&srq_table->bitmap, dev->caps.num_srqs, - dev->caps.num_srqs - 1, dev->caps.reserved_srqs, 0); - if (err) - return err; - - return 0; -} - -void mlx4_cleanup_srq_table(struct mlx4_dev *dev) -{ - if (mlx4_is_slave(dev)) - return; - mlx4_bitmap_cleanup(&mlx4_priv(dev)->srq_table.bitmap); -} - -struct mlx4_srq *mlx4_srq_lookup(struct mlx4_dev *dev, u32 srqn) -{ - struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; - struct mlx4_srq *srq; - unsigned long flags; - - spin_lock_irqsave(&srq_table->lock, flags); - srq = radix_tree_lookup(&srq_table->tree, - srqn & (dev->caps.num_srqs - 1)); - spin_unlock_irqrestore(&srq_table->lock, flags); - - return srq; -} -EXPORT_SYMBOL_GPL(mlx4_srq_lookup); Property changes on: stable/11/sys/ofed/drivers/net/mlx4/srq.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/main.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/main.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/main.c (nonexistent) @@ -1,3942 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#define LINUXKPI_PARAM_PREFIX mlx4_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "mlx4.h" -#include "fw.h" -#include "icm.h" -#include "mlx4_stats.h" - -/* Mellanox ConnectX HCA low-level driver */ - -struct workqueue_struct *mlx4_wq; - -#ifdef CONFIG_MLX4_DEBUG - -int mlx4_debug_level = 0; -module_param_named(debug_level, mlx4_debug_level, int, 0644); -MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0"); - -#endif /* CONFIG_MLX4_DEBUG */ - -#ifdef CONFIG_PCI_MSI - -static int msi_x = 1; -module_param(msi_x, int, 0444); -MODULE_PARM_DESC(msi_x, "0 - don't use MSI-X, 1 - use MSI-X, >1 - limit number of MSI-X irqs to msi_x (non-SRIOV only)"); - -#else /* CONFIG_PCI_MSI */ - -#define msi_x (0) - -#endif /* CONFIG_PCI_MSI */ - -static int enable_sys_tune = 0; -module_param(enable_sys_tune, int, 0444); -MODULE_PARM_DESC(enable_sys_tune, "Tune the cpu's for better performance (default 0)"); - -int mlx4_blck_lb = 1; -module_param_named(block_loopback, mlx4_blck_lb, int, 0644); -MODULE_PARM_DESC(block_loopback, "Block multicast loopback packets if > 0 " - "(default: 1)"); -enum { - DEFAULT_DOMAIN = 0, - BDF_STR_SIZE = 8, /* bb:dd.f- */ - DBDF_STR_SIZE = 13 /* mmmm:bb:dd.f- */ -}; - -enum { - NUM_VFS, - PROBE_VF, - PORT_TYPE_ARRAY -}; - -enum { - VALID_DATA, - INVALID_DATA, - INVALID_STR -}; - -struct param_data { - int id; - struct mlx4_dbdf2val_lst dbdf2val; -}; - -static struct param_data num_vfs = { - .id = NUM_VFS, - .dbdf2val = { - .name = "num_vfs param", - .num_vals = 1, - .def_val = {0}, - .range = {0, MLX4_MAX_NUM_VF} - } -}; -module_param_string(num_vfs, num_vfs.dbdf2val.str, - sizeof(num_vfs.dbdf2val.str), 0444); -MODULE_PARM_DESC(num_vfs, - "Either single value (e.g. '5') to define uniform num_vfs value for all devices functions\n" - "\t\tor a string to map device function numbers to their num_vfs values (e.g. '0000:04:00.0-5,002b:1c:0b.a-15').\n" - "\t\tHexadecimal digits for the device function (e.g. 002b:1c:0b.a) and decimal for num_vfs value (e.g. 15)."); - -static struct param_data probe_vf = { - .id = PROBE_VF, - .dbdf2val = { - .name = "probe_vf param", - .num_vals = 1, - .def_val = {0}, - .range = {0, MLX4_MAX_NUM_VF} - } -}; -module_param_string(probe_vf, probe_vf.dbdf2val.str, - sizeof(probe_vf.dbdf2val.str), 0444); -MODULE_PARM_DESC(probe_vf, - "Either single value (e.g. '3') to define uniform number of VFs to probe by the pf driver for all devices functions\n" - "\t\tor a string to map device function numbers to their probe_vf values (e.g. '0000:04:00.0-3,002b:1c:0b.a-13').\n" - "\t\tHexadecimal digits for the device function (e.g. 002b:1c:0b.a) and decimal for probe_vf value (e.g. 13)."); - -int mlx4_log_num_mgm_entry_size = MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE; - -module_param_named(log_num_mgm_entry_size, - mlx4_log_num_mgm_entry_size, int, 0444); -MODULE_PARM_DESC(log_num_mgm_entry_size, "log mgm size, that defines the num" - " of qp per mcg, for example:" - " 10 gives 248.range: 7 <=" - " log_num_mgm_entry_size <= 12." - " To activate device managed" - " flow steering when available, set to -1"); - -static int high_rate_steer; -module_param(high_rate_steer, int, 0444); -MODULE_PARM_DESC(high_rate_steer, "Enable steering mode for higher packet rate" - " (default off)"); - -static int fast_drop; -module_param_named(fast_drop, fast_drop, int, 0444); -MODULE_PARM_DESC(fast_drop, - "Enable fast packet drop when no receive WQEs are posted"); - -int mlx4_enable_64b_cqe_eqe = 1; -module_param_named(enable_64b_cqe_eqe, mlx4_enable_64b_cqe_eqe, int, 0644); -MODULE_PARM_DESC(enable_64b_cqe_eqe, - "Enable 64 byte CQEs/EQEs when the FW supports this if non-zero (default: 1)"); - -#define HCA_GLOBAL_CAP_MASK 0 - -#define PF_CONTEXT_BEHAVIOUR_MASK MLX4_FUNC_CAP_64B_EQE_CQE - -static char mlx4_version[] __devinitdata = - DRV_NAME ": Mellanox ConnectX VPI driver v" - DRV_VERSION " (" DRV_RELDATE ")\n"; - -static int log_num_mac = 7; -module_param_named(log_num_mac, log_num_mac, int, 0444); -MODULE_PARM_DESC(log_num_mac, "Log2 max number of MACs per ETH port (1-7)"); - -static int log_num_vlan; -module_param_named(log_num_vlan, log_num_vlan, int, 0444); -MODULE_PARM_DESC(log_num_vlan, - "(Obsolete) Log2 max number of VLANs per ETH port (0-7)"); -/* Log2 max number of VLANs per ETH port (0-7) */ -#define MLX4_LOG_NUM_VLANS 7 - -int log_mtts_per_seg = ilog2(1); -module_param_named(log_mtts_per_seg, log_mtts_per_seg, int, 0444); -MODULE_PARM_DESC(log_mtts_per_seg, "Log2 number of MTT entries per segment " - "(0-7) (default: 0)"); - -static struct param_data port_type_array = { - .id = PORT_TYPE_ARRAY, - .dbdf2val = { - .name = "port_type_array param", - .num_vals = 2, - .def_val = {MLX4_PORT_TYPE_ETH, MLX4_PORT_TYPE_ETH}, - .range = {MLX4_PORT_TYPE_IB, MLX4_PORT_TYPE_NA} - } -}; -module_param_string(port_type_array, port_type_array.dbdf2val.str, - sizeof(port_type_array.dbdf2val.str), 0444); -MODULE_PARM_DESC(port_type_array, - "Either pair of values (e.g. '1,2') to define uniform port1/port2 types configuration for all devices functions\n" - "\t\tor a string to map device function numbers to their pair of port types values (e.g. '0000:04:00.0-1;2,002b:1c:0b.a-1;1').\n" - "\t\tValid port types: 1-ib, 2-eth, 3-auto, 4-N/A\n" - "\t\tIn case that only one port is available use the N/A port type for port2 (e.g '1,4')."); - - -struct mlx4_port_config { - struct list_head list; - enum mlx4_port_type port_type[MLX4_MAX_PORTS + 1]; - struct pci_dev *pdev; -}; - -#define MLX4_LOG_NUM_MTT 20 -/* We limit to 30 as of a bit map issue which uses int and not uint. - see mlx4_buddy_init -> bitmap_zero which gets int. -*/ -#define MLX4_MAX_LOG_NUM_MTT 30 -static struct mlx4_profile mod_param_profile = { - .num_qp = 19, - .num_srq = 16, - .rdmarc_per_qp = 4, - .num_cq = 16, - .num_mcg = 13, - .num_mpt = 19, - .num_mtt_segs = 0, /* max(20, 2*MTTs for host memory)) */ -}; - -module_param_named(log_num_qp, mod_param_profile.num_qp, int, 0444); -MODULE_PARM_DESC(log_num_qp, "log maximum number of QPs per HCA (default: 19)"); - -module_param_named(log_num_srq, mod_param_profile.num_srq, int, 0444); -MODULE_PARM_DESC(log_num_srq, "log maximum number of SRQs per HCA " - "(default: 16)"); - -module_param_named(log_rdmarc_per_qp, mod_param_profile.rdmarc_per_qp, int, - 0444); -MODULE_PARM_DESC(log_rdmarc_per_qp, "log number of RDMARC buffers per QP " - "(default: 4)"); - -module_param_named(log_num_cq, mod_param_profile.num_cq, int, 0444); -MODULE_PARM_DESC(log_num_cq, "log maximum number of CQs per HCA (default: 16)"); - -module_param_named(log_num_mcg, mod_param_profile.num_mcg, int, 0444); -MODULE_PARM_DESC(log_num_mcg, "log maximum number of multicast groups per HCA " - "(default: 13)"); - -module_param_named(log_num_mpt, mod_param_profile.num_mpt, int, 0444); -MODULE_PARM_DESC(log_num_mpt, - "log maximum number of memory protection table entries per " - "HCA (default: 19)"); - -module_param_named(log_num_mtt, mod_param_profile.num_mtt_segs, int, 0444); -MODULE_PARM_DESC(log_num_mtt, - "log maximum number of memory translation table segments per " - "HCA (default: max(20, 2*MTTs for register all of the host memory limited to 30))"); - -enum { - MLX4_IF_STATE_BASIC, - MLX4_IF_STATE_EXTENDED -}; - -static inline u64 dbdf_to_u64(int domain, int bus, int dev, int fn) -{ - return (domain << 20) | (bus << 12) | (dev << 4) | fn; -} - -static inline void pr_bdf_err(const char *dbdf, const char *pname) -{ - pr_warn("mlx4_core: '%s' is not valid bdf in '%s'\n", dbdf, pname); -} - -static inline void pr_val_err(const char *dbdf, const char *pname, - const char *val) -{ - pr_warn("mlx4_core: value '%s' of bdf '%s' in '%s' is not valid\n" - , val, dbdf, pname); -} - -static inline void pr_out_of_range_bdf(const char *dbdf, int val, - struct mlx4_dbdf2val_lst *dbdf2val) -{ - pr_warn("mlx4_core: value %d in bdf '%s' of '%s' is out of its valid range (%d,%d)\n" - , val, dbdf, dbdf2val->name , dbdf2val->range.min, - dbdf2val->range.max); -} - -static inline void pr_out_of_range(struct mlx4_dbdf2val_lst *dbdf2val) -{ - pr_warn("mlx4_core: value of '%s' is out of its valid range (%d,%d)\n" - , dbdf2val->name , dbdf2val->range.min, dbdf2val->range.max); -} - -static inline int is_in_range(int val, struct mlx4_range *r) -{ - return (val >= r->min && val <= r->max); -} - -static int update_defaults(struct param_data *pdata) -{ - long int val[MLX4_MAX_BDF_VALS]; - int ret; - char *t, *p = pdata->dbdf2val.str; - char sval[32]; - int val_len; - - if (!strlen(p) || strchr(p, ':') || strchr(p, '.') || strchr(p, ';')) - return INVALID_STR; - - switch (pdata->id) { - case PORT_TYPE_ARRAY: - t = strchr(p, ','); - if (!t || t == p || (t - p) > sizeof(sval)) - return INVALID_STR; - - val_len = t - p; - strncpy(sval, p, val_len); - sval[val_len] = 0; - - ret = kstrtol(sval, 0, &val[0]); - if (ret == -EINVAL) - return INVALID_STR; - if (ret || !is_in_range(val[0], &pdata->dbdf2val.range)) { - pr_out_of_range(&pdata->dbdf2val); - return INVALID_DATA; - } - - ret = kstrtol(t + 1, 0, &val[1]); - if (ret == -EINVAL) - return INVALID_STR; - if (ret || !is_in_range(val[1], &pdata->dbdf2val.range)) { - pr_out_of_range(&pdata->dbdf2val); - return INVALID_DATA; - } - - pdata->dbdf2val.tbl[0].val[0] = val[0]; - pdata->dbdf2val.tbl[0].val[1] = val[1]; - break; - - case NUM_VFS: - case PROBE_VF: - ret = kstrtol(p, 0, &val[0]); - if (ret == -EINVAL) - return INVALID_STR; - if (ret || !is_in_range(val[0], &pdata->dbdf2val.range)) { - pr_out_of_range(&pdata->dbdf2val); - return INVALID_DATA; - } - pdata->dbdf2val.tbl[0].val[0] = val[0]; - break; - } - pdata->dbdf2val.tbl[1].dbdf = MLX4_ENDOF_TBL; - - return VALID_DATA; -} - -int mlx4_fill_dbdf2val_tbl(struct mlx4_dbdf2val_lst *dbdf2val_lst) -{ - int domain, bus, dev, fn; - u64 dbdf; - char *p, *t, *v; - char tmp[32]; - char sbdf[32]; - char sep = ','; - int j, k, str_size, i = 1; - int prfx_size; - - p = dbdf2val_lst->str; - - for (j = 0; j < dbdf2val_lst->num_vals; j++) - dbdf2val_lst->tbl[0].val[j] = dbdf2val_lst->def_val[j]; - dbdf2val_lst->tbl[1].dbdf = MLX4_ENDOF_TBL; - - str_size = strlen(dbdf2val_lst->str); - - if (str_size == 0) - return 0; - - while (strlen(p)) { - prfx_size = BDF_STR_SIZE; - sbdf[prfx_size] = 0; - strncpy(sbdf, p, prfx_size); - domain = DEFAULT_DOMAIN; - if (sscanf(sbdf, "%02x:%02x.%x-", &bus, &dev, &fn) != 3) { - prfx_size = DBDF_STR_SIZE; - sbdf[prfx_size] = 0; - strncpy(sbdf, p, prfx_size); - if (sscanf(sbdf, "%04x:%02x:%02x.%x-", &domain, &bus, - &dev, &fn) != 4) { - pr_bdf_err(sbdf, dbdf2val_lst->name); - goto err; - } - sprintf(tmp, "%04x:%02x:%02x.%x-", domain, bus, dev, - fn); - } else { - sprintf(tmp, "%02x:%02x.%x-", bus, dev, fn); - } - - if (strnicmp(sbdf, tmp, sizeof(tmp))) { - pr_bdf_err(sbdf, dbdf2val_lst->name); - goto err; - } - - dbdf = dbdf_to_u64(domain, bus, dev, fn); - - for (j = 1; j < i; j++) - if (dbdf2val_lst->tbl[j].dbdf == dbdf) { - pr_warn("mlx4_core: in '%s', %s appears multiple times\n" - , dbdf2val_lst->name, sbdf); - goto err; - } - - if (i >= MLX4_DEVS_TBL_SIZE) { - pr_warn("mlx4_core: Too many devices in '%s'\n" - , dbdf2val_lst->name); - goto err; - } - - p += prfx_size; - t = strchr(p, sep); - t = t ? t : p + strlen(p); - if (p >= t) { - pr_val_err(sbdf, dbdf2val_lst->name, ""); - goto err; - } - - for (k = 0; k < dbdf2val_lst->num_vals; k++) { - char sval[32]; - long int val; - int ret, val_len; - char vsep = ';'; - - v = (k == dbdf2val_lst->num_vals - 1) ? t : strchr(p, vsep); - if (!v || v > t || v == p || (v - p) > sizeof(sval)) { - pr_val_err(sbdf, dbdf2val_lst->name, p); - goto err; - } - val_len = v - p; - strncpy(sval, p, val_len); - sval[val_len] = 0; - - ret = kstrtol(sval, 0, &val); - if (ret) { - if (strchr(p, vsep)) - pr_warn("mlx4_core: too many vals in bdf '%s' of '%s'\n" - , sbdf, dbdf2val_lst->name); - else - pr_val_err(sbdf, dbdf2val_lst->name, - sval); - goto err; - } - if (!is_in_range(val, &dbdf2val_lst->range)) { - pr_out_of_range_bdf(sbdf, val, dbdf2val_lst); - goto err; - } - - dbdf2val_lst->tbl[i].val[k] = val; - p = v; - if (p[0] == vsep) - p++; - } - - dbdf2val_lst->tbl[i].dbdf = dbdf; - if (strlen(p)) { - if (p[0] != sep) { - pr_warn("mlx4_core: expect separator '%c' before '%s' in '%s'\n" - , sep, p, dbdf2val_lst->name); - goto err; - } - p++; - } - i++; - if (i < MLX4_DEVS_TBL_SIZE) - dbdf2val_lst->tbl[i].dbdf = MLX4_ENDOF_TBL; - } - - return 0; - -err: - dbdf2val_lst->tbl[1].dbdf = MLX4_ENDOF_TBL; - pr_warn("mlx4_core: The value of '%s' is incorrect. The value is discarded!\n" - , dbdf2val_lst->name); - - return -EINVAL; -} -EXPORT_SYMBOL(mlx4_fill_dbdf2val_tbl); - -int mlx4_get_val(struct mlx4_dbdf2val *tbl, struct pci_dev *pdev, int idx, - int *val) -{ - u64 dbdf; - int i = 1; - - *val = tbl[0].val[idx]; - if (!pdev) - return -EINVAL; - - dbdf = dbdf_to_u64(pci_get_domain(pdev->dev.bsddev), pci_get_bus(pdev->dev.bsddev), - PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn)); - - while ((i < MLX4_DEVS_TBL_SIZE) && (tbl[i].dbdf != MLX4_ENDOF_TBL)) { - if (tbl[i].dbdf == dbdf) { - *val = tbl[i].val[idx]; - return 0; - } - i++; - } - - return 0; -} -EXPORT_SYMBOL(mlx4_get_val); - -static void process_mod_param_profile(struct mlx4_profile *profile) -{ - vm_size_t hwphyssz; - hwphyssz = 0; - TUNABLE_ULONG_FETCH("hw.realmem", (u_long *) &hwphyssz); - - profile->num_qp = 1 << mod_param_profile.num_qp; - profile->num_srq = 1 << mod_param_profile.num_srq; - profile->rdmarc_per_qp = 1 << mod_param_profile.rdmarc_per_qp; - profile->num_cq = 1 << mod_param_profile.num_cq; - profile->num_mcg = 1 << mod_param_profile.num_mcg; - profile->num_mpt = 1 << mod_param_profile.num_mpt; - /* - * We want to scale the number of MTTs with the size of the - * system memory, since it makes sense to register a lot of - * memory on a system with a lot of memory. As a heuristic, - * make sure we have enough MTTs to register twice the system - * memory (with PAGE_SIZE entries). - * - * This number has to be a power of two and fit into 32 bits - * due to device limitations. We cap this at 2^30 as of bit map - * limitation to work with int instead of uint (mlx4_buddy_init -> bitmap_zero) - * That limits us to 4TB of memory registration per HCA with - * 4KB pages, which is probably OK for the next few months. - */ - if (mod_param_profile.num_mtt_segs) - profile->num_mtt_segs = 1 << mod_param_profile.num_mtt_segs; - else { - profile->num_mtt_segs = - roundup_pow_of_two(max_t(unsigned, - 1 << (MLX4_LOG_NUM_MTT - log_mtts_per_seg), - min(1UL << - (MLX4_MAX_LOG_NUM_MTT - - log_mtts_per_seg), - (hwphyssz << 1) - >> log_mtts_per_seg))); - /* set the actual value, so it will be reflected to the user - using the sysfs */ - mod_param_profile.num_mtt_segs = ilog2(profile->num_mtt_segs); - } -} - -int mlx4_check_port_params(struct mlx4_dev *dev, - enum mlx4_port_type *port_type) -{ - int i; - - for (i = 0; i < dev->caps.num_ports - 1; i++) { - if (port_type[i] != port_type[i + 1]) { - if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) { - mlx4_err(dev, "Only same port types supported " - "on this HCA, aborting.\n"); - return -EINVAL; - } - } - } - - for (i = 0; i < dev->caps.num_ports; i++) { - if (!(port_type[i] & dev->caps.supported_type[i+1])) { - mlx4_err(dev, "Requested port type for port %d is not " - "supported on this HCA\n", i + 1); - return -EINVAL; - } - } - return 0; -} - -static void mlx4_set_port_mask(struct mlx4_dev *dev) -{ - int i; - - for (i = 1; i <= dev->caps.num_ports; ++i) - dev->caps.port_mask[i] = dev->caps.port_type[i]; -} - -enum { - MLX4_QUERY_FUNC_NUM_SYS_EQS = 1 << 0, -}; - -static int mlx4_query_func(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) -{ - int err = 0; - struct mlx4_func func; - - if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) { - err = mlx4_QUERY_FUNC(dev, &func, 0); - if (err) { - mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n"); - return err; - } - dev_cap->max_eqs = func.max_eq; - dev_cap->reserved_eqs = func.rsvd_eqs; - dev_cap->reserved_uars = func.rsvd_uars; - err |= MLX4_QUERY_FUNC_NUM_SYS_EQS; - } - return err; -} - -static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) -{ - int err; - int i; - - err = mlx4_QUERY_DEV_CAP(dev, dev_cap); - if (err) { - mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n"); - return err; - } - - if (dev_cap->min_page_sz > PAGE_SIZE) { - mlx4_err(dev, "HCA minimum page size of %d bigger than " - "kernel PAGE_SIZE of %d, aborting.\n", - dev_cap->min_page_sz, (int)PAGE_SIZE); - return -ENODEV; - } - if (dev_cap->num_ports > MLX4_MAX_PORTS) { - mlx4_err(dev, "HCA has %d ports, but we only support %d, " - "aborting.\n", - dev_cap->num_ports, MLX4_MAX_PORTS); - return -ENODEV; - } - - if (dev_cap->uar_size > pci_resource_len(dev->pdev, 2)) { - mlx4_err(dev, "HCA reported UAR size of 0x%x bigger than " - "PCI resource 2 size of 0x%llx, aborting.\n", - dev_cap->uar_size, - (unsigned long long) pci_resource_len(dev->pdev, 2)); - return -ENODEV; - } - - dev->caps.num_ports = dev_cap->num_ports; - dev->caps.num_sys_eqs = dev_cap->num_sys_eqs; - dev->phys_caps.num_phys_eqs = dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS ? - dev->caps.num_sys_eqs : - MLX4_MAX_EQ_NUM; - for (i = 1; i <= dev->caps.num_ports; ++i) { - dev->caps.vl_cap[i] = dev_cap->max_vl[i]; - dev->caps.ib_mtu_cap[i] = dev_cap->ib_mtu[i]; - dev->phys_caps.gid_phys_table_len[i] = dev_cap->max_gids[i]; - dev->phys_caps.pkey_phys_table_len[i] = dev_cap->max_pkeys[i]; - /* set gid and pkey table operating lengths by default - * to non-sriov values */ - dev->caps.gid_table_len[i] = dev_cap->max_gids[i]; - dev->caps.pkey_table_len[i] = dev_cap->max_pkeys[i]; - dev->caps.port_width_cap[i] = dev_cap->max_port_width[i]; - dev->caps.eth_mtu_cap[i] = dev_cap->eth_mtu[i]; - dev->caps.def_mac[i] = dev_cap->def_mac[i]; - dev->caps.supported_type[i] = dev_cap->supported_port_types[i]; - dev->caps.suggested_type[i] = dev_cap->suggested_type[i]; - dev->caps.default_sense[i] = dev_cap->default_sense[i]; - dev->caps.trans_type[i] = dev_cap->trans_type[i]; - dev->caps.vendor_oui[i] = dev_cap->vendor_oui[i]; - dev->caps.wavelength[i] = dev_cap->wavelength[i]; - dev->caps.trans_code[i] = dev_cap->trans_code[i]; - } - - dev->caps.uar_page_size = PAGE_SIZE; - dev->caps.num_uars = dev_cap->uar_size / PAGE_SIZE; - dev->caps.local_ca_ack_delay = dev_cap->local_ca_ack_delay; - dev->caps.bf_reg_size = dev_cap->bf_reg_size; - dev->caps.bf_regs_per_page = dev_cap->bf_regs_per_page; - dev->caps.max_sq_sg = dev_cap->max_sq_sg; - dev->caps.max_rq_sg = dev_cap->max_rq_sg; - dev->caps.max_wqes = dev_cap->max_qp_sz; - dev->caps.max_qp_init_rdma = dev_cap->max_requester_per_qp; - dev->caps.max_srq_wqes = dev_cap->max_srq_sz; - dev->caps.max_srq_sge = dev_cap->max_rq_sg - 1; - dev->caps.reserved_srqs = dev_cap->reserved_srqs; - dev->caps.max_sq_desc_sz = dev_cap->max_sq_desc_sz; - dev->caps.max_rq_desc_sz = dev_cap->max_rq_desc_sz; - /* - * Subtract 1 from the limit because we need to allocate a - * spare CQE to enable resizing the CQ - */ - dev->caps.max_cqes = dev_cap->max_cq_sz - 1; - dev->caps.reserved_cqs = dev_cap->reserved_cqs; - dev->caps.reserved_eqs = dev_cap->reserved_eqs; - dev->caps.reserved_mtts = dev_cap->reserved_mtts; - dev->caps.reserved_mrws = dev_cap->reserved_mrws; - - /* The first 128 UARs are used for EQ doorbells */ - dev->caps.reserved_uars = max_t(int, 128, dev_cap->reserved_uars); - dev->caps.reserved_pds = dev_cap->reserved_pds; - dev->caps.reserved_xrcds = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ? - dev_cap->reserved_xrcds : 0; - dev->caps.max_xrcds = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ? - dev_cap->max_xrcds : 0; - dev->caps.mtt_entry_sz = dev_cap->mtt_entry_sz; - - dev->caps.max_msg_sz = dev_cap->max_msg_sz; - dev->caps.page_size_cap = ~(u32) (dev_cap->min_page_sz - 1); - dev->caps.flags = dev_cap->flags; - dev->caps.flags2 = dev_cap->flags2; - dev->caps.bmme_flags = dev_cap->bmme_flags; - dev->caps.reserved_lkey = dev_cap->reserved_lkey; - dev->caps.stat_rate_support = dev_cap->stat_rate_support; - dev->caps.cq_timestamp = dev_cap->timestamp_support; - dev->caps.max_gso_sz = dev_cap->max_gso_sz; - dev->caps.max_rss_tbl_sz = dev_cap->max_rss_tbl_sz; - - /* Sense port always allowed on supported devices for ConnectX-1 and -2 */ - if (mlx4_priv(dev)->pci_dev_data & MLX4_PCI_DEV_FORCE_SENSE_PORT) - dev->caps.flags |= MLX4_DEV_CAP_FLAG_SENSE_SUPPORT; - /* Don't do sense port on multifunction devices (for now at least) */ - if (mlx4_is_mfunc(dev)) - dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_SENSE_SUPPORT; - - dev->caps.log_num_macs = log_num_mac; - dev->caps.log_num_vlans = MLX4_LOG_NUM_VLANS; - - dev->caps.fast_drop = fast_drop ? - !!(dev->caps.flags & MLX4_DEV_CAP_FLAG_FAST_DROP) : - 0; - - for (i = 1; i <= dev->caps.num_ports; ++i) { - dev->caps.port_type[i] = MLX4_PORT_TYPE_NONE; - if (dev->caps.supported_type[i]) { - /* if only ETH is supported - assign ETH */ - if (dev->caps.supported_type[i] == MLX4_PORT_TYPE_ETH) - dev->caps.port_type[i] = MLX4_PORT_TYPE_ETH; - /* if only IB is supported, assign IB */ - else if (dev->caps.supported_type[i] == - MLX4_PORT_TYPE_IB) - dev->caps.port_type[i] = MLX4_PORT_TYPE_IB; - else { - /* - * if IB and ETH are supported, we set the port - * type according to user selection of port type; - * if there is no user selection, take the FW hint - */ - int pta; - mlx4_get_val(port_type_array.dbdf2val.tbl, - pci_physfn(dev->pdev), i - 1, - &pta); - if (pta == MLX4_PORT_TYPE_NONE) { - dev->caps.port_type[i] = dev->caps.suggested_type[i] ? - MLX4_PORT_TYPE_ETH : MLX4_PORT_TYPE_IB; - } else if (pta == MLX4_PORT_TYPE_NA) { - mlx4_err(dev, "Port %d is valid port. " - "It is not allowed to configure its type to N/A(%d)\n", - i, MLX4_PORT_TYPE_NA); - return -EINVAL; - } else { - dev->caps.port_type[i] = pta; - } - } - } - /* - * Link sensing is allowed on the port if 3 conditions are true: - * 1. Both protocols are supported on the port. - * 2. Different types are supported on the port - * 3. FW declared that it supports link sensing - */ - mlx4_priv(dev)->sense.sense_allowed[i] = - ((dev->caps.supported_type[i] == MLX4_PORT_TYPE_AUTO) && - (dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP) && - (dev->caps.flags & MLX4_DEV_CAP_FLAG_SENSE_SUPPORT)); - - /* Disablling auto sense for default Eth ports support */ - mlx4_priv(dev)->sense.sense_allowed[i] = 0; - - /* - * If "default_sense" bit is set, we move the port to "AUTO" mode - * and perform sense_port FW command to try and set the correct - * port type from beginning - */ - if (mlx4_priv(dev)->sense.sense_allowed[i] && dev->caps.default_sense[i]) { - enum mlx4_port_type sensed_port = MLX4_PORT_TYPE_NONE; - dev->caps.possible_type[i] = MLX4_PORT_TYPE_AUTO; - mlx4_SENSE_PORT(dev, i, &sensed_port); - if (sensed_port != MLX4_PORT_TYPE_NONE) - dev->caps.port_type[i] = sensed_port; - } else { - dev->caps.possible_type[i] = dev->caps.port_type[i]; - } - - if (dev->caps.log_num_macs > dev_cap->log_max_macs[i]) { - dev->caps.log_num_macs = dev_cap->log_max_macs[i]; - mlx4_warn(dev, "Requested number of MACs is too much " - "for port %d, reducing to %d.\n", - i, 1 << dev->caps.log_num_macs); - } - if (dev->caps.log_num_vlans > dev_cap->log_max_vlans[i]) { - dev->caps.log_num_vlans = dev_cap->log_max_vlans[i]; - mlx4_warn(dev, "Requested number of VLANs is too much " - "for port %d, reducing to %d.\n", - i, 1 << dev->caps.log_num_vlans); - } - } - - dev->caps.max_basic_counters = dev_cap->max_basic_counters; - dev->caps.max_extended_counters = dev_cap->max_extended_counters; - /* support extended counters if available */ - if (dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS_EXT) - dev->caps.max_counters = dev->caps.max_extended_counters; - else - dev->caps.max_counters = dev->caps.max_basic_counters; - - dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] = dev_cap->reserved_qps; - dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] = - dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] = - (1 << dev->caps.log_num_macs) * - (1 << dev->caps.log_num_vlans) * - dev->caps.num_ports; - dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH] = MLX4_NUM_FEXCH; - - dev->caps.reserved_qps = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] + - dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] + - dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] + - dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH]; - - dev->caps.sync_qp = dev_cap->sync_qp; - if (dev->pdev->device == 0x1003) - dev->caps.cq_flags |= MLX4_DEV_CAP_CQ_FLAG_IO; - - dev->caps.sqp_demux = (mlx4_is_master(dev)) ? MLX4_MAX_NUM_SLAVES : 0; - - if (!mlx4_enable_64b_cqe_eqe && !mlx4_is_slave(dev)) { - if (dev_cap->flags & - (MLX4_DEV_CAP_FLAG_64B_CQE | MLX4_DEV_CAP_FLAG_64B_EQE)) { - mlx4_warn(dev, "64B EQEs/CQEs supported by the device but not enabled\n"); - dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_CQE; - dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_EQE; - } - } - - if ((dev->caps.flags & - (MLX4_DEV_CAP_FLAG_64B_CQE | MLX4_DEV_CAP_FLAG_64B_EQE)) && - mlx4_is_master(dev)) - dev->caps.function_caps |= MLX4_FUNC_CAP_64B_EQE_CQE; - - if (!mlx4_is_slave(dev)) { - for (i = 0; i < dev->caps.num_ports; ++i) - dev->caps.def_counter_index[i] = i << 1; - - dev->caps.alloc_res_qp_mask = - (dev->caps.bf_reg_size ? MLX4_RESERVE_ETH_BF_QP : 0); - } else { - dev->caps.alloc_res_qp_mask = 0; - } - - return 0; -} -/*The function checks if there are live vf, return the num of them*/ -static int mlx4_how_many_lives_vf(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_slave_state *s_state; - int i; - int ret = 0; - - for (i = 1/*the ppf is 0*/; i < dev->num_slaves; ++i) { - s_state = &priv->mfunc.master.slave_state[i]; - if (s_state->active && s_state->last_cmd != - MLX4_COMM_CMD_RESET) { - mlx4_warn(dev, "%s: slave: %d is still active\n", - __func__, i); - ret++; - } - } - return ret; -} - -int mlx4_get_parav_qkey(struct mlx4_dev *dev, u32 qpn, u32 *qkey) -{ - u32 qk = MLX4_RESERVED_QKEY_BASE; - - if (qpn >= dev->phys_caps.base_tunnel_sqpn + 8 * MLX4_MFUNC_MAX || - qpn < dev->phys_caps.base_proxy_sqpn) - return -EINVAL; - - if (qpn >= dev->phys_caps.base_tunnel_sqpn) - /* tunnel qp */ - qk += qpn - dev->phys_caps.base_tunnel_sqpn; - else - qk += qpn - dev->phys_caps.base_proxy_sqpn; - *qkey = qk; - return 0; -} -EXPORT_SYMBOL(mlx4_get_parav_qkey); - -void mlx4_sync_pkey_table(struct mlx4_dev *dev, int slave, int port, int i, int val) -{ - struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev); - - if (!mlx4_is_master(dev)) - return; - - priv->virt2phys_pkey[slave][port - 1][i] = val; -} -EXPORT_SYMBOL(mlx4_sync_pkey_table); - -void mlx4_put_slave_node_guid(struct mlx4_dev *dev, int slave, __be64 guid) -{ - struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev); - - if (!mlx4_is_master(dev)) - return; - - priv->slave_node_guids[slave] = guid; -} -EXPORT_SYMBOL(mlx4_put_slave_node_guid); - -__be64 mlx4_get_slave_node_guid(struct mlx4_dev *dev, int slave) -{ - struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev); - - if (!mlx4_is_master(dev)) - return 0; - - return priv->slave_node_guids[slave]; -} -EXPORT_SYMBOL(mlx4_get_slave_node_guid); - -int mlx4_is_slave_active(struct mlx4_dev *dev, int slave) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_slave_state *s_slave; - - if (!mlx4_is_master(dev)) - return 0; - - s_slave = &priv->mfunc.master.slave_state[slave]; - return !!s_slave->active; -} -EXPORT_SYMBOL(mlx4_is_slave_active); - -static void slave_adjust_steering_mode(struct mlx4_dev *dev, - struct mlx4_dev_cap *dev_cap, - struct mlx4_init_hca_param *hca_param) -{ - dev->caps.steering_mode = hca_param->steering_mode; - if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) - dev->caps.num_qp_per_mgm = dev_cap->fs_max_num_qp_per_entry; - else - dev->caps.num_qp_per_mgm = - 4 * ((1 << hca_param->log_mc_entry_sz)/16 - 2); - - mlx4_dbg(dev, "Steering mode is: %s\n", - mlx4_steering_mode_str(dev->caps.steering_mode)); -} - -static int mlx4_slave_cap(struct mlx4_dev *dev) -{ - int err; - u32 page_size; - struct mlx4_dev_cap dev_cap; - struct mlx4_func_cap func_cap; - struct mlx4_init_hca_param hca_param; - int i; - - memset(&hca_param, 0, sizeof(hca_param)); - err = mlx4_QUERY_HCA(dev, &hca_param); - if (err) { - mlx4_err(dev, "QUERY_HCA command failed, aborting.\n"); - return err; - } - - /*fail if the hca has an unknown capability */ - if ((hca_param.global_caps | HCA_GLOBAL_CAP_MASK) != - HCA_GLOBAL_CAP_MASK) { - mlx4_err(dev, "Unknown hca global capabilities\n"); - return -ENOSYS; - } - - mlx4_log_num_mgm_entry_size = hca_param.log_mc_entry_sz; - - dev->caps.hca_core_clock = hca_param.hca_core_clock; - - memset(&dev_cap, 0, sizeof(dev_cap)); - dev->caps.max_qp_dest_rdma = 1 << hca_param.log_rd_per_qp; - err = mlx4_dev_cap(dev, &dev_cap); - if (err) { - mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n"); - return err; - } - - err = mlx4_QUERY_FW(dev); - if (err) - mlx4_err(dev, "QUERY_FW command failed: could not get FW version.\n"); - - if (!hca_param.mw_enable) { - dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_MEM_WINDOW; - dev->caps.bmme_flags &= ~MLX4_BMME_FLAG_TYPE_2_WIN; - } - - page_size = ~dev->caps.page_size_cap + 1; - mlx4_warn(dev, "HCA minimum page size:%d\n", page_size); - if (page_size > PAGE_SIZE) { - mlx4_err(dev, "HCA minimum page size of %d bigger than " - "kernel PAGE_SIZE of %d, aborting.\n", - page_size, (int)PAGE_SIZE); - return -ENODEV; - } - - /* slave gets uar page size from QUERY_HCA fw command */ - dev->caps.uar_page_size = 1 << (hca_param.uar_page_sz + 12); - - /* TODO: relax this assumption */ - if (dev->caps.uar_page_size != PAGE_SIZE) { - mlx4_err(dev, "UAR size:%d != kernel PAGE_SIZE of %d\n", - dev->caps.uar_page_size, (int)PAGE_SIZE); - return -ENODEV; - } - - memset(&func_cap, 0, sizeof(func_cap)); - err = mlx4_QUERY_FUNC_CAP(dev, 0, &func_cap); - if (err) { - mlx4_err(dev, "QUERY_FUNC_CAP general command failed, aborting (%d).\n", - err); - return err; - } - - if ((func_cap.pf_context_behaviour | PF_CONTEXT_BEHAVIOUR_MASK) != - PF_CONTEXT_BEHAVIOUR_MASK) { - mlx4_err(dev, "Unknown pf context behaviour\n"); - return -ENOSYS; - } - - dev->caps.num_ports = func_cap.num_ports; - dev->quotas.qp = func_cap.qp_quota; - dev->quotas.srq = func_cap.srq_quota; - dev->quotas.cq = func_cap.cq_quota; - dev->quotas.mpt = func_cap.mpt_quota; - dev->quotas.mtt = func_cap.mtt_quota; - dev->caps.num_qps = 1 << hca_param.log_num_qps; - dev->caps.num_srqs = 1 << hca_param.log_num_srqs; - dev->caps.num_cqs = 1 << hca_param.log_num_cqs; - dev->caps.num_mpts = 1 << hca_param.log_mpt_sz; - dev->caps.num_eqs = func_cap.max_eq; - dev->caps.reserved_eqs = func_cap.reserved_eq; - dev->caps.num_pds = MLX4_NUM_PDS; - dev->caps.num_mgms = 0; - dev->caps.num_amgms = 0; - - if (dev->caps.num_ports > MLX4_MAX_PORTS) { - mlx4_err(dev, "HCA has %d ports, but we only support %d, " - "aborting.\n", dev->caps.num_ports, MLX4_MAX_PORTS); - return -ENODEV; - } - - dev->caps.qp0_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); - dev->caps.qp0_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); - dev->caps.qp1_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); - dev->caps.qp1_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); - - if (!dev->caps.qp0_tunnel || !dev->caps.qp0_proxy || - !dev->caps.qp1_tunnel || !dev->caps.qp1_proxy) { - err = -ENOMEM; - goto err_mem; - } - - for (i = 1; i <= dev->caps.num_ports; ++i) { - err = mlx4_QUERY_FUNC_CAP(dev, (u32) i, &func_cap); - if (err) { - mlx4_err(dev, "QUERY_FUNC_CAP port command failed for" - " port %d, aborting (%d).\n", i, err); - goto err_mem; - } - dev->caps.qp0_tunnel[i - 1] = func_cap.qp0_tunnel_qpn; - dev->caps.qp0_proxy[i - 1] = func_cap.qp0_proxy_qpn; - dev->caps.qp1_tunnel[i - 1] = func_cap.qp1_tunnel_qpn; - dev->caps.qp1_proxy[i - 1] = func_cap.qp1_proxy_qpn; - dev->caps.def_counter_index[i - 1] = func_cap.def_counter_index; - - dev->caps.port_mask[i] = dev->caps.port_type[i]; - err = mlx4_get_slave_pkey_gid_tbl_len(dev, i, - &dev->caps.gid_table_len[i], - &dev->caps.pkey_table_len[i]); - if (err) - goto err_mem; - } - - if (dev->caps.uar_page_size * (dev->caps.num_uars - - dev->caps.reserved_uars) > - pci_resource_len(dev->pdev, 2)) { - mlx4_err(dev, "HCA reported UAR region size of 0x%x bigger than " - "PCI resource 2 size of 0x%llx, aborting.\n", - dev->caps.uar_page_size * dev->caps.num_uars, - (unsigned long long) pci_resource_len(dev->pdev, 2)); - err = -ENOMEM; - goto err_mem; - } - - if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_64B_EQE_ENABLED) { - dev->caps.eqe_size = 64; - dev->caps.eqe_factor = 1; - } else { - dev->caps.eqe_size = 32; - dev->caps.eqe_factor = 0; - } - - if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_64B_CQE_ENABLED) { - dev->caps.cqe_size = 64; - dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_64B_CQE; - } else { - dev->caps.cqe_size = 32; - } - - dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS; - mlx4_warn(dev, "Timestamping is not supported in slave mode.\n"); - - slave_adjust_steering_mode(dev, &dev_cap, &hca_param); - - if (func_cap.extra_flags & MLX4_QUERY_FUNC_FLAGS_BF_RES_QP && - dev->caps.bf_reg_size) - dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_ETH_BF_QP; - - return 0; - -err_mem: - kfree(dev->caps.qp0_tunnel); - kfree(dev->caps.qp0_proxy); - kfree(dev->caps.qp1_tunnel); - kfree(dev->caps.qp1_proxy); - dev->caps.qp0_tunnel = dev->caps.qp0_proxy = - dev->caps.qp1_tunnel = dev->caps.qp1_proxy = NULL; - - return err; -} - -static void mlx4_request_modules(struct mlx4_dev *dev) -{ - int port; - int has_ib_port = false; - int has_eth_port = false; -#define EN_DRV_NAME "mlx4_en" -#define IB_DRV_NAME "mlx4_ib" - - for (port = 1; port <= dev->caps.num_ports; port++) { - if (dev->caps.port_type[port] == MLX4_PORT_TYPE_IB) - has_ib_port = true; - else if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) - has_eth_port = true; - } - - if (has_ib_port) - request_module_nowait(IB_DRV_NAME); - if (has_eth_port) - request_module_nowait(EN_DRV_NAME); -} - -/* - * Change the port configuration of the device. - * Every user of this function must hold the port mutex. - */ -int mlx4_change_port_types(struct mlx4_dev *dev, - enum mlx4_port_type *port_types) -{ - int err = 0; - int change = 0; - int port; - - for (port = 0; port < dev->caps.num_ports; port++) { - /* Change the port type only if the new type is different - * from the current, and not set to Auto */ - if (port_types[port] != dev->caps.port_type[port + 1]) - change = 1; - } - if (change) { - mlx4_unregister_device(dev); - for (port = 1; port <= dev->caps.num_ports; port++) { - mlx4_CLOSE_PORT(dev, port); - dev->caps.port_type[port] = port_types[port - 1]; - err = mlx4_SET_PORT(dev, port, -1); - if (err) { - mlx4_err(dev, "Failed to set port %d, " - "aborting\n", port); - goto out; - } - } - mlx4_set_port_mask(dev); - err = mlx4_register_device(dev); - if (err) { - mlx4_err(dev, "Failed to register device\n"); - goto out; - } - mlx4_request_modules(dev); - } - -out: - return err; -} - -static ssize_t show_port_type(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info, - port_attr); - struct mlx4_dev *mdev = info->dev; - char type[8]; - - sprintf(type, "%s", - (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_IB) ? - "ib" : "eth"); - if (mdev->caps.possible_type[info->port] == MLX4_PORT_TYPE_AUTO) - sprintf(buf, "auto (%s)\n", type); - else - sprintf(buf, "%s\n", type); - - return strlen(buf); -} - -static ssize_t set_port_type(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info, - port_attr); - struct mlx4_dev *mdev = info->dev; - struct mlx4_priv *priv = mlx4_priv(mdev); - enum mlx4_port_type types[MLX4_MAX_PORTS]; - enum mlx4_port_type new_types[MLX4_MAX_PORTS]; - int i; - int err = 0; - - if (!strcmp(buf, "ib\n")) - info->tmp_type = MLX4_PORT_TYPE_IB; - else if (!strcmp(buf, "eth\n")) - info->tmp_type = MLX4_PORT_TYPE_ETH; - else if (!strcmp(buf, "auto\n")) - info->tmp_type = MLX4_PORT_TYPE_AUTO; - else { - mlx4_err(mdev, "%s is not supported port type\n", buf); - return -EINVAL; - } - - if ((info->tmp_type & mdev->caps.supported_type[info->port]) != - info->tmp_type) { - mlx4_err(mdev, "Requested port type for port %d is not supported on this HCA\n", - info->port); - return -EINVAL; - } - - mlx4_stop_sense(mdev); - mutex_lock(&priv->port_mutex); - /* Possible type is always the one that was delivered */ - mdev->caps.possible_type[info->port] = info->tmp_type; - - for (i = 0; i < mdev->caps.num_ports; i++) { - types[i] = priv->port[i+1].tmp_type ? priv->port[i+1].tmp_type : - mdev->caps.possible_type[i+1]; - if (types[i] == MLX4_PORT_TYPE_AUTO) - types[i] = mdev->caps.port_type[i+1]; - } - - if (!(mdev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP) && - !(mdev->caps.flags & MLX4_DEV_CAP_FLAG_SENSE_SUPPORT)) { - for (i = 1; i <= mdev->caps.num_ports; i++) { - if (mdev->caps.possible_type[i] == MLX4_PORT_TYPE_AUTO) { - mdev->caps.possible_type[i] = mdev->caps.port_type[i]; - err = -EINVAL; - } - } - } - if (err) { - mlx4_err(mdev, "Auto sensing is not supported on this HCA. " - "Set only 'eth' or 'ib' for both ports " - "(should be the same)\n"); - goto out; - } - - mlx4_do_sense_ports(mdev, new_types, types); - - err = mlx4_check_port_params(mdev, new_types); - if (err) - goto out; - - /* We are about to apply the changes after the configuration - * was verified, no need to remember the temporary types - * any more */ - for (i = 0; i < mdev->caps.num_ports; i++) - priv->port[i + 1].tmp_type = 0; - - err = mlx4_change_port_types(mdev, new_types); - -out: - mlx4_start_sense(mdev); - mutex_unlock(&priv->port_mutex); - return err ? err : count; -} - -enum ibta_mtu { - IB_MTU_256 = 1, - IB_MTU_512 = 2, - IB_MTU_1024 = 3, - IB_MTU_2048 = 4, - IB_MTU_4096 = 5 -}; - -static inline int int_to_ibta_mtu(int mtu) -{ - switch (mtu) { - case 256: return IB_MTU_256; - case 512: return IB_MTU_512; - case 1024: return IB_MTU_1024; - case 2048: return IB_MTU_2048; - case 4096: return IB_MTU_4096; - default: return -1; - } -} - -static inline int ibta_mtu_to_int(enum ibta_mtu mtu) -{ - switch (mtu) { - case IB_MTU_256: return 256; - case IB_MTU_512: return 512; - case IB_MTU_1024: return 1024; - case IB_MTU_2048: return 2048; - case IB_MTU_4096: return 4096; - default: return -1; - } -} - -static ssize_t -show_board(struct device *device, struct device_attribute *attr, - char *buf) -{ - struct mlx4_hca_info *info = container_of(attr, struct mlx4_hca_info, - board_attr); - struct mlx4_dev *mdev = info->dev; - - return sprintf(buf, "%.*s\n", MLX4_BOARD_ID_LEN, - mdev->board_id); -} - -static ssize_t -show_hca(struct device *device, struct device_attribute *attr, - char *buf) -{ - struct mlx4_hca_info *info = container_of(attr, struct mlx4_hca_info, - hca_attr); - struct mlx4_dev *mdev = info->dev; - - return sprintf(buf, "MT%d\n", mdev->pdev->device); -} - -static ssize_t -show_firmware_version(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct mlx4_hca_info *info = container_of(attr, struct mlx4_hca_info, - firmware_attr); - struct mlx4_dev *mdev = info->dev; - - return sprintf(buf, "%d.%d.%d\n", (int)(mdev->caps.fw_ver >> 32), - (int)(mdev->caps.fw_ver >> 16) & 0xffff, - (int)mdev->caps.fw_ver & 0xffff); -} - -static ssize_t show_port_ib_mtu(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info, - port_mtu_attr); - struct mlx4_dev *mdev = info->dev; - - /* When port type is eth, port mtu value isn't used. */ - if (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_ETH) - return -EINVAL; - - sprintf(buf, "%d\n", - ibta_mtu_to_int(mdev->caps.port_ib_mtu[info->port])); - return strlen(buf); -} - -static ssize_t set_port_ib_mtu(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info, - port_mtu_attr); - struct mlx4_dev *mdev = info->dev; - struct mlx4_priv *priv = mlx4_priv(mdev); - int err, port, mtu, ibta_mtu = -1; - - if (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_ETH) { - mlx4_warn(mdev, "port level mtu is only used for IB ports\n"); - return -EINVAL; - } - - mtu = (int) simple_strtol(buf, NULL, 0); - ibta_mtu = int_to_ibta_mtu(mtu); - - if (ibta_mtu < 0) { - mlx4_err(mdev, "%s is invalid IBTA mtu\n", buf); - return -EINVAL; - } - - mdev->caps.port_ib_mtu[info->port] = ibta_mtu; - - mlx4_stop_sense(mdev); - mutex_lock(&priv->port_mutex); - mlx4_unregister_device(mdev); - for (port = 1; port <= mdev->caps.num_ports; port++) { - mlx4_CLOSE_PORT(mdev, port); - err = mlx4_SET_PORT(mdev, port, -1); - if (err) { - mlx4_err(mdev, "Failed to set port %d, " - "aborting\n", port); - goto err_set_port; - } - } - err = mlx4_register_device(mdev); -err_set_port: - mutex_unlock(&priv->port_mutex); - mlx4_start_sense(mdev); - return err ? err : count; -} - -static int mlx4_load_fw(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int err, unmap_flag = 0; - - priv->fw.fw_icm = mlx4_alloc_icm(dev, priv->fw.fw_pages, - GFP_HIGHUSER | __GFP_NOWARN, 0); - if (!priv->fw.fw_icm) { - mlx4_err(dev, "Couldn't allocate FW area, aborting.\n"); - return -ENOMEM; - } - - err = mlx4_MAP_FA(dev, priv->fw.fw_icm); - if (err) { - mlx4_err(dev, "MAP_FA command failed, aborting.\n"); - goto err_free; - } - - err = mlx4_RUN_FW(dev); - if (err) { - mlx4_err(dev, "RUN_FW command failed, aborting.\n"); - goto err_unmap_fa; - } - - return 0; - -err_unmap_fa: - unmap_flag = mlx4_UNMAP_FA(dev); - if (unmap_flag) - pr_warn("mlx4_core: mlx4_UNMAP_FA failed.\n"); - -err_free: - if (!unmap_flag) - mlx4_free_icm(dev, priv->fw.fw_icm, 0); - return err; -} - -static int mlx4_init_cmpt_table(struct mlx4_dev *dev, u64 cmpt_base, - int cmpt_entry_sz) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int err; - int num_eqs; - - err = mlx4_init_icm_table(dev, &priv->qp_table.cmpt_table, - cmpt_base + - ((u64) (MLX4_CMPT_TYPE_QP * - cmpt_entry_sz) << MLX4_CMPT_SHIFT), - cmpt_entry_sz, dev->caps.num_qps, - dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], - 0, 0); - if (err) - goto err; - - err = mlx4_init_icm_table(dev, &priv->srq_table.cmpt_table, - cmpt_base + - ((u64) (MLX4_CMPT_TYPE_SRQ * - cmpt_entry_sz) << MLX4_CMPT_SHIFT), - cmpt_entry_sz, dev->caps.num_srqs, - dev->caps.reserved_srqs, 0, 0); - if (err) - goto err_qp; - - err = mlx4_init_icm_table(dev, &priv->cq_table.cmpt_table, - cmpt_base + - ((u64) (MLX4_CMPT_TYPE_CQ * - cmpt_entry_sz) << MLX4_CMPT_SHIFT), - cmpt_entry_sz, dev->caps.num_cqs, - dev->caps.reserved_cqs, 0, 0); - if (err) - goto err_srq; - - num_eqs = dev->phys_caps.num_phys_eqs; - err = mlx4_init_icm_table(dev, &priv->eq_table.cmpt_table, - cmpt_base + - ((u64) (MLX4_CMPT_TYPE_EQ * - cmpt_entry_sz) << MLX4_CMPT_SHIFT), - cmpt_entry_sz, num_eqs, num_eqs, 0, 0); - if (err) - goto err_cq; - - return 0; - -err_cq: - mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table); - -err_srq: - mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table); - -err_qp: - mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table); - -err: - return err; -} - -static int mlx4_init_icm(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap, - struct mlx4_init_hca_param *init_hca, u64 icm_size) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - u64 aux_pages; - int num_eqs; - int err, unmap_flag = 0; - - err = mlx4_SET_ICM_SIZE(dev, icm_size, &aux_pages); - if (err) { - mlx4_err(dev, "SET_ICM_SIZE command failed, aborting.\n"); - return err; - } - - mlx4_dbg(dev, "%lld KB of HCA context requires %lld KB aux memory.\n", - (unsigned long long) icm_size >> 10, - (unsigned long long) aux_pages << 2); - - priv->fw.aux_icm = mlx4_alloc_icm(dev, aux_pages, - GFP_HIGHUSER | __GFP_NOWARN, 0); - if (!priv->fw.aux_icm) { - mlx4_err(dev, "Couldn't allocate aux memory, aborting.\n"); - return -ENOMEM; - } - - err = mlx4_MAP_ICM_AUX(dev, priv->fw.aux_icm); - if (err) { - mlx4_err(dev, "MAP_ICM_AUX command failed, aborting.\n"); - goto err_free_aux; - } - - err = mlx4_init_cmpt_table(dev, init_hca->cmpt_base, dev_cap->cmpt_entry_sz); - if (err) { - mlx4_err(dev, "Failed to map cMPT context memory, aborting.\n"); - goto err_unmap_aux; - } - - - num_eqs = dev->phys_caps.num_phys_eqs; - err = mlx4_init_icm_table(dev, &priv->eq_table.table, - init_hca->eqc_base, dev_cap->eqc_entry_sz, - num_eqs, num_eqs, 0, 0); - if (err) { - mlx4_err(dev, "Failed to map EQ context memory, aborting.\n"); - goto err_unmap_cmpt; - } - - /* - * Reserved MTT entries must be aligned up to a cacheline - * boundary, since the FW will write to them, while the driver - * writes to all other MTT entries. (The variable - * dev->caps.mtt_entry_sz below is really the MTT segment - * size, not the raw entry size) - */ - dev->caps.reserved_mtts = - ALIGN(dev->caps.reserved_mtts * dev->caps.mtt_entry_sz, - dma_get_cache_alignment()) / dev->caps.mtt_entry_sz; - - err = mlx4_init_icm_table(dev, &priv->mr_table.mtt_table, - init_hca->mtt_base, - dev->caps.mtt_entry_sz, - dev->caps.num_mtts, - dev->caps.reserved_mtts, 1, 0); - if (err) { - mlx4_err(dev, "Failed to map MTT context memory, aborting.\n"); - goto err_unmap_eq; - } - - err = mlx4_init_icm_table(dev, &priv->mr_table.dmpt_table, - init_hca->dmpt_base, - dev_cap->dmpt_entry_sz, - dev->caps.num_mpts, - dev->caps.reserved_mrws, 1, 1); - if (err) { - mlx4_err(dev, "Failed to map dMPT context memory, aborting.\n"); - goto err_unmap_mtt; - } - - err = mlx4_init_icm_table(dev, &priv->qp_table.qp_table, - init_hca->qpc_base, - dev_cap->qpc_entry_sz, - dev->caps.num_qps, - dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], - 0, 0); - if (err) { - mlx4_err(dev, "Failed to map QP context memory, aborting.\n"); - goto err_unmap_dmpt; - } - - err = mlx4_init_icm_table(dev, &priv->qp_table.auxc_table, - init_hca->auxc_base, - dev_cap->aux_entry_sz, - dev->caps.num_qps, - dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], - 0, 0); - if (err) { - mlx4_err(dev, "Failed to map AUXC context memory, aborting.\n"); - goto err_unmap_qp; - } - - err = mlx4_init_icm_table(dev, &priv->qp_table.altc_table, - init_hca->altc_base, - dev_cap->altc_entry_sz, - dev->caps.num_qps, - dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], - 0, 0); - if (err) { - mlx4_err(dev, "Failed to map ALTC context memory, aborting.\n"); - goto err_unmap_auxc; - } - - err = mlx4_init_icm_table(dev, &priv->qp_table.rdmarc_table, - init_hca->rdmarc_base, - dev_cap->rdmarc_entry_sz << priv->qp_table.rdmarc_shift, - dev->caps.num_qps, - dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], - 0, 0); - if (err) { - mlx4_err(dev, "Failed to map RDMARC context memory, aborting\n"); - goto err_unmap_altc; - } - - err = mlx4_init_icm_table(dev, &priv->cq_table.table, - init_hca->cqc_base, - dev_cap->cqc_entry_sz, - dev->caps.num_cqs, - dev->caps.reserved_cqs, 0, 0); - if (err) { - mlx4_err(dev, "Failed to map CQ context memory, aborting.\n"); - goto err_unmap_rdmarc; - } - - err = mlx4_init_icm_table(dev, &priv->srq_table.table, - init_hca->srqc_base, - dev_cap->srq_entry_sz, - dev->caps.num_srqs, - dev->caps.reserved_srqs, 0, 0); - if (err) { - mlx4_err(dev, "Failed to map SRQ context memory, aborting.\n"); - goto err_unmap_cq; - } - - /* - * For flow steering device managed mode it is required to use - * mlx4_init_icm_table. For B0 steering mode it's not strictly - * required, but for simplicity just map the whole multicast - * group table now. The table isn't very big and it's a lot - * easier than trying to track ref counts. - */ - err = mlx4_init_icm_table(dev, &priv->mcg_table.table, - init_hca->mc_base, - mlx4_get_mgm_entry_size(dev), - dev->caps.num_mgms + dev->caps.num_amgms, - dev->caps.num_mgms + dev->caps.num_amgms, - 0, 0); - if (err) { - mlx4_err(dev, "Failed to map MCG context memory, aborting.\n"); - goto err_unmap_srq; - } - - return 0; - -err_unmap_srq: - mlx4_cleanup_icm_table(dev, &priv->srq_table.table); - -err_unmap_cq: - mlx4_cleanup_icm_table(dev, &priv->cq_table.table); - -err_unmap_rdmarc: - mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table); - -err_unmap_altc: - mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table); - -err_unmap_auxc: - mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table); - -err_unmap_qp: - mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table); - -err_unmap_dmpt: - mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table); - -err_unmap_mtt: - mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table); - -err_unmap_eq: - mlx4_cleanup_icm_table(dev, &priv->eq_table.table); - -err_unmap_cmpt: - mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table); - mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table); - mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table); - mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table); - -err_unmap_aux: - unmap_flag = mlx4_UNMAP_ICM_AUX(dev); - if (unmap_flag) - pr_warn("mlx4_core: mlx4_UNMAP_ICM_AUX failed.\n"); - -err_free_aux: - if (!unmap_flag) - mlx4_free_icm(dev, priv->fw.aux_icm, 0); - - return err; -} - -static void mlx4_free_icms(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - mlx4_cleanup_icm_table(dev, &priv->mcg_table.table); - mlx4_cleanup_icm_table(dev, &priv->srq_table.table); - mlx4_cleanup_icm_table(dev, &priv->cq_table.table); - mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table); - mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table); - mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table); - mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table); - mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table); - mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table); - mlx4_cleanup_icm_table(dev, &priv->eq_table.table); - mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table); - mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table); - mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table); - mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table); - - if (!mlx4_UNMAP_ICM_AUX(dev)) - mlx4_free_icm(dev, priv->fw.aux_icm, 0); - else - pr_warn("mlx4_core: mlx4_UNMAP_ICM_AUX failed.\n"); -} - -static void mlx4_slave_exit(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - mutex_lock(&priv->cmd.slave_cmd_mutex); - if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, MLX4_COMM_TIME)) - mlx4_warn(dev, "Failed to close slave function.\n"); - mutex_unlock(&priv->cmd.slave_cmd_mutex); -} - -static int map_bf_area(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - resource_size_t bf_start; - resource_size_t bf_len; - int err = 0; - - if (!dev->caps.bf_reg_size) - return -ENXIO; - - bf_start = pci_resource_start(dev->pdev, 2) + - (dev->caps.num_uars << PAGE_SHIFT); - bf_len = pci_resource_len(dev->pdev, 2) - - (dev->caps.num_uars << PAGE_SHIFT); - priv->bf_mapping = io_mapping_create_wc(bf_start, bf_len); - if (!priv->bf_mapping) - err = -ENOMEM; - - return err; -} - -static void unmap_bf_area(struct mlx4_dev *dev) -{ - if (mlx4_priv(dev)->bf_mapping) - io_mapping_free(mlx4_priv(dev)->bf_mapping); -} - -s64 mlx4_read_clock(struct mlx4_dev *dev) -{ - u32 clockhi, clocklo, clockhi1; - s64 cycles; - int i; - struct mlx4_priv *priv = mlx4_priv(dev); - - if (!priv->clock_mapping) - return -ENOTSUPP; - - for (i = 0; i < 10; i++) { - clockhi = swab32(readl(priv->clock_mapping)); - clocklo = swab32(readl(priv->clock_mapping + 4)); - clockhi1 = swab32(readl(priv->clock_mapping)); - if (clockhi == clockhi1) - break; - } - - cycles = (u64) clockhi << 32 | (u64) clocklo; - - return cycles & CORE_CLOCK_MASK; -} -EXPORT_SYMBOL_GPL(mlx4_read_clock); - - -static int map_internal_clock(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - priv->clock_mapping = ioremap(pci_resource_start(dev->pdev, - priv->fw.clock_bar) + - priv->fw.clock_offset, MLX4_CLOCK_SIZE); - - if (!priv->clock_mapping) - return -ENOMEM; - - return 0; -} - - -int mlx4_get_internal_clock_params(struct mlx4_dev *dev, - struct mlx4_clock_params *params) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - if (mlx4_is_slave(dev)) - return -ENOTSUPP; - if (!params) - return -EINVAL; - - params->bar = priv->fw.clock_bar; - params->offset = priv->fw.clock_offset; - params->size = MLX4_CLOCK_SIZE; - - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_get_internal_clock_params); - -static void unmap_internal_clock(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - if (priv->clock_mapping) - iounmap(priv->clock_mapping); -} - -static void mlx4_close_hca(struct mlx4_dev *dev) -{ - unmap_internal_clock(dev); - unmap_bf_area(dev); - if (mlx4_is_slave(dev)) { - mlx4_slave_exit(dev); - } else { - mlx4_CLOSE_HCA(dev, 0); - mlx4_free_icms(dev); - - if (!mlx4_UNMAP_FA(dev)) - mlx4_free_icm(dev, mlx4_priv(dev)->fw.fw_icm, 0); - else - pr_warn("mlx4_core: mlx4_UNMAP_FA failed.\n"); - } -} - -static int mlx4_init_slave(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - u64 dma = (u64) priv->mfunc.vhcr_dma; - int num_of_reset_retries = NUM_OF_RESET_RETRIES; - int ret_from_reset = 0; - u32 slave_read; - u32 cmd_channel_ver; - - mutex_lock(&priv->cmd.slave_cmd_mutex); - priv->cmd.max_cmds = 1; - mlx4_warn(dev, "Sending reset\n"); - ret_from_reset = mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, - MLX4_COMM_TIME); - /* if we are in the middle of flr the slave will try - * NUM_OF_RESET_RETRIES times before leaving.*/ - if (ret_from_reset) { - if (MLX4_DELAY_RESET_SLAVE == ret_from_reset) { - msleep(SLEEP_TIME_IN_RESET); - while (ret_from_reset && num_of_reset_retries) { - mlx4_warn(dev, "slave is currently in the" - "middle of FLR. retrying..." - "(try num:%d)\n", - (NUM_OF_RESET_RETRIES - - num_of_reset_retries + 1)); - ret_from_reset = - mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, - 0, MLX4_COMM_TIME); - num_of_reset_retries = num_of_reset_retries - 1; - } - } else - goto err; - } - - /* check the driver version - the slave I/F revision - * must match the master's */ - slave_read = swab32(readl(&priv->mfunc.comm->slave_read)); - cmd_channel_ver = mlx4_comm_get_version(); - - if (MLX4_COMM_GET_IF_REV(cmd_channel_ver) != - MLX4_COMM_GET_IF_REV(slave_read)) { - mlx4_err(dev, "slave driver version is not supported" - " by the master\n"); - goto err; - } - - mlx4_warn(dev, "Sending vhcr0\n"); - if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR0, dma >> 48, - MLX4_COMM_TIME)) - goto err; - if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR1, dma >> 32, - MLX4_COMM_TIME)) - goto err; - if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR2, dma >> 16, - MLX4_COMM_TIME)) - goto err; - if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR_EN, dma, MLX4_COMM_TIME)) - goto err; - - mutex_unlock(&priv->cmd.slave_cmd_mutex); - return 0; - -err: - mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, 0); - mutex_unlock(&priv->cmd.slave_cmd_mutex); - return -EIO; -} - -static void mlx4_parav_master_pf_caps(struct mlx4_dev *dev) -{ - int i; - - for (i = 1; i <= dev->caps.num_ports; i++) { - if (dev->caps.port_type[i] == MLX4_PORT_TYPE_ETH) - dev->caps.gid_table_len[i] = - mlx4_get_slave_num_gids(dev, 0); - else - dev->caps.gid_table_len[i] = 1; - dev->caps.pkey_table_len[i] = - dev->phys_caps.pkey_phys_table_len[i] - 1; - } -} - -static int choose_log_fs_mgm_entry_size(int qp_per_entry) -{ - int i = MLX4_MIN_MGM_LOG_ENTRY_SIZE; - - for (i = MLX4_MIN_MGM_LOG_ENTRY_SIZE; i <= MLX4_MAX_MGM_LOG_ENTRY_SIZE; - i++) { - if (qp_per_entry <= 4 * ((1 << i) / 16 - 2)) - break; - } - - return (i <= MLX4_MAX_MGM_LOG_ENTRY_SIZE) ? i : -1; -} - -static void choose_steering_mode(struct mlx4_dev *dev, - struct mlx4_dev_cap *dev_cap) -{ - int nvfs; - - mlx4_get_val(num_vfs.dbdf2val.tbl, pci_physfn(dev->pdev), 0, &nvfs); - if (high_rate_steer && !mlx4_is_mfunc(dev)) { - dev->caps.flags &= ~(MLX4_DEV_CAP_FLAG_VEP_MC_STEER | - MLX4_DEV_CAP_FLAG_VEP_UC_STEER); - dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_FS_EN; - } - - if (mlx4_log_num_mgm_entry_size == -1 && - dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_FS_EN && - (!mlx4_is_mfunc(dev) || - (dev_cap->fs_max_num_qp_per_entry >= (nvfs + 1))) && - choose_log_fs_mgm_entry_size(dev_cap->fs_max_num_qp_per_entry) >= - MLX4_MIN_MGM_LOG_ENTRY_SIZE) { - dev->oper_log_mgm_entry_size = - choose_log_fs_mgm_entry_size(dev_cap->fs_max_num_qp_per_entry); - dev->caps.steering_mode = MLX4_STEERING_MODE_DEVICE_MANAGED; - dev->caps.num_qp_per_mgm = dev_cap->fs_max_num_qp_per_entry; - } else { - if (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_UC_STEER && - dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) - dev->caps.steering_mode = MLX4_STEERING_MODE_B0; - else { - dev->caps.steering_mode = MLX4_STEERING_MODE_A0; - - if (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_UC_STEER || - dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) - mlx4_warn(dev, "Must have both UC_STEER and MC_STEER flags " - "set to use B0 steering. Falling back to A0 steering mode.\n"); - } - dev->oper_log_mgm_entry_size = - mlx4_log_num_mgm_entry_size > 0 ? - mlx4_log_num_mgm_entry_size : - MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE; - dev->caps.num_qp_per_mgm = mlx4_get_qp_per_mgm(dev); - } - mlx4_dbg(dev, "Steering mode is: %s, oper_log_mgm_entry_size = %d, " - "log_num_mgm_entry_size = %d\n", - mlx4_steering_mode_str(dev->caps.steering_mode), - dev->oper_log_mgm_entry_size, mlx4_log_num_mgm_entry_size); -} - -static int mlx4_init_hca(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_dev_cap *dev_cap = NULL; - struct mlx4_adapter adapter; - struct mlx4_mod_stat_cfg mlx4_cfg; - struct mlx4_profile profile; - struct mlx4_init_hca_param init_hca; - u64 icm_size; - int err; - - if (!mlx4_is_slave(dev)) { - err = mlx4_QUERY_FW(dev); - if (err) { - if (err == -EACCES) - mlx4_info(dev, "non-primary physical function, skipping.\n"); - else - mlx4_err(dev, "QUERY_FW command failed, aborting.\n"); - return err; - } - - err = mlx4_load_fw(dev); - if (err) { - mlx4_err(dev, "Failed to start FW, aborting.\n"); - return err; - } - - mlx4_cfg.log_pg_sz_m = 1; - mlx4_cfg.log_pg_sz = 0; - err = mlx4_MOD_STAT_CFG(dev, &mlx4_cfg); - if (err) - mlx4_warn(dev, "Failed to override log_pg_sz parameter\n"); - - dev_cap = kzalloc(sizeof *dev_cap, GFP_KERNEL); - if (!dev_cap) { - mlx4_err(dev, "Failed to allocate memory for dev_cap\n"); - err = -ENOMEM; - goto err_stop_fw; - } - - err = mlx4_dev_cap(dev, dev_cap); - if (err) { - mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n"); - goto err_stop_fw; - } - - choose_steering_mode(dev, dev_cap); - - if (mlx4_is_master(dev)) - mlx4_parav_master_pf_caps(dev); - - process_mod_param_profile(&profile); - if (dev->caps.steering_mode == - MLX4_STEERING_MODE_DEVICE_MANAGED) - profile.num_mcg = MLX4_FS_NUM_MCG; - - icm_size = mlx4_make_profile(dev, &profile, dev_cap, - &init_hca); - if ((long long) icm_size < 0) { - err = icm_size; - goto err_stop_fw; - } - - dev->caps.max_fmr_maps = (1 << (32 - ilog2(dev->caps.num_mpts))) - 1; - - init_hca.log_uar_sz = ilog2(dev->caps.num_uars); - init_hca.uar_page_sz = PAGE_SHIFT - 12; - - err = mlx4_init_icm(dev, dev_cap, &init_hca, icm_size); - if (err) - goto err_stop_fw; - - init_hca.mw_enable = 1; - - err = mlx4_INIT_HCA(dev, &init_hca); - if (err) { - mlx4_err(dev, "INIT_HCA command failed, aborting.\n"); - goto err_free_icm; - } - - if (dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) { - err = mlx4_query_func(dev, dev_cap); - if (err < 0) { - mlx4_err(dev, "QUERY_FUNC command failed, aborting.\n"); - goto err_stop_fw; - } else if (err & MLX4_QUERY_FUNC_NUM_SYS_EQS) { - dev->caps.num_eqs = dev_cap->max_eqs; - dev->caps.reserved_eqs = dev_cap->reserved_eqs; - dev->caps.reserved_uars = dev_cap->reserved_uars; - } - } - - /* - * Read HCA frequency by QUERY_HCA command - */ - if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) { - memset(&init_hca, 0, sizeof(init_hca)); - err = mlx4_QUERY_HCA(dev, &init_hca); - if (err) { - mlx4_err(dev, "QUERY_HCA command failed, disable timestamp.\n"); - dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS; - } else { - dev->caps.hca_core_clock = - init_hca.hca_core_clock; - } - - /* In case we got HCA frequency 0 - disable timestamping - * to avoid dividing by zero - */ - if (!dev->caps.hca_core_clock) { - dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS; - mlx4_err(dev, "HCA frequency is 0. Timestamping is not supported."); - } else if (map_internal_clock(dev)) { - /* Map internal clock, - * in case of failure disable timestamping - */ - dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS; - mlx4_err(dev, "Failed to map internal clock. Timestamping is not supported.\n"); - } - } - } else { - err = mlx4_init_slave(dev); - if (err) { - mlx4_err(dev, "Failed to initialize slave\n"); - return err; - } - - err = mlx4_slave_cap(dev); - if (err) { - mlx4_err(dev, "Failed to obtain slave caps\n"); - goto err_close; - } - } - - if (map_bf_area(dev)) - mlx4_dbg(dev, "Failed to map blue flame area\n"); - - /* Only the master set the ports, all the rest got it from it.*/ - if (!mlx4_is_slave(dev)) - mlx4_set_port_mask(dev); - - err = mlx4_QUERY_ADAPTER(dev, &adapter); - if (err) { - mlx4_err(dev, "QUERY_ADAPTER command failed, aborting.\n"); - goto unmap_bf; - } - - priv->eq_table.inta_pin = adapter.inta_pin; - memcpy(dev->board_id, adapter.board_id, sizeof dev->board_id); - memcpy(dev->vsd, adapter.vsd, sizeof(dev->vsd)); - dev->vsd_vendor_id = adapter.vsd_vendor_id; - - if (!mlx4_is_slave(dev)) - kfree(dev_cap); - - return 0; - -unmap_bf: - if (!mlx4_is_slave(dev)) - unmap_internal_clock(dev); - unmap_bf_area(dev); - - if (mlx4_is_slave(dev)) { - kfree(dev->caps.qp0_tunnel); - kfree(dev->caps.qp0_proxy); - kfree(dev->caps.qp1_tunnel); - kfree(dev->caps.qp1_proxy); - } - -err_close: - if (mlx4_is_slave(dev)) - mlx4_slave_exit(dev); - else - mlx4_CLOSE_HCA(dev, 0); - -err_free_icm: - if (!mlx4_is_slave(dev)) - mlx4_free_icms(dev); - -err_stop_fw: - if (!mlx4_is_slave(dev)) { - if (!mlx4_UNMAP_FA(dev)) - mlx4_free_icm(dev, priv->fw.fw_icm, 0); - else - pr_warn("mlx4_core: mlx4_UNMAP_FA failed.\n"); - kfree(dev_cap); - } - return err; -} - -static int mlx4_init_counters_table(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int nent_pow2, port_indx, vf_index, num_counters; - int res, index = 0; - struct counter_index *new_counter_index; - - - if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS)) - return -ENOENT; - - if (!mlx4_is_slave(dev) && - dev->caps.max_counters == dev->caps.max_extended_counters) { - res = mlx4_cmd(dev, MLX4_IF_STATE_EXTENDED, 0, 0, - MLX4_CMD_SET_IF_STAT, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); - if (res) { - mlx4_err(dev, "Failed to set extended counters (err=%d)\n", res); - return res; - } - } - - mutex_init(&priv->counters_table.mutex); - - if (mlx4_is_slave(dev)) { - for (port_indx = 0; port_indx < dev->caps.num_ports; port_indx++) { - INIT_LIST_HEAD(&priv->counters_table.global_port_list[port_indx]); - if (dev->caps.def_counter_index[port_indx] != 0xFF) { - new_counter_index = kmalloc(sizeof(struct counter_index), GFP_KERNEL); - if (!new_counter_index) - return -ENOMEM; - new_counter_index->index = dev->caps.def_counter_index[port_indx]; - list_add_tail(&new_counter_index->list, &priv->counters_table.global_port_list[port_indx]); - } - } - mlx4_dbg(dev, "%s: slave allocated %d counters for %d ports\n", - __func__, dev->caps.num_ports, dev->caps.num_ports); - return 0; - } - - nent_pow2 = roundup_pow_of_two(dev->caps.max_counters); - - for (port_indx = 0; port_indx < dev->caps.num_ports; port_indx++) { - INIT_LIST_HEAD(&priv->counters_table.global_port_list[port_indx]); - /* allocating 2 counters per port for PFs */ - /* For the PF, the ETH default counters are 0,2; */ - /* and the RoCE default counters are 1,3 */ - for (num_counters = 0; num_counters < 2; num_counters++, index++) { - new_counter_index = kmalloc(sizeof(struct counter_index), GFP_KERNEL); - if (!new_counter_index) - return -ENOMEM; - new_counter_index->index = index; - list_add_tail(&new_counter_index->list, - &priv->counters_table.global_port_list[port_indx]); - } - } - - if (mlx4_is_master(dev)) { - for (vf_index = 0; vf_index < dev->num_vfs; vf_index++) { - for (port_indx = 0; port_indx < dev->caps.num_ports; port_indx++) { - INIT_LIST_HEAD(&priv->counters_table.vf_list[vf_index][port_indx]); - new_counter_index = kmalloc(sizeof(struct counter_index), GFP_KERNEL); - if (!new_counter_index) - return -ENOMEM; - if (index < nent_pow2 - 2) { - new_counter_index->index = index; - index++; - } else { - new_counter_index->index = MLX4_SINK_COUNTER_INDEX; - } - - list_add_tail(&new_counter_index->list, - &priv->counters_table.vf_list[vf_index][port_indx]); - } - } - - res = mlx4_bitmap_init(&priv->counters_table.bitmap, - nent_pow2, nent_pow2 - 1, - index, 1); - mlx4_dbg(dev, "%s: master allocated %d counters for %d VFs\n", - __func__, index, dev->num_vfs); - } else { - res = mlx4_bitmap_init(&priv->counters_table.bitmap, - nent_pow2, nent_pow2 - 1, - index, 1); - mlx4_dbg(dev, "%s: native allocated %d counters for %d ports\n", - __func__, index, dev->caps.num_ports); - } - - return 0; - -} - -static void mlx4_cleanup_counters_table(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int i, j; - struct counter_index *port, *tmp_port; - struct counter_index *vf, *tmp_vf; - - mutex_lock(&priv->counters_table.mutex); - - if (dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS) { - for (i = 0; i < dev->caps.num_ports; i++) { - list_for_each_entry_safe(port, tmp_port, - &priv->counters_table.global_port_list[i], - list) { - list_del(&port->list); - kfree(port); - } - } - if (!mlx4_is_slave(dev)) { - for (i = 0; i < dev->num_vfs; i++) { - for (j = 0; j < dev->caps.num_ports; j++) { - list_for_each_entry_safe(vf, tmp_vf, - &priv->counters_table.vf_list[i][j], - list) { - /* clear the counter statistic */ - if (__mlx4_clear_if_stat(dev, vf->index)) - mlx4_dbg(dev, "%s: reset counter %d failed\n", - __func__, vf->index); - list_del(&vf->list); - kfree(vf); - } - } - } - mlx4_bitmap_cleanup(&priv->counters_table.bitmap); - } - } - mutex_unlock(&priv->counters_table.mutex); -} - -int __mlx4_slave_counters_free(struct mlx4_dev *dev, int slave) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int i, first; - struct counter_index *vf, *tmp_vf; - - /* clean VF's counters for the next useg */ - if (slave > 0 && slave <= dev->num_vfs) { - mlx4_dbg(dev, "%s: free counters of slave(%d)\n" - , __func__, slave); - - mutex_lock(&priv->counters_table.mutex); - for (i = 0; i < dev->caps.num_ports; i++) { - first = 0; - list_for_each_entry_safe(vf, tmp_vf, - &priv->counters_table.vf_list[slave - 1][i], - list) { - /* clear the counter statistic */ - if (__mlx4_clear_if_stat(dev, vf->index)) - mlx4_dbg(dev, "%s: reset counter %d failed\n", - __func__, vf->index); - if (first++ && vf->index != MLX4_SINK_COUNTER_INDEX) { - mlx4_dbg(dev, "%s: delete counter index %d for slave %d and port %d\n" - , __func__, vf->index, slave, i + 1); - mlx4_bitmap_free(&priv->counters_table.bitmap, vf->index, MLX4_USE_RR); - list_del(&vf->list); - kfree(vf); - } else { - mlx4_dbg(dev, "%s: can't delete default counter index %d for slave %d and port %d\n" - , __func__, vf->index, slave, i + 1); - } - } - } - mutex_unlock(&priv->counters_table.mutex); - } - - return 0; -} - -int __mlx4_counter_alloc(struct mlx4_dev *dev, int slave, int port, u32 *idx) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct counter_index *new_counter_index; - - if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS)) - return -ENOENT; - - if ((slave > MLX4_MAX_NUM_VF) || (slave < 0) || - (port < 0) || (port > MLX4_MAX_PORTS)) { - mlx4_dbg(dev, "%s: invalid slave(%d) or port(%d) index\n", - __func__, slave, port); - return -EINVAL; - } - - /* handle old guest request does not support request by port index */ - if (port == 0) { - *idx = MLX4_SINK_COUNTER_INDEX; - mlx4_dbg(dev, "%s: allocated default counter index %d for slave %d port %d\n" - , __func__, *idx, slave, port); - return 0; - } - - mutex_lock(&priv->counters_table.mutex); - - *idx = mlx4_bitmap_alloc(&priv->counters_table.bitmap); - /* if no resources return the default counter of the slave and port */ - if (*idx == -1) { - if (slave == 0) { /* its the ethernet counter ?????? */ - new_counter_index = list_entry(priv->counters_table.global_port_list[port - 1].next, - struct counter_index, - list); - } else { - new_counter_index = list_entry(priv->counters_table.vf_list[slave - 1][port - 1].next, - struct counter_index, - list); - } - - *idx = new_counter_index->index; - mlx4_dbg(dev, "%s: allocated defualt counter index %d for slave %d port %d\n" - , __func__, *idx, slave, port); - goto out; - } - - if (slave == 0) { /* native or master */ - new_counter_index = kmalloc(sizeof(struct counter_index), GFP_KERNEL); - if (!new_counter_index) - goto no_mem; - new_counter_index->index = *idx; - list_add_tail(&new_counter_index->list, &priv->counters_table.global_port_list[port - 1]); - } else { - new_counter_index = kmalloc(sizeof(struct counter_index), GFP_KERNEL); - if (!new_counter_index) - goto no_mem; - new_counter_index->index = *idx; - list_add_tail(&new_counter_index->list, &priv->counters_table.vf_list[slave - 1][port - 1]); - } - - mlx4_dbg(dev, "%s: allocated counter index %d for slave %d port %d\n" - , __func__, *idx, slave, port); -out: - mutex_unlock(&priv->counters_table.mutex); - return 0; - -no_mem: - mlx4_bitmap_free(&priv->counters_table.bitmap, *idx, MLX4_USE_RR); - mutex_unlock(&priv->counters_table.mutex); - *idx = MLX4_SINK_COUNTER_INDEX; - mlx4_dbg(dev, "%s: failed err (%d)\n" - , __func__, -ENOMEM); - return -ENOMEM; -} - -int mlx4_counter_alloc(struct mlx4_dev *dev, u8 port, u32 *idx) -{ - u64 out_param; - int err; - struct mlx4_priv *priv = mlx4_priv(dev); - struct counter_index *new_counter_index, *c_index; - - if (mlx4_is_mfunc(dev)) { - err = mlx4_cmd_imm(dev, 0, &out_param, - ((u32) port) << 8 | (u32) RES_COUNTER, - RES_OP_RESERVE, MLX4_CMD_ALLOC_RES, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); - if (!err) { - *idx = get_param_l(&out_param); - if (*idx == MLX4_SINK_COUNTER_INDEX) - return -ENOSPC; - - mutex_lock(&priv->counters_table.mutex); - c_index = list_entry(priv->counters_table.global_port_list[port - 1].next, - struct counter_index, - list); - mutex_unlock(&priv->counters_table.mutex); - if (c_index->index == *idx) - return -EEXIST; - - if (mlx4_is_slave(dev)) { - new_counter_index = kmalloc(sizeof(struct counter_index), GFP_KERNEL); - if (!new_counter_index) { - mlx4_counter_free(dev, port, *idx); - return -ENOMEM; - } - new_counter_index->index = *idx; - mutex_lock(&priv->counters_table.mutex); - list_add_tail(&new_counter_index->list, &priv->counters_table.global_port_list[port - 1]); - mutex_unlock(&priv->counters_table.mutex); - mlx4_dbg(dev, "%s: allocated counter index %d for port %d\n" - , __func__, *idx, port); - } - } - return err; - } - return __mlx4_counter_alloc(dev, 0, port, idx); -} -EXPORT_SYMBOL_GPL(mlx4_counter_alloc); - -void __mlx4_counter_free(struct mlx4_dev *dev, int slave, int port, u32 idx) -{ - /* check if native or slave and deletes accordingly */ - struct mlx4_priv *priv = mlx4_priv(dev); - struct counter_index *pf, *tmp_pf; - struct counter_index *vf, *tmp_vf; - int first; - - - if (idx == MLX4_SINK_COUNTER_INDEX) { - mlx4_dbg(dev, "%s: try to delete default counter index %d for port %d\n" - , __func__, idx, port); - return; - } - - if ((slave > MLX4_MAX_NUM_VF) || (slave < 0) || - (port < 0) || (port > MLX4_MAX_PORTS)) { - mlx4_warn(dev, "%s: deletion failed due to invalid slave(%d) or port(%d) index\n" - , __func__, slave, idx); - return; - } - - mutex_lock(&priv->counters_table.mutex); - if (slave == 0) { - first = 0; - list_for_each_entry_safe(pf, tmp_pf, - &priv->counters_table.global_port_list[port - 1], - list) { - /* the first 2 counters are reserved */ - if (pf->index == idx) { - /* clear the counter statistic */ - if (__mlx4_clear_if_stat(dev, pf->index)) - mlx4_dbg(dev, "%s: reset counter %d failed\n", - __func__, pf->index); - if (1 < first && idx != MLX4_SINK_COUNTER_INDEX) { - list_del(&pf->list); - kfree(pf); - mlx4_dbg(dev, "%s: delete counter index %d for native device (%d) port %d\n" - , __func__, idx, slave, port); - mlx4_bitmap_free(&priv->counters_table.bitmap, idx, MLX4_USE_RR); - goto out; - } else { - mlx4_dbg(dev, "%s: can't delete default counter index %d for native device (%d) port %d\n" - , __func__, idx, slave, port); - goto out; - } - } - first++; - } - mlx4_dbg(dev, "%s: can't delete counter index %d for native device (%d) port %d\n" - , __func__, idx, slave, port); - } else { - first = 0; - list_for_each_entry_safe(vf, tmp_vf, - &priv->counters_table.vf_list[slave - 1][port - 1], - list) { - /* the first element is reserved */ - if (vf->index == idx) { - /* clear the counter statistic */ - if (__mlx4_clear_if_stat(dev, vf->index)) - mlx4_dbg(dev, "%s: reset counter %d failed\n", - __func__, vf->index); - if (first) { - list_del(&vf->list); - kfree(vf); - mlx4_dbg(dev, "%s: delete counter index %d for slave %d port %d\n", - __func__, idx, slave, port); - mlx4_bitmap_free(&priv->counters_table.bitmap, idx, MLX4_USE_RR); - goto out; - } else { - mlx4_dbg(dev, "%s: can't delete default slave (%d) counter index %d for port %d\n" - , __func__, slave, idx, port); - goto out; - } - } - first++; - } - mlx4_dbg(dev, "%s: can't delete slave (%d) counter index %d for port %d\n" - , __func__, slave, idx, port); - } - -out: - mutex_unlock(&priv->counters_table.mutex); -} - -void mlx4_counter_free(struct mlx4_dev *dev, u8 port, u32 idx) -{ - u64 in_param = 0; - struct mlx4_priv *priv = mlx4_priv(dev); - struct counter_index *counter, *tmp_counter; - int first = 0; - - if (mlx4_is_mfunc(dev)) { - set_param_l(&in_param, idx); - mlx4_cmd(dev, in_param, - ((u32) port) << 8 | (u32) RES_COUNTER, - RES_OP_RESERVE, - MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_WRAPPED); - - if (mlx4_is_slave(dev) && idx != MLX4_SINK_COUNTER_INDEX) { - mutex_lock(&priv->counters_table.mutex); - list_for_each_entry_safe(counter, tmp_counter, - &priv->counters_table.global_port_list[port - 1], - list) { - if (counter->index == idx && first++) { - list_del(&counter->list); - kfree(counter); - mlx4_dbg(dev, "%s: delete counter index %d for port %d\n" - , __func__, idx, port); - mutex_unlock(&priv->counters_table.mutex); - return; - } - } - mutex_unlock(&priv->counters_table.mutex); - } - - return; - } - __mlx4_counter_free(dev, 0, port, idx); -} -EXPORT_SYMBOL_GPL(mlx4_counter_free); - -int __mlx4_clear_if_stat(struct mlx4_dev *dev, - u8 counter_index) -{ - struct mlx4_cmd_mailbox *if_stat_mailbox = NULL; - int err = 0; - u32 if_stat_in_mod = (counter_index & 0xff) | (1 << 31); - - if (counter_index == MLX4_SINK_COUNTER_INDEX) - return -EINVAL; - - if (mlx4_is_slave(dev)) - return 0; - - if_stat_mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(if_stat_mailbox)) { - err = PTR_ERR(if_stat_mailbox); - return err; - } - - err = mlx4_cmd_box(dev, 0, if_stat_mailbox->dma, if_stat_in_mod, 0, - MLX4_CMD_QUERY_IF_STAT, MLX4_CMD_TIME_CLASS_C, - MLX4_CMD_NATIVE); - - mlx4_free_cmd_mailbox(dev, if_stat_mailbox); - return err; -} - -u8 mlx4_get_default_counter_index(struct mlx4_dev *dev, int slave, int port) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct counter_index *new_counter_index; - - if (dev->caps.port_type[port] == MLX4_PORT_TYPE_IB) { - mlx4_dbg(dev, "%s: return counter index %d for slave %d port (MLX4_PORT_TYPE_IB) %d\n", - __func__, MLX4_SINK_COUNTER_INDEX, slave, port); - return (u8)MLX4_SINK_COUNTER_INDEX; - } - - mutex_lock(&priv->counters_table.mutex); - if (slave == 0) { - new_counter_index = list_entry(priv->counters_table.global_port_list[port - 1].next, - struct counter_index, - list); - } else { - new_counter_index = list_entry(priv->counters_table.vf_list[slave - 1][port - 1].next, - struct counter_index, - list); - } - mutex_unlock(&priv->counters_table.mutex); - - mlx4_dbg(dev, "%s: return counter index %d for slave %d port %d\n", - __func__, new_counter_index->index, slave, port); - - - return (u8)new_counter_index->index; -} - -int mlx4_get_vport_ethtool_stats(struct mlx4_dev *dev, int port, - struct mlx4_en_vport_stats *vport_stats, - int reset) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_cmd_mailbox *if_stat_mailbox = NULL; - union mlx4_counter *counter; - int err = 0; - u32 if_stat_in_mod; - struct counter_index *vport, *tmp_vport; - - if (!vport_stats) - return -EINVAL; - - if_stat_mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(if_stat_mailbox)) { - err = PTR_ERR(if_stat_mailbox); - return err; - } - - mutex_lock(&priv->counters_table.mutex); - list_for_each_entry_safe(vport, tmp_vport, - &priv->counters_table.global_port_list[port - 1], - list) { - if (vport->index == MLX4_SINK_COUNTER_INDEX) - continue; - - memset(if_stat_mailbox->buf, 0, sizeof(union mlx4_counter)); - if_stat_in_mod = (vport->index & 0xff) | ((reset & 1) << 31); - err = mlx4_cmd_box(dev, 0, if_stat_mailbox->dma, - if_stat_in_mod, 0, - MLX4_CMD_QUERY_IF_STAT, - MLX4_CMD_TIME_CLASS_C, - MLX4_CMD_NATIVE); - if (err) { - mlx4_dbg(dev, "%s: failed to read statistics for counter index %d\n", - __func__, vport->index); - goto if_stat_out; - } - counter = (union mlx4_counter *)if_stat_mailbox->buf; - if ((counter->control.cnt_mode & 0xf) == 1) { - vport_stats->rx_broadcast_packets += be64_to_cpu(counter->ext.counters[0].IfRxBroadcastFrames); - vport_stats->rx_unicast_packets += be64_to_cpu(counter->ext.counters[0].IfRxUnicastFrames); - vport_stats->rx_multicast_packets += be64_to_cpu(counter->ext.counters[0].IfRxMulticastFrames); - vport_stats->tx_broadcast_packets += be64_to_cpu(counter->ext.counters[0].IfTxBroadcastFrames); - vport_stats->tx_unicast_packets += be64_to_cpu(counter->ext.counters[0].IfTxUnicastFrames); - vport_stats->tx_multicast_packets += be64_to_cpu(counter->ext.counters[0].IfTxMulticastFrames); - vport_stats->rx_broadcast_bytes += be64_to_cpu(counter->ext.counters[0].IfRxBroadcastOctets); - vport_stats->rx_unicast_bytes += be64_to_cpu(counter->ext.counters[0].IfRxUnicastOctets); - vport_stats->rx_multicast_bytes += be64_to_cpu(counter->ext.counters[0].IfRxMulticastOctets); - vport_stats->tx_broadcast_bytes += be64_to_cpu(counter->ext.counters[0].IfTxBroadcastOctets); - vport_stats->tx_unicast_bytes += be64_to_cpu(counter->ext.counters[0].IfTxUnicastOctets); - vport_stats->tx_multicast_bytes += be64_to_cpu(counter->ext.counters[0].IfTxMulticastOctets); - vport_stats->rx_errors += be64_to_cpu(counter->ext.counters[0].IfRxErrorFrames); - vport_stats->rx_dropped += be64_to_cpu(counter->ext.counters[0].IfRxNoBufferFrames); - vport_stats->tx_errors += be64_to_cpu(counter->ext.counters[0].IfTxDroppedFrames); - } - } - -if_stat_out: - mutex_unlock(&priv->counters_table.mutex); - mlx4_free_cmd_mailbox(dev, if_stat_mailbox); - - return err; -} -EXPORT_SYMBOL_GPL(mlx4_get_vport_ethtool_stats); - -static int mlx4_setup_hca(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int err; - int port; - __be32 ib_port_default_caps; - - err = mlx4_init_uar_table(dev); - if (err) { - mlx4_err(dev, "Failed to initialize " - "user access region table (err=%d), aborting.\n", - err); - return err; - } - - err = mlx4_uar_alloc(dev, &priv->driver_uar); - if (err) { - mlx4_err(dev, "Failed to allocate driver access region " - "(err=%d), aborting.\n", err); - goto err_uar_table_free; - } - - priv->kar = ioremap((phys_addr_t) priv->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE); - if (!priv->kar) { - mlx4_err(dev, "Couldn't map kernel access region, " - "aborting.\n"); - err = -ENOMEM; - goto err_uar_free; - } - - err = mlx4_init_pd_table(dev); - if (err) { - mlx4_err(dev, "Failed to initialize " - "protection domain table (err=%d), aborting.\n", err); - goto err_kar_unmap; - } - - err = mlx4_init_xrcd_table(dev); - if (err) { - mlx4_err(dev, "Failed to initialize " - "reliable connection domain table (err=%d), " - "aborting.\n", err); - goto err_pd_table_free; - } - - err = mlx4_init_mr_table(dev); - if (err) { - mlx4_err(dev, "Failed to initialize " - "memory region table (err=%d), aborting.\n", err); - goto err_xrcd_table_free; - } - - if (!mlx4_is_slave(dev)) { - err = mlx4_init_mcg_table(dev); - if (err) { - mlx4_err(dev, "Failed to initialize " - "multicast group table (err=%d), aborting.\n", - err); - goto err_mr_table_free; - } - } - - err = mlx4_init_eq_table(dev); - if (err) { - mlx4_err(dev, "Failed to initialize " - "event queue table (err=%d), aborting.\n", err); - goto err_mcg_table_free; - } - - err = mlx4_cmd_use_events(dev); - if (err) { - mlx4_err(dev, "Failed to switch to event-driven " - "firmware commands (err=%d), aborting.\n", err); - goto err_eq_table_free; - } - - err = mlx4_NOP(dev); - if (err) { - if (dev->flags & MLX4_FLAG_MSI_X) { - mlx4_warn(dev, "NOP command failed to generate MSI-X " - "interrupt IRQ %d).\n", - priv->eq_table.eq[dev->caps.num_comp_vectors].irq); - mlx4_warn(dev, "Trying again without MSI-X.\n"); - } else { - mlx4_err(dev, "NOP command failed to generate interrupt " - "(IRQ %d), aborting.\n", - priv->eq_table.eq[dev->caps.num_comp_vectors].irq); - mlx4_err(dev, "BIOS or ACPI interrupt routing problem?\n"); - } - - goto err_cmd_poll; - } - - mlx4_dbg(dev, "NOP command IRQ test passed\n"); - - err = mlx4_init_cq_table(dev); - if (err) { - mlx4_err(dev, "Failed to initialize " - "completion queue table (err=%d), aborting.\n", err); - goto err_cmd_poll; - } - - err = mlx4_init_srq_table(dev); - if (err) { - mlx4_err(dev, "Failed to initialize " - "shared receive queue table (err=%d), aborting.\n", - err); - goto err_cq_table_free; - } - - err = mlx4_init_qp_table(dev); - if (err) { - mlx4_err(dev, "Failed to initialize " - "queue pair table (err=%d), aborting.\n", err); - goto err_srq_table_free; - } - - err = mlx4_init_counters_table(dev); - if (err && err != -ENOENT) { - mlx4_err(dev, "Failed to initialize counters table (err=%d), " - "aborting.\n", err); - goto err_qp_table_free; - } - - if (!mlx4_is_slave(dev)) { - for (port = 1; port <= dev->caps.num_ports; port++) { - ib_port_default_caps = 0; - err = mlx4_get_port_ib_caps(dev, port, - &ib_port_default_caps); - if (err) - mlx4_warn(dev, "failed to get port %d default " - "ib capabilities (%d). Continuing " - "with caps = 0\n", port, err); - dev->caps.ib_port_def_cap[port] = ib_port_default_caps; - - /* initialize per-slave default ib port capabilities */ - if (mlx4_is_master(dev)) { - int i; - for (i = 0; i < dev->num_slaves; i++) { - if (i == mlx4_master_func_num(dev)) - continue; - priv->mfunc.master.slave_state[i].ib_cap_mask[port] = - ib_port_default_caps; - } - } - - dev->caps.port_ib_mtu[port] = IB_MTU_4096; - - err = mlx4_SET_PORT(dev, port, mlx4_is_master(dev) ? - dev->caps.pkey_table_len[port] : -1); - if (err) { - mlx4_err(dev, "Failed to set port %d (err=%d), " - "aborting\n", port, err); - goto err_counters_table_free; - } - } - } - - return 0; - -err_counters_table_free: - mlx4_cleanup_counters_table(dev); - -err_qp_table_free: - mlx4_cleanup_qp_table(dev); - -err_srq_table_free: - mlx4_cleanup_srq_table(dev); - -err_cq_table_free: - mlx4_cleanup_cq_table(dev); - -err_cmd_poll: - mlx4_cmd_use_polling(dev); - -err_eq_table_free: - mlx4_cleanup_eq_table(dev); - -err_mcg_table_free: - if (!mlx4_is_slave(dev)) - mlx4_cleanup_mcg_table(dev); - -err_mr_table_free: - mlx4_cleanup_mr_table(dev); - -err_xrcd_table_free: - mlx4_cleanup_xrcd_table(dev); - -err_pd_table_free: - mlx4_cleanup_pd_table(dev); - -err_kar_unmap: - iounmap(priv->kar); - -err_uar_free: - mlx4_uar_free(dev, &priv->driver_uar); - -err_uar_table_free: - mlx4_cleanup_uar_table(dev); - return err; -} - -static void mlx4_enable_msi_x(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct msix_entry *entries; - int err; - int i; - - if (msi_x) { - int nreq = dev->caps.num_ports * num_online_cpus() + MSIX_LEGACY_SZ; - - nreq = min_t(int, dev->caps.num_eqs - dev->caps.reserved_eqs, - nreq); - - if (msi_x > 1 && !mlx4_is_mfunc(dev)) - nreq = min_t(int, nreq, msi_x); - - entries = kcalloc(nreq, sizeof *entries, GFP_KERNEL); - if (!entries) - goto no_msi; - - for (i = 0; i < nreq; ++i) - entries[i].entry = i; - - retry: - err = pci_enable_msix(dev->pdev, entries, nreq); - if (err) { - /* Try again if at least 2 vectors are available */ - if (err > 1) { - mlx4_info(dev, "Requested %d vectors, " - "but only %d MSI-X vectors available, " - "trying again\n", nreq, err); - nreq = err; - goto retry; - } - kfree(entries); - /* if error, or can't alloc even 1 IRQ */ - if (err < 0) { - mlx4_err(dev, "No IRQs left, device can't " - "be started.\n"); - goto no_irq; - } - goto no_msi; - } - - if (nreq < - MSIX_LEGACY_SZ + dev->caps.num_ports * MIN_MSIX_P_PORT) { - /*Working in legacy mode , all EQ's shared*/ - dev->caps.comp_pool = 0; - dev->caps.num_comp_vectors = nreq - 1; - } else { - dev->caps.comp_pool = nreq - MSIX_LEGACY_SZ; - dev->caps.num_comp_vectors = MSIX_LEGACY_SZ - 1; - } - for (i = 0; i < nreq; ++i) - priv->eq_table.eq[i].irq = entries[i].vector; - - dev->flags |= MLX4_FLAG_MSI_X; - - kfree(entries); - return; - } - -no_msi: - dev->caps.num_comp_vectors = 1; - dev->caps.comp_pool = 0; - - for (i = 0; i < 2; ++i) - priv->eq_table.eq[i].irq = dev->pdev->irq; - return; -no_irq: - dev->caps.num_comp_vectors = 0; - dev->caps.comp_pool = 0; - return; -} - -static void -mlx4_init_hca_info(struct mlx4_dev *dev) -{ - struct mlx4_hca_info *info = &mlx4_priv(dev)->hca_info; - - info->dev = dev; - - info->firmware_attr = (struct device_attribute)__ATTR(fw_ver, S_IRUGO, - show_firmware_version, NULL); - if (device_create_file(&dev->pdev->dev, &info->firmware_attr)) - mlx4_err(dev, "Failed to add file firmware version"); - - info->hca_attr = (struct device_attribute)__ATTR(hca, S_IRUGO, show_hca, - NULL); - if (device_create_file(&dev->pdev->dev, &info->hca_attr)) - mlx4_err(dev, "Failed to add file hca type"); - - info->board_attr = (struct device_attribute)__ATTR(board_id, S_IRUGO, - show_board, NULL); - if (device_create_file(&dev->pdev->dev, &info->board_attr)) - mlx4_err(dev, "Failed to add file board id type"); -} - -static int mlx4_init_port_info(struct mlx4_dev *dev, int port) -{ - struct mlx4_port_info *info = &mlx4_priv(dev)->port[port]; - int err = 0; - - info->dev = dev; - info->port = port; - if (!mlx4_is_slave(dev)) { - mlx4_init_mac_table(dev, &info->mac_table); - mlx4_init_vlan_table(dev, &info->vlan_table); - info->base_qpn = mlx4_get_base_qpn(dev, port); - } - - sprintf(info->dev_name, "mlx4_port%d", port); - info->port_attr.attr.name = info->dev_name; - if (mlx4_is_mfunc(dev)) - info->port_attr.attr.mode = S_IRUGO; - else { - info->port_attr.attr.mode = S_IRUGO | S_IWUSR; - info->port_attr.store = set_port_type; - } - info->port_attr.show = show_port_type; - sysfs_attr_init(&info->port_attr.attr); - - err = device_create_file(&dev->pdev->dev, &info->port_attr); - if (err) { - mlx4_err(dev, "Failed to create file for port %d\n", port); - info->port = -1; - } - - sprintf(info->dev_mtu_name, "mlx4_port%d_mtu", port); - info->port_mtu_attr.attr.name = info->dev_mtu_name; - if (mlx4_is_mfunc(dev)) - info->port_mtu_attr.attr.mode = S_IRUGO; - else { - info->port_mtu_attr.attr.mode = S_IRUGO | S_IWUSR; - info->port_mtu_attr.store = set_port_ib_mtu; - } - info->port_mtu_attr.show = show_port_ib_mtu; - sysfs_attr_init(&info->port_mtu_attr.attr); - - err = device_create_file(&dev->pdev->dev, &info->port_mtu_attr); - if (err) { - mlx4_err(dev, "Failed to create mtu file for port %d\n", port); - device_remove_file(&info->dev->pdev->dev, &info->port_attr); - info->port = -1; - } - - return err; -} - -static void -mlx4_cleanup_hca_info(struct mlx4_hca_info *info) -{ - device_remove_file(&info->dev->pdev->dev, &info->firmware_attr); - device_remove_file(&info->dev->pdev->dev, &info->board_attr); - device_remove_file(&info->dev->pdev->dev, &info->hca_attr); -} - -static void mlx4_cleanup_port_info(struct mlx4_port_info *info) -{ - if (info->port < 0) - return; - - device_remove_file(&info->dev->pdev->dev, &info->port_attr); - device_remove_file(&info->dev->pdev->dev, &info->port_mtu_attr); -} - -static int mlx4_init_steering(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int num_entries = dev->caps.num_ports; - int i, j; - - priv->steer = kzalloc(sizeof(struct mlx4_steer) * num_entries, GFP_KERNEL); - if (!priv->steer) - return -ENOMEM; - - for (i = 0; i < num_entries; i++) - for (j = 0; j < MLX4_NUM_STEERS; j++) { - INIT_LIST_HEAD(&priv->steer[i].promisc_qps[j]); - INIT_LIST_HEAD(&priv->steer[i].steer_entries[j]); - } - return 0; -} - -static void mlx4_clear_steering(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_steer_index *entry, *tmp_entry; - struct mlx4_promisc_qp *pqp, *tmp_pqp; - int num_entries = dev->caps.num_ports; - int i, j; - - for (i = 0; i < num_entries; i++) { - for (j = 0; j < MLX4_NUM_STEERS; j++) { - list_for_each_entry_safe(pqp, tmp_pqp, - &priv->steer[i].promisc_qps[j], - list) { - list_del(&pqp->list); - kfree(pqp); - } - list_for_each_entry_safe(entry, tmp_entry, - &priv->steer[i].steer_entries[j], - list) { - list_del(&entry->list); - list_for_each_entry_safe(pqp, tmp_pqp, - &entry->duplicates, - list) { - list_del(&pqp->list); - kfree(pqp); - } - kfree(entry); - } - } - } - kfree(priv->steer); -} - -static int extended_func_num(struct pci_dev *pdev) -{ - return PCI_SLOT(pdev->devfn) * 8 + PCI_FUNC(pdev->devfn); -} - -#define MLX4_OWNER_BASE 0x8069c -#define MLX4_OWNER_SIZE 4 - -static int mlx4_get_ownership(struct mlx4_dev *dev) -{ - void __iomem *owner; - u32 ret; - - if (pci_channel_offline(dev->pdev)) - return -EIO; - - owner = ioremap(pci_resource_start(dev->pdev, 0) + MLX4_OWNER_BASE, - MLX4_OWNER_SIZE); - if (!owner) { - mlx4_err(dev, "Failed to obtain ownership bit\n"); - return -ENOMEM; - } - - ret = readl(owner); - iounmap(owner); - return (int) !!ret; -} - -static void mlx4_free_ownership(struct mlx4_dev *dev) -{ - void __iomem *owner; - - if (pci_channel_offline(dev->pdev)) - return; - - owner = ioremap(pci_resource_start(dev->pdev, 0) + MLX4_OWNER_BASE, - MLX4_OWNER_SIZE); - if (!owner) { - mlx4_err(dev, "Failed to obtain ownership bit\n"); - return; - } - writel(0, owner); - msleep(1000); - iounmap(owner); -} - -static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data) -{ - struct mlx4_priv *priv; - struct mlx4_dev *dev; - int err; - int port; - int nvfs, prb_vf; - - pr_info(DRV_NAME ": Initializing %s\n", pci_name(pdev)); - - err = pci_enable_device(pdev); - if (err) { - dev_err(&pdev->dev, "Cannot enable PCI device, " - "aborting.\n"); - return err; - } - - mlx4_get_val(num_vfs.dbdf2val.tbl, pci_physfn(pdev), 0, &nvfs); - mlx4_get_val(probe_vf.dbdf2val.tbl, pci_physfn(pdev), 0, &prb_vf); - if (nvfs > MLX4_MAX_NUM_VF) { - dev_err(&pdev->dev, "There are more VF's (%d) than allowed(%d)\n", - nvfs, MLX4_MAX_NUM_VF); - return -EINVAL; - } - - if (nvfs < 0) { - dev_err(&pdev->dev, "num_vfs module parameter cannot be negative\n"); - return -EINVAL; - } - /* - * Check for BARs. - */ - if (!(pci_dev_data & MLX4_PCI_DEV_IS_VF) && - !(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { - dev_err(&pdev->dev, "Missing DCS, aborting." - "(driver_data: 0x%x, pci_resource_flags(pdev, 0):0x%x)\n", - pci_dev_data, pci_resource_flags(pdev, 0)); - err = -ENODEV; - goto err_disable_pdev; - } - if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) { - dev_err(&pdev->dev, "Missing UAR, aborting.\n"); - err = -ENODEV; - goto err_disable_pdev; - } - - err = pci_request_regions(pdev, DRV_NAME); - if (err) { - dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n"); - goto err_disable_pdev; - } - - pci_set_master(pdev); - - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); - if (err) { - dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask.\n"); - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); - if (err) { - dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n"); - goto err_release_regions; - } - } - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); - if (err) { - dev_warn(&pdev->dev, "Warning: couldn't set 64-bit " - "consistent PCI DMA mask.\n"); - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); - if (err) { - dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, " - "aborting.\n"); - goto err_release_regions; - } - } - - /* Allow large DMA segments, up to the firmware limit of 1 GB */ - dma_set_max_seg_size(&pdev->dev, 1024 * 1024 * 1024); - - priv = kzalloc(sizeof *priv, GFP_KERNEL); - if (!priv) { - dev_err(&pdev->dev, "Device struct alloc failed, " - "aborting.\n"); - err = -ENOMEM; - goto err_release_regions; - } - - dev = &priv->dev; - dev->pdev = pdev; - INIT_LIST_HEAD(&priv->dev_list); - INIT_LIST_HEAD(&priv->ctx_list); - spin_lock_init(&priv->ctx_lock); - - mutex_init(&priv->port_mutex); - - INIT_LIST_HEAD(&priv->pgdir_list); - mutex_init(&priv->pgdir_mutex); - - INIT_LIST_HEAD(&priv->bf_list); - mutex_init(&priv->bf_mutex); - - dev->rev_id = pdev->revision; - dev->numa_node = dev_to_node(&pdev->dev); - /* Detect if this device is a virtual function */ - if (pci_dev_data & MLX4_PCI_DEV_IS_VF) { - /* When acting as pf, we normally skip vfs unless explicitly - * requested to probe them. */ - if (nvfs && extended_func_num(pdev) > prb_vf) { - mlx4_warn(dev, "Skipping virtual function:%d\n", - extended_func_num(pdev)); - err = -ENODEV; - goto err_free_dev; - } - mlx4_warn(dev, "Detected virtual function - running in slave mode\n"); - dev->flags |= MLX4_FLAG_SLAVE; - } else { - /* We reset the device and enable SRIOV only for physical - * devices. Try to claim ownership on the device; - * if already taken, skip -- do not allow multiple PFs */ - err = mlx4_get_ownership(dev); - if (err) { - if (err < 0) - goto err_free_dev; - else { - mlx4_warn(dev, "Multiple PFs not yet supported." - " Skipping PF.\n"); - err = -EINVAL; - goto err_free_dev; - } - } - - if (nvfs) { - mlx4_warn(dev, "Enabling SR-IOV with %d VFs\n", nvfs); - err = pci_enable_sriov(pdev, nvfs); - if (err) { - mlx4_err(dev, "Failed to enable SR-IOV, continuing without SR-IOV (err = %d).\n", - err); - err = 0; - } else { - mlx4_warn(dev, "Running in master mode\n"); - dev->flags |= MLX4_FLAG_SRIOV | - MLX4_FLAG_MASTER; - dev->num_vfs = nvfs; - } - } - - atomic_set(&priv->opreq_count, 0); - INIT_WORK(&priv->opreq_task, mlx4_opreq_action); - - /* - * Now reset the HCA before we touch the PCI capabilities or - * attempt a firmware command, since a boot ROM may have left - * the HCA in an undefined state. - */ - err = mlx4_reset(dev); - if (err) { - mlx4_err(dev, "Failed to reset HCA, aborting.\n"); - goto err_sriov; - } - } - -slave_start: - err = mlx4_cmd_init(dev); - if (err) { - mlx4_err(dev, "Failed to init command interface, aborting.\n"); - goto err_sriov; - } - - /* In slave functions, the communication channel must be initialized - * before posting commands. Also, init num_slaves before calling - * mlx4_init_hca */ - if (mlx4_is_mfunc(dev)) { - if (mlx4_is_master(dev)) - dev->num_slaves = MLX4_MAX_NUM_SLAVES; - else { - dev->num_slaves = 0; - err = mlx4_multi_func_init(dev); - if (err) { - mlx4_err(dev, "Failed to init slave mfunc" - " interface, aborting.\n"); - goto err_cmd; - } - } - } - - err = mlx4_init_hca(dev); - if (err) { - if (err == -EACCES) { - /* Not primary Physical function - * Running in slave mode */ - mlx4_cmd_cleanup(dev); - dev->flags |= MLX4_FLAG_SLAVE; - dev->flags &= ~MLX4_FLAG_MASTER; - goto slave_start; - } else - goto err_mfunc; - } - - /* In master functions, the communication channel must be initialized - * after obtaining its address from fw */ - if (mlx4_is_master(dev)) { - err = mlx4_multi_func_init(dev); - if (err) { - mlx4_err(dev, "Failed to init master mfunc" - "interface, aborting.\n"); - goto err_close; - } - } - - err = mlx4_alloc_eq_table(dev); - if (err) - goto err_master_mfunc; - - priv->msix_ctl.pool_bm = 0; - mutex_init(&priv->msix_ctl.pool_lock); - - mlx4_enable_msi_x(dev); - - /* no MSIX and no shared IRQ */ - if (!dev->caps.num_comp_vectors && !dev->caps.comp_pool) { - err = -ENOSPC; - goto err_free_eq; - } - - if ((mlx4_is_mfunc(dev)) && - !(dev->flags & MLX4_FLAG_MSI_X)) { - err = -ENOSYS; - mlx4_err(dev, "INTx is not supported in multi-function mode." - " aborting.\n"); - goto err_free_eq; - } - - if (!mlx4_is_slave(dev)) { - err = mlx4_init_steering(dev); - if (err) - goto err_free_eq; - } - - mlx4_init_quotas(dev); - - err = mlx4_setup_hca(dev); - if (err == -EBUSY && (dev->flags & MLX4_FLAG_MSI_X) && - !mlx4_is_mfunc(dev)) { - dev->flags &= ~MLX4_FLAG_MSI_X; - dev->caps.num_comp_vectors = 1; - dev->caps.comp_pool = 0; - pci_disable_msix(pdev); - err = mlx4_setup_hca(dev); - } - - if (err) - goto err_steer; - - mlx4_init_hca_info(dev); - - for (port = 1; port <= dev->caps.num_ports; port++) { - err = mlx4_init_port_info(dev, port); - if (err) - goto err_port; - } - - err = mlx4_register_device(dev); - if (err) - goto err_port; - - mlx4_request_modules(dev); - - mlx4_sense_init(dev); - mlx4_start_sense(dev); - - priv->pci_dev_data = pci_dev_data; - pci_set_drvdata(pdev, dev); - - return 0; - -err_port: - for (--port; port >= 1; --port) - mlx4_cleanup_port_info(&priv->port[port]); - - mlx4_cleanup_counters_table(dev); - mlx4_cleanup_qp_table(dev); - mlx4_cleanup_srq_table(dev); - mlx4_cleanup_cq_table(dev); - mlx4_cmd_use_polling(dev); - mlx4_cleanup_eq_table(dev); - mlx4_cleanup_mcg_table(dev); - mlx4_cleanup_mr_table(dev); - mlx4_cleanup_xrcd_table(dev); - mlx4_cleanup_pd_table(dev); - mlx4_cleanup_uar_table(dev); - -err_steer: - if (!mlx4_is_slave(dev)) - mlx4_clear_steering(dev); - -err_free_eq: - mlx4_free_eq_table(dev); - -err_master_mfunc: - if (mlx4_is_master(dev)) { - mlx4_free_resource_tracker(dev, RES_TR_FREE_STRUCTS_ONLY); - mlx4_multi_func_cleanup(dev); - } - - if (mlx4_is_slave(dev)) { - kfree(dev->caps.qp0_tunnel); - kfree(dev->caps.qp0_proxy); - kfree(dev->caps.qp1_tunnel); - kfree(dev->caps.qp1_proxy); - } - -err_close: - if (dev->flags & MLX4_FLAG_MSI_X) - pci_disable_msix(pdev); - - mlx4_close_hca(dev); - -err_mfunc: - if (mlx4_is_slave(dev)) - mlx4_multi_func_cleanup(dev); - -err_cmd: - mlx4_cmd_cleanup(dev); - -err_sriov: - if (dev->flags & MLX4_FLAG_SRIOV) - pci_disable_sriov(pdev); - - if (!mlx4_is_slave(dev)) - mlx4_free_ownership(dev); - -err_free_dev: - kfree(priv); - -err_release_regions: - pci_release_regions(pdev); - -err_disable_pdev: - pci_disable_device(pdev); - pci_set_drvdata(pdev, NULL); - return err; -} - -static int __devinit mlx4_init_one(struct pci_dev *pdev, - const struct pci_device_id *id) -{ - device_set_desc(pdev->dev.bsddev, mlx4_version); - return __mlx4_init_one(pdev, id->driver_data); -} - -static void mlx4_remove_one(struct pci_dev *pdev) -{ - struct mlx4_dev *dev = pci_get_drvdata(pdev); - struct mlx4_priv *priv = mlx4_priv(dev); - int p; - - if (dev) { - /* in SRIOV it is not allowed to unload the pf's - * driver while there are alive vf's */ - if (mlx4_is_master(dev)) { - if (mlx4_how_many_lives_vf(dev)) - mlx4_err(dev, "Removing PF when there are assigned VF's !!!\n"); - } - mlx4_stop_sense(dev); - mlx4_unregister_device(dev); - - mlx4_cleanup_hca_info(&priv->hca_info); - for (p = 1; p <= dev->caps.num_ports; p++) { - mlx4_cleanup_port_info(&priv->port[p]); - mlx4_CLOSE_PORT(dev, p); - } - - if (mlx4_is_master(dev)) - mlx4_free_resource_tracker(dev, - RES_TR_FREE_SLAVES_ONLY); - - mlx4_cleanup_counters_table(dev); - mlx4_cleanup_qp_table(dev); - mlx4_cleanup_srq_table(dev); - mlx4_cleanup_cq_table(dev); - mlx4_cmd_use_polling(dev); - mlx4_cleanup_eq_table(dev); - mlx4_cleanup_mcg_table(dev); - mlx4_cleanup_mr_table(dev); - mlx4_cleanup_xrcd_table(dev); - mlx4_cleanup_pd_table(dev); - - if (mlx4_is_master(dev)) - mlx4_free_resource_tracker(dev, - RES_TR_FREE_STRUCTS_ONLY); - - iounmap(priv->kar); - mlx4_uar_free(dev, &priv->driver_uar); - mlx4_cleanup_uar_table(dev); - if (!mlx4_is_slave(dev)) - mlx4_clear_steering(dev); - mlx4_free_eq_table(dev); - if (mlx4_is_master(dev)) - mlx4_multi_func_cleanup(dev); - mlx4_close_hca(dev); - if (mlx4_is_slave(dev)) - mlx4_multi_func_cleanup(dev); - mlx4_cmd_cleanup(dev); - - if (dev->flags & MLX4_FLAG_MSI_X) - pci_disable_msix(pdev); - if (dev->flags & MLX4_FLAG_SRIOV) { - mlx4_warn(dev, "Disabling SR-IOV\n"); - pci_disable_sriov(pdev); - } - - if (!mlx4_is_slave(dev)) - mlx4_free_ownership(dev); - - kfree(dev->caps.qp0_tunnel); - kfree(dev->caps.qp0_proxy); - kfree(dev->caps.qp1_tunnel); - kfree(dev->caps.qp1_proxy); - - kfree(priv); - pci_release_regions(pdev); - pci_disable_device(pdev); - pci_set_drvdata(pdev, NULL); - } -} - -static int restore_current_port_types(struct mlx4_dev *dev, - enum mlx4_port_type *types, - enum mlx4_port_type *poss_types) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int err, i; - - mlx4_stop_sense(dev); - mutex_lock(&priv->port_mutex); - for (i = 0; i < dev->caps.num_ports; i++) - dev->caps.possible_type[i + 1] = poss_types[i]; - err = mlx4_change_port_types(dev, types); - mlx4_start_sense(dev); - mutex_unlock(&priv->port_mutex); - return err; -} - -int mlx4_restart_one(struct pci_dev *pdev) -{ - struct mlx4_dev *dev = pci_get_drvdata(pdev); - struct mlx4_priv *priv = mlx4_priv(dev); - enum mlx4_port_type curr_type[MLX4_MAX_PORTS]; - enum mlx4_port_type poss_type[MLX4_MAX_PORTS]; - int pci_dev_data, err, i; - - pci_dev_data = priv->pci_dev_data; - for (i = 0; i < dev->caps.num_ports; i++) { - curr_type[i] = dev->caps.port_type[i + 1]; - poss_type[i] = dev->caps.possible_type[i + 1]; - } - - mlx4_remove_one(pdev); - err = __mlx4_init_one(pdev, pci_dev_data); - if (err) - return err; - - dev = pci_get_drvdata(pdev); - err = restore_current_port_types(dev, curr_type, poss_type); - if (err) - mlx4_err(dev, "mlx4_restart_one: could not restore original port types (%d)\n", - err); - return 0; -} - -static DEFINE_PCI_DEVICE_TABLE(mlx4_pci_table) = { - /* MT25408 "Hermon" SDR */ - { PCI_VDEVICE(MELLANOX, 0x6340), - .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, - /* MT25408 "Hermon" DDR */ - { PCI_VDEVICE(MELLANOX, 0x634a), - .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, - /* MT25408 "Hermon" QDR */ - { PCI_VDEVICE(MELLANOX, 0x6354), - .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, - /* MT25408 "Hermon" DDR PCIe gen2 */ - { PCI_VDEVICE(MELLANOX, 0x6732), - .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, - /* MT25408 "Hermon" QDR PCIe gen2 */ - { PCI_VDEVICE(MELLANOX, 0x673c), - .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, - /* MT25408 "Hermon" EN 10GigE */ - { PCI_VDEVICE(MELLANOX, 0x6368), - .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, - /* MT25408 "Hermon" EN 10GigE PCIe gen2 */ - { PCI_VDEVICE(MELLANOX, 0x6750), - .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, - /* MT25458 ConnectX EN 10GBASE-T 10GigE */ - { PCI_VDEVICE(MELLANOX, 0x6372), - .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, - /* MT25458 ConnectX EN 10GBASE-T+Gen2 10GigE */ - { PCI_VDEVICE(MELLANOX, 0x675a), - .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, - /* MT26468 ConnectX EN 10GigE PCIe gen2*/ - { PCI_VDEVICE(MELLANOX, 0x6764), - .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, - /* MT26438 ConnectX EN 40GigE PCIe gen2 5GT/s */ - { PCI_VDEVICE(MELLANOX, 0x6746), - .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, - /* MT26478 ConnectX2 40GigE PCIe gen2 */ - { PCI_VDEVICE(MELLANOX, 0x676e), - .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }, - /* MT25400 Family [ConnectX-2 Virtual Function] */ - { PCI_VDEVICE(MELLANOX, 0x1002), - .driver_data = MLX4_PCI_DEV_IS_VF }, - /* MT27500 Family [ConnectX-3] */ - { PCI_VDEVICE(MELLANOX, 0x1003) }, - /* MT27500 Family [ConnectX-3 Virtual Function] */ - { PCI_VDEVICE(MELLANOX, 0x1004), - .driver_data = MLX4_PCI_DEV_IS_VF }, - { PCI_VDEVICE(MELLANOX, 0x1005) }, /* MT27510 Family */ - { PCI_VDEVICE(MELLANOX, 0x1006) }, /* MT27511 Family */ - { PCI_VDEVICE(MELLANOX, 0x1007) }, /* MT27520 Family */ - { PCI_VDEVICE(MELLANOX, 0x1008) }, /* MT27521 Family */ - { PCI_VDEVICE(MELLANOX, 0x1009) }, /* MT27530 Family */ - { PCI_VDEVICE(MELLANOX, 0x100a) }, /* MT27531 Family */ - { PCI_VDEVICE(MELLANOX, 0x100b) }, /* MT27540 Family */ - { PCI_VDEVICE(MELLANOX, 0x100c) }, /* MT27541 Family */ - { PCI_VDEVICE(MELLANOX, 0x100d) }, /* MT27550 Family */ - { PCI_VDEVICE(MELLANOX, 0x100e) }, /* MT27551 Family */ - { PCI_VDEVICE(MELLANOX, 0x100f) }, /* MT27560 Family */ - { PCI_VDEVICE(MELLANOX, 0x1010) }, /* MT27561 Family */ - { 0, } -}; - -MODULE_DEVICE_TABLE(pci, mlx4_pci_table); - -static pci_ers_result_t mlx4_pci_err_detected(struct pci_dev *pdev, - pci_channel_state_t state) -{ - mlx4_remove_one(pdev); - - return state == pci_channel_io_perm_failure ? - PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; -} - -static pci_ers_result_t mlx4_pci_slot_reset(struct pci_dev *pdev) -{ - int ret = __mlx4_init_one(pdev, 0); - - return ret ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; -} - -static const struct pci_error_handlers mlx4_err_handler = { - .error_detected = mlx4_pci_err_detected, - .slot_reset = mlx4_pci_slot_reset, -}; - -static int suspend(struct pci_dev *pdev, pm_message_t state) -{ - mlx4_remove_one(pdev); - - return 0; -} - -static int resume(struct pci_dev *pdev) -{ - return __mlx4_init_one(pdev, 0); -} - -static struct pci_driver mlx4_driver = { - .name = DRV_NAME, - .id_table = mlx4_pci_table, - .probe = mlx4_init_one, - .remove = __devexit_p(mlx4_remove_one), - .suspend = suspend, - .resume = resume, - .err_handler = &mlx4_err_handler, -}; - -static int __init mlx4_verify_params(void) -{ - int status; - - status = update_defaults(&port_type_array); - if (status == INVALID_STR) { - if (mlx4_fill_dbdf2val_tbl(&port_type_array.dbdf2val)) - return -1; - } else if (status == INVALID_DATA) { - return -1; - } - - status = update_defaults(&num_vfs); - if (status == INVALID_STR) { - if (mlx4_fill_dbdf2val_tbl(&num_vfs.dbdf2val)) - return -1; - } else if (status == INVALID_DATA) { - return -1; - } - - status = update_defaults(&probe_vf); - if (status == INVALID_STR) { - if (mlx4_fill_dbdf2val_tbl(&probe_vf.dbdf2val)) - return -1; - } else if (status == INVALID_DATA) { - return -1; - } - - if (msi_x < 0) { - pr_warn("mlx4_core: bad msi_x: %d\n", msi_x); - return -1; - } - - if ((log_num_mac < 0) || (log_num_mac > 7)) { - pr_warning("mlx4_core: bad num_mac: %d\n", log_num_mac); - return -1; - } - - if (log_num_vlan != 0) - pr_warning("mlx4_core: log_num_vlan - obsolete module param, using %d\n", - MLX4_LOG_NUM_VLANS); - - if (mlx4_set_4k_mtu != -1) - pr_warning("mlx4_core: set_4k_mtu - obsolete module param\n"); - - if ((log_mtts_per_seg < 0) || (log_mtts_per_seg > 7)) { - pr_warning("mlx4_core: bad log_mtts_per_seg: %d\n", log_mtts_per_seg); - return -1; - } - - if (mlx4_log_num_mgm_entry_size != -1 && - (mlx4_log_num_mgm_entry_size < MLX4_MIN_MGM_LOG_ENTRY_SIZE || - mlx4_log_num_mgm_entry_size > MLX4_MAX_MGM_LOG_ENTRY_SIZE)) { - pr_warning("mlx4_core: mlx4_log_num_mgm_entry_size (%d) not " - "in legal range (-1 or %d..%d)\n", - mlx4_log_num_mgm_entry_size, - MLX4_MIN_MGM_LOG_ENTRY_SIZE, - MLX4_MAX_MGM_LOG_ENTRY_SIZE); - return -1; - } - - if (mod_param_profile.num_qp < 18 || mod_param_profile.num_qp > 23) { - pr_warning("mlx4_core: bad log_num_qp: %d\n", - mod_param_profile.num_qp); - return -1; - } - - if (mod_param_profile.num_srq < 10) { - pr_warning("mlx4_core: too low log_num_srq: %d\n", - mod_param_profile.num_srq); - return -1; - } - - if (mod_param_profile.num_cq < 10) { - pr_warning("mlx4_core: too low log_num_cq: %d\n", - mod_param_profile.num_cq); - return -1; - } - - if (mod_param_profile.num_mpt < 10) { - pr_warning("mlx4_core: too low log_num_mpt: %d\n", - mod_param_profile.num_mpt); - return -1; - } - - if (mod_param_profile.num_mtt_segs && - mod_param_profile.num_mtt_segs < 15) { - pr_warning("mlx4_core: too low log_num_mtt: %d\n", - mod_param_profile.num_mtt_segs); - return -1; - } - - if (mod_param_profile.num_mtt_segs > MLX4_MAX_LOG_NUM_MTT) { - pr_warning("mlx4_core: too high log_num_mtt: %d\n", - mod_param_profile.num_mtt_segs); - return -1; - } - return 0; -} - -static int __init mlx4_init(void) -{ - int ret; - - if (mlx4_verify_params()) - return -EINVAL; - - mlx4_catas_init(); - - mlx4_wq = create_singlethread_workqueue("mlx4"); - if (!mlx4_wq) - return -ENOMEM; - - if (enable_sys_tune) - sys_tune_init(); - - ret = pci_register_driver(&mlx4_driver); - if (ret < 0) - goto err; - - return 0; - -err: - if (enable_sys_tune) - sys_tune_fini(); - - destroy_workqueue(mlx4_wq); - - return ret; -} - -static void __exit mlx4_cleanup(void) -{ - if (enable_sys_tune) - sys_tune_fini(); - - pci_unregister_driver(&mlx4_driver); - destroy_workqueue(mlx4_wq); -} - -module_init_order(mlx4_init, SI_ORDER_MIDDLE); -module_exit(mlx4_cleanup); - -static int -mlx4_evhand(module_t mod, int event, void *arg) -{ - return (0); -} - -static moduledata_t mlx4_mod = { - .name = "mlx4", - .evhand = mlx4_evhand, -}; -MODULE_VERSION(mlx4, 1); -DECLARE_MODULE(mlx4, mlx4_mod, SI_SUB_OFED_PREINIT, SI_ORDER_ANY); -MODULE_DEPEND(mlx4, linuxkpi, 1, 1, 1); - Property changes on: stable/11/sys/ofed/drivers/net/mlx4/main.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/intf.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/intf.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/intf.c (nonexistent) @@ -1,188 +0,0 @@ -/* - * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include - -#include "mlx4.h" - -struct mlx4_device_context { - struct list_head list; - struct mlx4_interface *intf; - void *context; -}; - -static LIST_HEAD(intf_list); -static LIST_HEAD(dev_list); -static DEFINE_MUTEX(intf_mutex); - -static void mlx4_add_device(struct mlx4_interface *intf, struct mlx4_priv *priv) -{ - struct mlx4_device_context *dev_ctx; - - dev_ctx = kmalloc(sizeof *dev_ctx, GFP_KERNEL); - if (!dev_ctx) - return; - - dev_ctx->intf = intf; - dev_ctx->context = intf->add(&priv->dev); - - if (dev_ctx->context) { - spin_lock_irq(&priv->ctx_lock); - list_add_tail(&dev_ctx->list, &priv->ctx_list); - spin_unlock_irq(&priv->ctx_lock); - } else - kfree(dev_ctx); -} - -static void mlx4_remove_device(struct mlx4_interface *intf, struct mlx4_priv *priv) -{ - struct mlx4_device_context *dev_ctx; - - list_for_each_entry(dev_ctx, &priv->ctx_list, list) - if (dev_ctx->intf == intf) { - spin_lock_irq(&priv->ctx_lock); - list_del(&dev_ctx->list); - spin_unlock_irq(&priv->ctx_lock); - - intf->remove(&priv->dev, dev_ctx->context); - kfree(dev_ctx); - return; - } -} - -int mlx4_register_interface(struct mlx4_interface *intf) -{ - struct mlx4_priv *priv; - - if (!intf->add || !intf->remove) - return -EINVAL; - - mutex_lock(&intf_mutex); - - list_add_tail(&intf->list, &intf_list); - list_for_each_entry(priv, &dev_list, dev_list) - mlx4_add_device(intf, priv); - - mutex_unlock(&intf_mutex); - - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_register_interface); - -void mlx4_unregister_interface(struct mlx4_interface *intf) -{ - struct mlx4_priv *priv; - - mutex_lock(&intf_mutex); - - list_for_each_entry(priv, &dev_list, dev_list) - mlx4_remove_device(intf, priv); - - list_del(&intf->list); - - mutex_unlock(&intf_mutex); -} -EXPORT_SYMBOL_GPL(mlx4_unregister_interface); - -void mlx4_dispatch_event(struct mlx4_dev *dev, enum mlx4_dev_event type, - unsigned long param) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_device_context *dev_ctx; - unsigned long flags; - - spin_lock_irqsave(&priv->ctx_lock, flags); - - list_for_each_entry(dev_ctx, &priv->ctx_list, list) - if (dev_ctx->intf->event) - dev_ctx->intf->event(dev, dev_ctx->context, type, param); - - spin_unlock_irqrestore(&priv->ctx_lock, flags); -} - -int mlx4_register_device(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_interface *intf; - - mutex_lock(&intf_mutex); - - list_add_tail(&priv->dev_list, &dev_list); - list_for_each_entry(intf, &intf_list, list) - mlx4_add_device(intf, priv); - - mutex_unlock(&intf_mutex); - if (!mlx4_is_slave(dev)) - mlx4_start_catas_poll(dev); - - return 0; -} - -void mlx4_unregister_device(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_interface *intf; - - if (!mlx4_is_slave(dev)) - mlx4_stop_catas_poll(dev); - mutex_lock(&intf_mutex); - - list_for_each_entry(intf, &intf_list, list) - mlx4_remove_device(intf, priv); - - list_del_init(&priv->dev_list); - - mutex_unlock(&intf_mutex); -} - -void *mlx4_get_protocol_dev(struct mlx4_dev *dev, enum mlx4_protocol proto, int port) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_device_context *dev_ctx; - unsigned long flags; - void *result = NULL; - - spin_lock_irqsave(&priv->ctx_lock, flags); - - list_for_each_entry(dev_ctx, &priv->ctx_list, list) - if (dev_ctx->intf->protocol == proto && dev_ctx->intf->get_dev) { - result = dev_ctx->intf->get_dev(dev, dev_ctx->context, port); - break; - } - - spin_unlock_irqrestore(&priv->ctx_lock, flags); - - return result; -} -EXPORT_SYMBOL_GPL(mlx4_get_protocol_dev); Property changes on: stable/11/sys/ofed/drivers/net/mlx4/intf.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/cq.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/cq.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/cq.c (nonexistent) @@ -1,400 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * Copyright (c) 2004 Voltaire, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include - -#include "mlx4.h" -#include "icm.h" - -#define MLX4_CQ_STATUS_OK ( 0 << 28) -#define MLX4_CQ_STATUS_OVERFLOW ( 9 << 28) -#define MLX4_CQ_STATUS_WRITE_FAIL (10 << 28) -#define MLX4_CQ_FLAG_CC ( 1 << 18) -#define MLX4_CQ_FLAG_OI ( 1 << 17) -#define MLX4_CQ_STATE_ARMED ( 9 << 8) -#define MLX4_CQ_STATE_ARMED_SOL ( 6 << 8) -#define MLX4_EQ_STATE_FIRED (10 << 8) - -void mlx4_cq_completion(struct mlx4_dev *dev, u32 cqn) -{ - struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table; - struct mlx4_cq *cq; - - read_lock(&cq_table->cq_table_lock); - - cq = radix_tree_lookup(&mlx4_priv(dev)->cq_table.tree, - cqn & (dev->caps.num_cqs - 1)); - if (cq) - atomic_inc(&cq->refcount); - - read_unlock(&cq_table->cq_table_lock); - - if (!cq) { - mlx4_dbg(dev, "Completion event for bogus CQ %08x\n", cqn); - return; - } - - ++cq->arm_sn; - - cq->comp(cq); - - if (atomic_dec_and_test(&cq->refcount)) - complete(&cq->free); -} - -void mlx4_cq_event(struct mlx4_dev *dev, u32 cqn, int event_type) -{ - struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table; - struct mlx4_cq *cq; - - read_lock(&cq_table->cq_table_lock); - - cq = radix_tree_lookup(&cq_table->tree, cqn & (dev->caps.num_cqs - 1)); - if (cq) - atomic_inc(&cq->refcount); - - read_unlock(&cq_table->cq_table_lock); - - if (!cq) { - mlx4_warn(dev, "Async event for bogus CQ %08x\n", cqn); - return; - } - - cq->event(cq, event_type); - - if (atomic_dec_and_test(&cq->refcount)) - complete(&cq->free); -} - -static int mlx4_SW2HW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, - int cq_num) -{ - return mlx4_cmd(dev, mailbox->dma, cq_num, 0, - MLX4_CMD_SW2HW_CQ, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_WRAPPED); -} - -static int mlx4_MODIFY_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, - int cq_num, u32 opmod) -{ - return mlx4_cmd(dev, mailbox->dma, cq_num, opmod, MLX4_CMD_MODIFY_CQ, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); -} - -static int mlx4_HW2SW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, - int cq_num) -{ - return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, - cq_num, mailbox ? 0 : 1, MLX4_CMD_HW2SW_CQ, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); -} - -int mlx4_cq_modify(struct mlx4_dev *dev, struct mlx4_cq *cq, - u16 count, u16 period) -{ - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_cq_context *cq_context; - int err; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - cq_context = mailbox->buf; - memset(cq_context, 0, sizeof *cq_context); - - cq_context->cq_max_count = cpu_to_be16(count); - cq_context->cq_period = cpu_to_be16(period); - - err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 1); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} -EXPORT_SYMBOL_GPL(mlx4_cq_modify); - -int mlx4_cq_resize(struct mlx4_dev *dev, struct mlx4_cq *cq, - int entries, struct mlx4_mtt *mtt) -{ - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_cq_context *cq_context; - u64 mtt_addr; - int err; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - cq_context = mailbox->buf; - memset(cq_context, 0, sizeof *cq_context); - - cq_context->logsize_usrpage = cpu_to_be32(ilog2(entries) << 24); - cq_context->log_page_size = mtt->page_shift - 12; - mtt_addr = mlx4_mtt_addr(dev, mtt); - cq_context->mtt_base_addr_h = mtt_addr >> 32; - cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); - - err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 0); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} -EXPORT_SYMBOL_GPL(mlx4_cq_resize); - -int mlx4_cq_ignore_overrun(struct mlx4_dev *dev, struct mlx4_cq *cq) -{ - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_cq_context *cq_context; - int err; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - cq_context = mailbox->buf; - memset(cq_context, 0, sizeof *cq_context); - - cq_context->flags |= cpu_to_be32(MLX4_CQ_FLAG_OI); - - err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 3); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} -EXPORT_SYMBOL_GPL(mlx4_cq_ignore_overrun); - -int __mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_cq_table *cq_table = &priv->cq_table; - int err; - - *cqn = mlx4_bitmap_alloc(&cq_table->bitmap); - if (*cqn == -1) - return -ENOMEM; - - err = mlx4_table_get(dev, &cq_table->table, *cqn); - if (err) - goto err_out; - - err = mlx4_table_get(dev, &cq_table->cmpt_table, *cqn); - if (err) - goto err_put; - return 0; - -err_put: - mlx4_table_put(dev, &cq_table->table, *cqn); - -err_out: - mlx4_bitmap_free(&cq_table->bitmap, *cqn, MLX4_NO_RR); - return err; -} - -static int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn) -{ - u64 out_param; - int err; - - if (mlx4_is_mfunc(dev)) { - err = mlx4_cmd_imm(dev, 0, &out_param, RES_CQ, - RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); - if (err) - return err; - else { - *cqn = get_param_l(&out_param); - return 0; - } - } - return __mlx4_cq_alloc_icm(dev, cqn); -} - -void __mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_cq_table *cq_table = &priv->cq_table; - - mlx4_table_put(dev, &cq_table->cmpt_table, cqn); - mlx4_table_put(dev, &cq_table->table, cqn); - mlx4_bitmap_free(&cq_table->bitmap, cqn, MLX4_NO_RR); -} - -static void mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn) -{ - u64 in_param = 0; - int err; - - if (mlx4_is_mfunc(dev)) { - set_param_l(&in_param, cqn); - err = mlx4_cmd(dev, in_param, RES_CQ, RES_OP_RESERVE_AND_MAP, - MLX4_CMD_FREE_RES, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); - if (err) - mlx4_warn(dev, "Failed freeing cq:%d\n", cqn); - } else - __mlx4_cq_free_icm(dev, cqn); -} - -int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, - struct mlx4_mtt *mtt, struct mlx4_uar *uar, u64 db_rec, - struct mlx4_cq *cq, unsigned vector, int collapsed, - int timestamp_en) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_cq_table *cq_table = &priv->cq_table; - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_cq_context *cq_context; - u64 mtt_addr; - int err; - - if (vector > dev->caps.num_comp_vectors + dev->caps.comp_pool) - return -EINVAL; - - cq->vector = vector; - - err = mlx4_cq_alloc_icm(dev, &cq->cqn); - if (err) - return err; - - spin_lock_irq(&cq_table->lock); - err = radix_tree_insert(&cq_table->tree, cq->cqn, cq); - spin_unlock_irq(&cq_table->lock); - if (err) - goto err_icm; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) { - err = PTR_ERR(mailbox); - goto err_radix; - } - - cq_context = mailbox->buf; - memset(cq_context, 0, sizeof *cq_context); - - cq_context->flags = cpu_to_be32(!!collapsed << 18); - if (timestamp_en) - cq_context->flags |= cpu_to_be32(1 << 19); - - cq_context->logsize_usrpage = cpu_to_be32((ilog2(nent) << 24) | uar->index); - cq_context->comp_eqn = priv->eq_table.eq[vector].eqn; - cq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT; - - mtt_addr = mlx4_mtt_addr(dev, mtt); - cq_context->mtt_base_addr_h = mtt_addr >> 32; - cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); - cq_context->db_rec_addr = cpu_to_be64(db_rec); - - err = mlx4_SW2HW_CQ(dev, mailbox, cq->cqn); - mlx4_free_cmd_mailbox(dev, mailbox); - if (err) - goto err_radix; - - cq->cons_index = 0; - cq->arm_sn = 1; - cq->uar = uar; - atomic_set(&cq->refcount, 1); - init_completion(&cq->free); - - cq->eqn = priv->eq_table.eq[cq->vector].eqn; - cq->irq = priv->eq_table.eq[cq->vector].irq; - - return 0; - -err_radix: - spin_lock_irq(&cq_table->lock); - radix_tree_delete(&cq_table->tree, cq->cqn); - spin_unlock_irq(&cq_table->lock); - -err_icm: - mlx4_cq_free_icm(dev, cq->cqn); - - return err; -} -EXPORT_SYMBOL_GPL(mlx4_cq_alloc); - -void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_cq_table *cq_table = &priv->cq_table; - int err; - - err = mlx4_HW2SW_CQ(dev, NULL, cq->cqn); - if (err) - mlx4_warn(dev, "HW2SW_CQ failed (%d) for CQN %06x\n", err, cq->cqn); - - synchronize_irq(priv->eq_table.eq[cq->vector].irq); - - spin_lock_irq(&cq_table->lock); - radix_tree_delete(&cq_table->tree, cq->cqn); - spin_unlock_irq(&cq_table->lock); - - if (atomic_dec_and_test(&cq->refcount)) - complete(&cq->free); - wait_for_completion(&cq->free); - - mlx4_cq_free_icm(dev, cq->cqn); -} -EXPORT_SYMBOL_GPL(mlx4_cq_free); - -int mlx4_init_cq_table(struct mlx4_dev *dev) -{ - struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table; - int err; - - spin_lock_init(&cq_table->lock); - rwlock_init(&cq_table->cq_table_lock); - INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC); - if (mlx4_is_slave(dev)) - return 0; - - err = mlx4_bitmap_init(&cq_table->bitmap, dev->caps.num_cqs, - dev->caps.num_cqs - 1, dev->caps.reserved_cqs, 0); - if (err) - return err; - - return 0; -} - -void mlx4_cleanup_cq_table(struct mlx4_dev *dev) -{ - if (mlx4_is_slave(dev)) - return; - /* Nothing to do to clean up radix_tree */ - mlx4_bitmap_cleanup(&mlx4_priv(dev)->cq_table.bitmap); -} Property changes on: stable/11/sys/ofed/drivers/net/mlx4/cq.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/port.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/port.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/port.c (nonexistent) @@ -1,1192 +0,0 @@ -/* - * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#define LINUXKPI_PARAM_PREFIX mlx4_ - -#include -#include -#include -#include - -#include -#include -#include "mlx4.h" -#include "mlx4_stats.h" - - -int mlx4_set_4k_mtu = -1; -module_param_named(set_4k_mtu, mlx4_set_4k_mtu, int, 0444); -MODULE_PARM_DESC(set_4k_mtu, - "(Obsolete) attempt to set 4K MTU to all ConnectX ports"); - - -#define MLX4_MAC_VALID (1ull << 63) - -#define MLX4_VLAN_VALID (1u << 31) -#define MLX4_VLAN_MASK 0xfff - -void mlx4_init_mac_table(struct mlx4_dev *dev, struct mlx4_mac_table *table) -{ - int i; - - mutex_init(&table->mutex); - for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { - table->entries[i] = 0; - table->refs[i] = 0; - } - table->max = 1 << dev->caps.log_num_macs; - table->total = 0; -} - -void mlx4_init_vlan_table(struct mlx4_dev *dev, struct mlx4_vlan_table *table) -{ - int i; - - mutex_init(&table->mutex); - for (i = 0; i < MLX4_MAX_VLAN_NUM; i++) { - table->entries[i] = 0; - table->refs[i] = 0; - } - table->max = (1 << dev->caps.log_num_vlans) - MLX4_VLAN_REGULAR; - table->total = 0; -} - -static int validate_index(struct mlx4_dev *dev, - struct mlx4_mac_table *table, int index) -{ - int err = 0; - - if (index < 0 || index >= table->max || !table->refs[index]) { - mlx4_warn(dev, "No valid Mac entry for the given index\n"); - err = -EINVAL; - } - return err; -} - -static int find_index(struct mlx4_dev *dev, - struct mlx4_mac_table *table, u64 mac) -{ - int i; - - for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { - if ((mac & MLX4_MAC_MASK) == - (MLX4_MAC_MASK & be64_to_cpu(table->entries[i]))) - return i; - } - /* Mac not found */ - return -EINVAL; -} - -static int mlx4_set_port_mac_table(struct mlx4_dev *dev, u8 port, - __be64 *entries) -{ - struct mlx4_cmd_mailbox *mailbox; - u32 in_mod; - int err; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - memcpy(mailbox->buf, entries, MLX4_MAC_TABLE_SIZE); - - in_mod = MLX4_SET_PORT_MAC_TABLE << 8 | port; - - err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - -int __mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac) -{ - struct mlx4_port_info *info = &mlx4_priv(dev)->port[port]; - struct mlx4_mac_table *table = &info->mac_table; - int i, err = 0; - int free = -1; - - mlx4_dbg(dev, "Registering MAC: 0x%llx for port %d\n", - (unsigned long long) mac, port); - - mutex_lock(&table->mutex); - for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { - if (free < 0 && !table->refs[i]) { - free = i; - continue; - } - - if ((mac == (MLX4_MAC_MASK & be64_to_cpu(table->entries[i]))) && - table->refs[i]) { - /* MAC already registered, Must not have duplicates */ - err = i; - ++table->refs[i]; - goto out; - } - } - - mlx4_dbg(dev, "Free MAC index is %d\n", free); - - if (table->total == table->max) { - /* No free mac entries */ - err = -ENOSPC; - goto out; - } - - /* Register new MAC */ - table->entries[free] = cpu_to_be64(mac | MLX4_MAC_VALID); - - err = mlx4_set_port_mac_table(dev, port, table->entries); - if (unlikely(err)) { - mlx4_err(dev, "Failed adding MAC: 0x%llx\n", - (unsigned long long) mac); - table->entries[free] = 0; - goto out; - } - table->refs[free] = 1; - - err = free; - ++table->total; -out: - mutex_unlock(&table->mutex); - return err; -} -EXPORT_SYMBOL_GPL(__mlx4_register_mac); - -int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac) -{ - u64 out_param = 0; - int err = -EINVAL; - - if (mlx4_is_mfunc(dev)) { - if (!(dev->flags & MLX4_FLAG_OLD_REG_MAC)) { - err = mlx4_cmd_imm(dev, mac, &out_param, - ((u32) port) << 8 | (u32) RES_MAC, - RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); - } - if (err && err == -EINVAL && mlx4_is_slave(dev)) { - /* retry using old REG_MAC format */ - set_param_l(&out_param, port); - err = mlx4_cmd_imm(dev, mac, &out_param, RES_MAC, - RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); - if (!err) - dev->flags |= MLX4_FLAG_OLD_REG_MAC; - } - if (err) - return err; - - return get_param_l(&out_param); - } - return __mlx4_register_mac(dev, port, mac); -} -EXPORT_SYMBOL_GPL(mlx4_register_mac); - -int mlx4_get_base_qpn(struct mlx4_dev *dev, u8 port) -{ - return dev->caps.reserved_qps_base[MLX4_QP_REGION_ETH_ADDR] + - (port - 1) * (1 << dev->caps.log_num_macs); -} -EXPORT_SYMBOL_GPL(mlx4_get_base_qpn); - -void __mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, u64 mac) -{ - struct mlx4_port_info *info; - struct mlx4_mac_table *table; - int index; - - if (port < 1 || port > dev->caps.num_ports) { - mlx4_warn(dev, "invalid port number (%d), aborting...\n", port); - return; - } - info = &mlx4_priv(dev)->port[port]; - table = &info->mac_table; - mutex_lock(&table->mutex); - - index = find_index(dev, table, mac); - - if (validate_index(dev, table, index)) - goto out; - - if (--table->refs[index]) { - mlx4_dbg(dev, "Have more references for index %d," - "no need to modify mac table\n", index); - goto out; - } - - table->entries[index] = 0; - mlx4_set_port_mac_table(dev, port, table->entries); - --table->total; -out: - mutex_unlock(&table->mutex); -} -EXPORT_SYMBOL_GPL(__mlx4_unregister_mac); - -void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, u64 mac) -{ - u64 out_param = 0; - - if (mlx4_is_mfunc(dev)) { - if (!(dev->flags & MLX4_FLAG_OLD_REG_MAC)) { - (void) mlx4_cmd_imm(dev, mac, &out_param, - ((u32) port) << 8 | (u32) RES_MAC, - RES_OP_RESERVE_AND_MAP, MLX4_CMD_FREE_RES, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); - } else { - /* use old unregister mac format */ - set_param_l(&out_param, port); - (void) mlx4_cmd_imm(dev, mac, &out_param, RES_MAC, - RES_OP_RESERVE_AND_MAP, MLX4_CMD_FREE_RES, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); - } - return; - } - __mlx4_unregister_mac(dev, port, mac); - return; -} -EXPORT_SYMBOL_GPL(mlx4_unregister_mac); - -int __mlx4_replace_mac(struct mlx4_dev *dev, u8 port, int qpn, u64 new_mac) -{ - struct mlx4_port_info *info = &mlx4_priv(dev)->port[port]; - struct mlx4_mac_table *table = &info->mac_table; - int index = qpn - info->base_qpn; - int err = 0; - - /* CX1 doesn't support multi-functions */ - mutex_lock(&table->mutex); - - err = validate_index(dev, table, index); - if (err) - goto out; - - table->entries[index] = cpu_to_be64(new_mac | MLX4_MAC_VALID); - - err = mlx4_set_port_mac_table(dev, port, table->entries); - if (unlikely(err)) { - mlx4_err(dev, "Failed adding MAC: 0x%llx\n", - (unsigned long long) new_mac); - table->entries[index] = 0; - } -out: - mutex_unlock(&table->mutex); - return err; -} -EXPORT_SYMBOL_GPL(__mlx4_replace_mac); - -static int mlx4_set_port_vlan_table(struct mlx4_dev *dev, u8 port, - __be32 *entries) -{ - struct mlx4_cmd_mailbox *mailbox; - u32 in_mod; - int err; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - memcpy(mailbox->buf, entries, MLX4_VLAN_TABLE_SIZE); - in_mod = MLX4_SET_PORT_VLAN_TABLE << 8 | port; - err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); - - mlx4_free_cmd_mailbox(dev, mailbox); - - return err; -} - -int mlx4_find_cached_vlan(struct mlx4_dev *dev, u8 port, u16 vid, int *idx) -{ - struct mlx4_vlan_table *table = &mlx4_priv(dev)->port[port].vlan_table; - int i; - - for (i = 0; i < MLX4_MAX_VLAN_NUM; ++i) { - if (table->refs[i] && - (vid == (MLX4_VLAN_MASK & - be32_to_cpu(table->entries[i])))) { - /* VLAN already registered, increase reference count */ - *idx = i; - return 0; - } - } - - return -ENOENT; -} -EXPORT_SYMBOL_GPL(mlx4_find_cached_vlan); - -int __mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, - int *index) -{ - struct mlx4_vlan_table *table = &mlx4_priv(dev)->port[port].vlan_table; - int i, err = 0; - int free = -1; - - mutex_lock(&table->mutex); - - if (table->total == table->max) { - /* No free vlan entries */ - err = -ENOSPC; - goto out; - } - - for (i = MLX4_VLAN_REGULAR; i < MLX4_MAX_VLAN_NUM; i++) { - if (free < 0 && (table->refs[i] == 0)) { - free = i; - continue; - } - - if (table->refs[i] && - (vlan == (MLX4_VLAN_MASK & - be32_to_cpu(table->entries[i])))) { - /* Vlan already registered, increase references count */ - *index = i; - ++table->refs[i]; - goto out; - } - } - - if (free < 0) { - err = -ENOMEM; - goto out; - } - - /* Register new VLAN */ - table->refs[free] = 1; - table->entries[free] = cpu_to_be32(vlan | MLX4_VLAN_VALID); - - err = mlx4_set_port_vlan_table(dev, port, table->entries); - if (unlikely(err)) { - mlx4_warn(dev, "Failed adding vlan: %u\n", vlan); - table->refs[free] = 0; - table->entries[free] = 0; - goto out; - } - - *index = free; - ++table->total; -out: - mutex_unlock(&table->mutex); - return err; -} - -int mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, int *index) -{ - u64 out_param = 0; - int err; - - if (vlan > 4095) - return -EINVAL; - - if (mlx4_is_mfunc(dev)) { - err = mlx4_cmd_imm(dev, vlan, &out_param, - ((u32) port) << 8 | (u32) RES_VLAN, - RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); - if (!err) - *index = get_param_l(&out_param); - - return err; - } - return __mlx4_register_vlan(dev, port, vlan, index); -} -EXPORT_SYMBOL_GPL(mlx4_register_vlan); - -void __mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, u16 vlan) -{ - struct mlx4_vlan_table *table = &mlx4_priv(dev)->port[port].vlan_table; - int index; - - mutex_lock(&table->mutex); - if (mlx4_find_cached_vlan(dev, port, vlan, &index)) { - mlx4_warn(dev, "vlan 0x%x is not in the vlan table\n", vlan); - goto out; - } - - if (index < MLX4_VLAN_REGULAR) { - mlx4_warn(dev, "Trying to free special vlan index %d\n", index); - goto out; - } - - if (--table->refs[index]) { - mlx4_dbg(dev, "Have %d more references for index %d, " - "no need to modify vlan table\n", table->refs[index], - index); - goto out; - } - table->entries[index] = 0; - mlx4_set_port_vlan_table(dev, port, table->entries); - --table->total; -out: - mutex_unlock(&table->mutex); -} - -void mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, u16 vlan) -{ - u64 out_param = 0; - - if (mlx4_is_mfunc(dev)) { - (void) mlx4_cmd_imm(dev, vlan, &out_param, - ((u32) port) << 8 | (u32) RES_VLAN, - RES_OP_RESERVE_AND_MAP, - MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_WRAPPED); - return; - } - __mlx4_unregister_vlan(dev, port, vlan); -} -EXPORT_SYMBOL_GPL(mlx4_unregister_vlan); - -int mlx4_get_port_ib_caps(struct mlx4_dev *dev, u8 port, __be32 *caps) -{ - struct mlx4_cmd_mailbox *inmailbox, *outmailbox; - u8 *inbuf, *outbuf; - int err; - - inmailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(inmailbox)) - return PTR_ERR(inmailbox); - - outmailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(outmailbox)) { - mlx4_free_cmd_mailbox(dev, inmailbox); - return PTR_ERR(outmailbox); - } - - inbuf = inmailbox->buf; - outbuf = outmailbox->buf; - memset(inbuf, 0, 256); - memset(outbuf, 0, 256); - inbuf[0] = 1; - inbuf[1] = 1; - inbuf[2] = 1; - inbuf[3] = 1; - *(__be16 *) (&inbuf[16]) = cpu_to_be16(0x0015); - *(__be32 *) (&inbuf[20]) = cpu_to_be32(port); - - err = mlx4_cmd_box(dev, inmailbox->dma, outmailbox->dma, port, 3, - MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C, - MLX4_CMD_NATIVE); - if (!err) - *caps = *(__be32 *) (outbuf + 84); - mlx4_free_cmd_mailbox(dev, inmailbox); - mlx4_free_cmd_mailbox(dev, outmailbox); - return err; -} -static struct mlx4_roce_gid_entry zgid_entry; - -int mlx4_get_slave_num_gids(struct mlx4_dev *dev, int slave) -{ - if (slave == 0) - return MLX4_ROCE_PF_GIDS; - if (slave <= ((MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS) % dev->num_vfs)) - return ((MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS) / dev->num_vfs) + 1; - return (MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS) / dev->num_vfs; -} - -int mlx4_get_base_gid_ix(struct mlx4_dev *dev, int slave) -{ - int gids; - int vfs; - - gids = MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS; - vfs = dev->num_vfs; - - if (slave == 0) - return 0; - if (slave <= gids % vfs) - return MLX4_ROCE_PF_GIDS + ((gids / vfs) + 1) * (slave - 1); - - return MLX4_ROCE_PF_GIDS + (gids % vfs) + ((gids / vfs) * (slave - 1)); -} - -static int mlx4_common_set_port(struct mlx4_dev *dev, int slave, u32 in_mod, - u8 op_mod, struct mlx4_cmd_mailbox *inbox) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_port_info *port_info; - struct mlx4_mfunc_master_ctx *master = &priv->mfunc.master; - struct mlx4_slave_state *slave_st = &master->slave_state[slave]; - struct mlx4_set_port_rqp_calc_context *qpn_context; - struct mlx4_set_port_general_context *gen_context; - struct mlx4_roce_gid_entry *gid_entry_tbl, *gid_entry_mbox, *gid_entry_mb1; - int reset_qkey_viols; - int port; - int is_eth; - int num_gids; - int base; - u32 in_modifier; - u32 promisc; - u16 mtu, prev_mtu; - int err; - int i, j; - int offset; - __be32 agg_cap_mask; - __be32 slave_cap_mask; - __be32 new_cap_mask; - - port = in_mod & 0xff; - in_modifier = (in_mod >> 8) & 0xff; - is_eth = op_mod; - port_info = &priv->port[port]; - - if (op_mod > 1) - return -EINVAL; - - /* Slaves cannot perform SET_PORT operations except changing MTU */ - if (is_eth) { - if (slave != dev->caps.function && - in_modifier != MLX4_SET_PORT_GENERAL && - in_modifier != MLX4_SET_PORT_GID_TABLE) { - mlx4_warn(dev, "denying SET_PORT for slave:%d," - "port %d, config_select 0x%x\n", - slave, port, in_modifier); - return -EINVAL; - } - switch (in_modifier) { - case MLX4_SET_PORT_RQP_CALC: - qpn_context = inbox->buf; - qpn_context->base_qpn = - cpu_to_be32(port_info->base_qpn); - qpn_context->n_mac = 0x7; - promisc = be32_to_cpu(qpn_context->promisc) >> - SET_PORT_PROMISC_SHIFT; - qpn_context->promisc = cpu_to_be32( - promisc << SET_PORT_PROMISC_SHIFT | - port_info->base_qpn); - promisc = be32_to_cpu(qpn_context->mcast) >> - SET_PORT_MC_PROMISC_SHIFT; - qpn_context->mcast = cpu_to_be32( - promisc << SET_PORT_MC_PROMISC_SHIFT | - port_info->base_qpn); - break; - case MLX4_SET_PORT_GENERAL: - gen_context = inbox->buf; - /* Mtu is configured as the max MTU among all the - * the functions on the port. */ - mtu = be16_to_cpu(gen_context->mtu); - mtu = min_t(int, mtu, dev->caps.eth_mtu_cap[port] + - ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN); - prev_mtu = slave_st->mtu[port]; - slave_st->mtu[port] = mtu; - if (mtu > master->max_mtu[port]) - master->max_mtu[port] = mtu; - if (mtu < prev_mtu && prev_mtu == - master->max_mtu[port]) { - slave_st->mtu[port] = mtu; - master->max_mtu[port] = mtu; - for (i = 0; i < dev->num_slaves; i++) { - master->max_mtu[port] = - max(master->max_mtu[port], - master->slave_state[i].mtu[port]); - } - } - - gen_context->mtu = cpu_to_be16(master->max_mtu[port]); - break; - case MLX4_SET_PORT_GID_TABLE: - /* change to MULTIPLE entries: number of guest's gids - * need a FOR-loop here over number of gids the guest has. - * 1. Check no duplicates in gids passed by slave - */ - num_gids = mlx4_get_slave_num_gids(dev, slave); - base = mlx4_get_base_gid_ix(dev, slave); - gid_entry_mbox = (struct mlx4_roce_gid_entry *) (inbox->buf); - for (i = 0; i < num_gids; gid_entry_mbox++, i++) { - if (!memcmp(gid_entry_mbox->raw, zgid_entry.raw, - sizeof(zgid_entry))) - continue; - gid_entry_mb1 = gid_entry_mbox + 1; - for (j = i + 1; j < num_gids; gid_entry_mb1++, j++) { - if (!memcmp(gid_entry_mb1->raw, - zgid_entry.raw, sizeof(zgid_entry))) - continue; - if (!memcmp(gid_entry_mb1->raw, gid_entry_mbox->raw, - sizeof(gid_entry_mbox->raw))) { - /* found duplicate */ - return -EINVAL; - } - } - } - - /* 2. Check that do not have duplicates in OTHER - * entries in the port GID table - */ - for (i = 0; i < MLX4_ROCE_MAX_GIDS; i++) { - if (i >= base && i < base + num_gids) - continue; /* don't compare to slave's current gids */ - gid_entry_tbl = &priv->roce_gids[port - 1][i]; - if (!memcmp(gid_entry_tbl->raw, zgid_entry.raw, sizeof(zgid_entry))) - continue; - gid_entry_mbox = (struct mlx4_roce_gid_entry *) (inbox->buf); - for (j = 0; j < num_gids; gid_entry_mbox++, j++) { - if (!memcmp(gid_entry_mbox->raw, zgid_entry.raw, - sizeof(zgid_entry))) - continue; - if (!memcmp(gid_entry_mbox->raw, gid_entry_tbl->raw, - sizeof(gid_entry_tbl->raw))) { - /* found duplicate */ - mlx4_warn(dev, "requested gid entry for slave:%d " - "is a duplicate of gid at index %d\n", - slave, i); - return -EINVAL; - } - } - } - - /* insert slave GIDs with memcpy, starting at slave's base index */ - gid_entry_mbox = (struct mlx4_roce_gid_entry *) (inbox->buf); - for (i = 0, offset = base; i < num_gids; gid_entry_mbox++, offset++, i++) - memcpy(priv->roce_gids[port - 1][offset].raw, gid_entry_mbox->raw, 16); - - /* Now, copy roce port gids table to current mailbox for passing to FW */ - gid_entry_mbox = (struct mlx4_roce_gid_entry *) (inbox->buf); - for (i = 0; i < MLX4_ROCE_MAX_GIDS; gid_entry_mbox++, i++) - memcpy(gid_entry_mbox->raw, priv->roce_gids[port - 1][i].raw, 16); - - break; - } - return mlx4_cmd(dev, inbox->dma, in_mod & 0xffff, op_mod, - MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_NATIVE); - } - - /* For IB, we only consider: - * - The capability mask, which is set to the aggregate of all - * slave function capabilities - * - The QKey violatin counter - reset according to each request. - */ - - if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { - reset_qkey_viols = (*(u8 *) inbox->buf) & 0x40; - new_cap_mask = ((__be32 *) inbox->buf)[2]; - } else { - reset_qkey_viols = ((u8 *) inbox->buf)[3] & 0x1; - new_cap_mask = ((__be32 *) inbox->buf)[1]; - } - - /* slave may not set the IS_SM capability for the port */ - if (slave != mlx4_master_func_num(dev) && - (be32_to_cpu(new_cap_mask) & MLX4_PORT_CAP_IS_SM)) - return -EINVAL; - - /* No DEV_MGMT in multifunc mode */ - if (mlx4_is_mfunc(dev) && - (be32_to_cpu(new_cap_mask) & MLX4_PORT_CAP_DEV_MGMT_SUP)) - return -EINVAL; - - agg_cap_mask = 0; - slave_cap_mask = - priv->mfunc.master.slave_state[slave].ib_cap_mask[port]; - priv->mfunc.master.slave_state[slave].ib_cap_mask[port] = new_cap_mask; - for (i = 0; i < dev->num_slaves; i++) - agg_cap_mask |= - priv->mfunc.master.slave_state[i].ib_cap_mask[port]; - - /* only clear mailbox for guests. Master may be setting - * MTU or PKEY table size - */ - if (slave != dev->caps.function) - memset(inbox->buf, 0, 256); - if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { - *(u8 *) inbox->buf |= !!reset_qkey_viols << 6; - ((__be32 *) inbox->buf)[2] = agg_cap_mask; - } else { - ((u8 *) inbox->buf)[3] |= !!reset_qkey_viols; - ((__be32 *) inbox->buf)[1] = agg_cap_mask; - } - - err = mlx4_cmd(dev, inbox->dma, port, is_eth, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); - if (err) - priv->mfunc.master.slave_state[slave].ib_cap_mask[port] = - slave_cap_mask; - return err; -} - -int mlx4_SET_PORT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - return mlx4_common_set_port(dev, slave, vhcr->in_modifier, - vhcr->op_modifier, inbox); -} - -/* bit locations for set port command with zero op modifier */ -enum { - MLX4_SET_PORT_VL_CAP = 4, /* bits 7:4 */ - MLX4_SET_PORT_MTU_CAP = 12, /* bits 15:12 */ - MLX4_CHANGE_PORT_PKEY_TBL_SZ = 20, - MLX4_CHANGE_PORT_VL_CAP = 21, - MLX4_CHANGE_PORT_MTU_CAP = 22, -}; - -int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port, int pkey_tbl_sz) -{ - struct mlx4_cmd_mailbox *mailbox; - int err = -EINVAL, vl_cap, pkey_tbl_flag = 0; - u32 in_mod; - - if (dev->caps.port_type[port] == MLX4_PORT_TYPE_NONE) - return 0; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - memset(mailbox->buf, 0, 256); - - if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) { - in_mod = MLX4_SET_PORT_GENERAL << 8 | port; - err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, - MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_WRAPPED); - } else { - ((__be32 *) mailbox->buf)[1] = dev->caps.ib_port_def_cap[port]; - - if (pkey_tbl_sz >= 0 && mlx4_is_master(dev)) { - pkey_tbl_flag = 1; - ((__be16 *) mailbox->buf)[20] = cpu_to_be16(pkey_tbl_sz); - } - - /* IB VL CAP enum isn't used by the firmware, just numerical values */ - for (vl_cap = dev->caps.vl_cap[port]; - vl_cap >= 1; vl_cap >>= 1) { - ((__be32 *) mailbox->buf)[0] = cpu_to_be32( - (1 << MLX4_CHANGE_PORT_MTU_CAP) | - (1 << MLX4_CHANGE_PORT_VL_CAP) | - (pkey_tbl_flag << MLX4_CHANGE_PORT_PKEY_TBL_SZ) | - (dev->caps.port_ib_mtu[port] << MLX4_SET_PORT_MTU_CAP) | - (vl_cap << MLX4_SET_PORT_VL_CAP)); - err = mlx4_cmd(dev, mailbox->dma, port, 0, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); - if (err != -ENOMEM) - break; - } - } - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - -int mlx4_SET_PORT_general(struct mlx4_dev *dev, u8 port, int mtu, - u8 pptx, u8 pfctx, u8 pprx, u8 pfcrx) -{ - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_set_port_general_context *context; - int err; - u32 in_mod; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - context = mailbox->buf; - memset(context, 0, sizeof *context); - - context->flags = SET_PORT_GEN_ALL_VALID; - context->mtu = cpu_to_be16(mtu); - context->pptx = (pptx * (!pfctx)) << 7; - context->pfctx = pfctx; - context->pprx = (pprx * (!pfcrx)) << 7; - context->pfcrx = pfcrx; - - in_mod = MLX4_SET_PORT_GENERAL << 8 | port; - err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} -EXPORT_SYMBOL(mlx4_SET_PORT_general); - -int mlx4_SET_PORT_qpn_calc(struct mlx4_dev *dev, u8 port, u32 base_qpn, - u8 promisc) -{ - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_set_port_rqp_calc_context *context; - int err; - u32 in_mod; - u32 m_promisc = (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) ? - MCAST_DIRECT : MCAST_DEFAULT; - - if (dev->caps.steering_mode != MLX4_STEERING_MODE_A0) - return 0; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - context = mailbox->buf; - memset(context, 0, sizeof *context); - - context->base_qpn = cpu_to_be32(base_qpn); - context->n_mac = dev->caps.log_num_macs; - context->promisc = cpu_to_be32(promisc << SET_PORT_PROMISC_SHIFT | - base_qpn); - context->mcast = cpu_to_be32(m_promisc << SET_PORT_MC_PROMISC_SHIFT | - base_qpn); - context->intra_no_vlan = 0; - context->no_vlan = MLX4_NO_VLAN_IDX; - context->intra_vlan_miss = 0; - context->vlan_miss = MLX4_VLAN_MISS_IDX; - - in_mod = MLX4_SET_PORT_RQP_CALC << 8 | port; - err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} -EXPORT_SYMBOL(mlx4_SET_PORT_qpn_calc); - -int mlx4_SET_PORT_PRIO2TC(struct mlx4_dev *dev, u8 port, u8 *prio2tc) -{ - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_set_port_prio2tc_context *context; - int err; - u32 in_mod; - int i; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - context = mailbox->buf; - memset(context, 0, sizeof *context); - - for (i = 0; i < MLX4_NUM_UP; i += 2) - context->prio2tc[i >> 1] = prio2tc[i] << 4 | prio2tc[i + 1]; - - in_mod = MLX4_SET_PORT_PRIO2TC << 8 | port; - err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} -EXPORT_SYMBOL(mlx4_SET_PORT_PRIO2TC); - -int mlx4_SET_PORT_SCHEDULER(struct mlx4_dev *dev, u8 port, u8 *tc_tx_bw, - u8 *pg, u16 *ratelimit) -{ - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_set_port_scheduler_context *context; - int err; - u32 in_mod; - int i; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - context = mailbox->buf; - memset(context, 0, sizeof *context); - - for (i = 0; i < MLX4_NUM_TC; i++) { - struct mlx4_port_scheduler_tc_cfg_be *tc = &context->tc[i]; - u16 r; - if (ratelimit && ratelimit[i]) { - if (ratelimit[i] <= MLX4_MAX_100M_UNITS_VAL) { - r = ratelimit[i]; - tc->max_bw_units = - htons(MLX4_RATELIMIT_100M_UNITS); - } else { - r = ratelimit[i]/10; - tc->max_bw_units = - htons(MLX4_RATELIMIT_1G_UNITS); - } - tc->max_bw_value = htons(r); - } else { - tc->max_bw_value = htons(MLX4_RATELIMIT_DEFAULT); - tc->max_bw_units = htons(MLX4_RATELIMIT_1G_UNITS); - } - - tc->pg = htons(pg[i]); - tc->bw_precentage = htons(tc_tx_bw[i]); - } - - in_mod = MLX4_SET_PORT_SCHEDULER << 8 | port; - err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} -EXPORT_SYMBOL(mlx4_SET_PORT_SCHEDULER); - -int mlx4_SET_MCAST_FLTR_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err = 0; - - return err; -} - -int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, - u64 mac, u64 clear, u8 mode) -{ - return mlx4_cmd(dev, (mac | (clear << 63)), port, mode, - MLX4_CMD_SET_MCAST_FLTR, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_WRAPPED); -} -EXPORT_SYMBOL(mlx4_SET_MCAST_FLTR); - -int mlx4_SET_VLAN_FLTR_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - int err = 0; - - return err; -} - -int mlx4_DUMP_ETH_STATS_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - return 0; -} - -int mlx4_get_slave_from_roce_gid(struct mlx4_dev *dev, int port, u8 *gid, int *slave_id) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int i, found_ix = -1; - int vf_gids = MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS; - - if (!mlx4_is_mfunc(dev)) - return -EINVAL; - - for (i = 0; i < MLX4_ROCE_MAX_GIDS; i++) { - if (!memcmp(priv->roce_gids[port - 1][i].raw, gid, 16)) { - found_ix = i; - break; - } - } - - if (found_ix >= 0) { - if (found_ix < MLX4_ROCE_PF_GIDS) - *slave_id = 0; - else if (found_ix < MLX4_ROCE_PF_GIDS + (vf_gids % dev->num_vfs) * - (vf_gids / dev->num_vfs + 1)) - *slave_id = ((found_ix - MLX4_ROCE_PF_GIDS) / - (vf_gids / dev->num_vfs + 1)) + 1; - else - *slave_id = - ((found_ix - MLX4_ROCE_PF_GIDS - - ((vf_gids % dev->num_vfs) * ((vf_gids / dev->num_vfs + 1)))) / - (vf_gids / dev->num_vfs)) + vf_gids % dev->num_vfs + 1; - } - - return (found_ix >= 0) ? 0 : -EINVAL; -} -EXPORT_SYMBOL(mlx4_get_slave_from_roce_gid); - -int mlx4_get_roce_gid_from_slave(struct mlx4_dev *dev, int port, int slave_id, u8 *gid) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - if (!mlx4_is_master(dev)) - return -EINVAL; - - memcpy(gid, priv->roce_gids[port - 1][slave_id].raw, 16); - return 0; -} -EXPORT_SYMBOL(mlx4_get_roce_gid_from_slave); - -/* Cable Module Info */ -#define MODULE_INFO_MAX_READ 48 - -#define I2C_ADDR_LOW 0x50 -#define I2C_ADDR_HIGH 0x51 -#define I2C_PAGE_SIZE 256 - -/* Module Info Data */ -struct mlx4_cable_info { - u8 i2c_addr; - u8 page_num; - __be16 dev_mem_address; - __be16 reserved1; - __be16 size; - __be32 reserved2[2]; - u8 data[MODULE_INFO_MAX_READ]; -}; - -enum cable_info_err { - CABLE_INF_INV_PORT = 0x1, - CABLE_INF_OP_NOSUP = 0x2, - CABLE_INF_NOT_CONN = 0x3, - CABLE_INF_NO_EEPRM = 0x4, - CABLE_INF_PAGE_ERR = 0x5, - CABLE_INF_INV_ADDR = 0x6, - CABLE_INF_I2C_ADDR = 0x7, - CABLE_INF_QSFP_VIO = 0x8, - CABLE_INF_I2C_BUSY = 0x9, -}; - -#define MAD_STATUS_2_CABLE_ERR(mad_status) ((mad_status >> 8) & 0xFF) - -#ifdef DEBUG -static inline const char *cable_info_mad_err_str(u16 mad_status) -{ - u8 err = MAD_STATUS_2_CABLE_ERR(mad_status); - - switch (err) { - case CABLE_INF_INV_PORT: - return "invalid port selected"; - case CABLE_INF_OP_NOSUP: - return "operation not supported for this port (the port is of type CX4 or internal)"; - case CABLE_INF_NOT_CONN: - return "cable is not connected"; - case CABLE_INF_NO_EEPRM: - return "the connected cable has no EPROM (passive copper cable)"; - case CABLE_INF_PAGE_ERR: - return "page number is greater than 15"; - case CABLE_INF_INV_ADDR: - return "invalid device_address or size (that is, size equals 0 or address+size is greater than 256)"; - case CABLE_INF_I2C_ADDR: - return "invalid I2C slave address"; - case CABLE_INF_QSFP_VIO: - return "at least one cable violates the QSFP specification and ignores the modsel signal"; - case CABLE_INF_I2C_BUSY: - return "I2C bus is constantly busy"; - } - return "Unknown Error"; -} -#endif /* DEBUG */ - -/** - * mlx4_get_module_info - Read cable module eeprom data - * @dev: mlx4_dev. - * @port: port number. - * @offset: byte offset in eeprom to start reading data from. - * @size: num of bytes to read. - * @data: output buffer to put the requested data into. - * - * Reads cable module eeprom data, puts the outcome data into - * data pointer paramer. - * Returns num of read bytes on success or a negative error - * code. - */ -int mlx4_get_module_info(struct mlx4_dev *dev, u8 port, u16 offset, - u16 size, u8 *data) -{ - struct mlx4_cmd_mailbox *inbox, *outbox; - struct mlx4_mad_ifc *inmad, *outmad; - struct mlx4_cable_info *cable_info; - u16 i2c_addr; - int ret; - - if (size > MODULE_INFO_MAX_READ) - size = MODULE_INFO_MAX_READ; - - inbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(inbox)) { - mlx4_err(dev, - "mlx4_alloc_cmd_mailbox returned with error(%lx)", PTR_ERR(inbox)); - return PTR_ERR(inbox); - } - - outbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(outbox)) { - mlx4_free_cmd_mailbox(dev, inbox); - mlx4_err(dev, - "mlx4_alloc_cmd_mailbox returned with error(%lx)", PTR_ERR(outbox)); - return PTR_ERR(outbox); - } - - inmad = (struct mlx4_mad_ifc *)(inbox->buf); - outmad = (struct mlx4_mad_ifc *)(outbox->buf); - - inmad->method = 0x1; /* Get */ - inmad->class_version = 0x1; - inmad->mgmt_class = 0x1; - inmad->base_version = 0x1; - inmad->attr_id = cpu_to_be16(0xFF60); /* Module Info */ - - if (offset < I2C_PAGE_SIZE && offset + size > I2C_PAGE_SIZE) - /* Cross pages reads are not allowed - * read until offset 256 in low page - */ - size -= offset + size - I2C_PAGE_SIZE; - - i2c_addr = I2C_ADDR_LOW; - if (offset >= I2C_PAGE_SIZE) { - /* Reset offset to high page */ - i2c_addr = I2C_ADDR_HIGH; - offset -= I2C_PAGE_SIZE; - } - - cable_info = (struct mlx4_cable_info *)inmad->data; - cable_info->dev_mem_address = cpu_to_be16(offset); - cable_info->page_num = 0; - cable_info->i2c_addr = i2c_addr; - cable_info->size = cpu_to_be16(size); - - ret = mlx4_cmd_box(dev, inbox->dma, outbox->dma, port, 3, - MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); - if (ret) - goto out; - - if (be16_to_cpu(outmad->status)) { - /* Mad returned with bad status */ - ret = be16_to_cpu(outmad->status); -#ifdef DEBUG - mlx4_warn(dev, "MLX4_CMD_MAD_IFC Get Module info attr(%x) " - "port(%d) i2c_addr(%x) offset(%d) size(%d): Response " - "Mad Status(%x) - %s\n", 0xFF60, port, i2c_addr, offset, - size, ret, cable_info_mad_err_str(ret)); -#endif - if (i2c_addr == I2C_ADDR_HIGH && - MAD_STATUS_2_CABLE_ERR(ret) == CABLE_INF_I2C_ADDR) - /* Some SFP cables do not support i2c slave - * address 0x51 (high page), abort silently. - */ - ret = 0; - else - ret = -ret; - goto out; - } - cable_info = (struct mlx4_cable_info *)outmad->data; - memcpy(data, cable_info->data, size); - ret = size; -out: - mlx4_free_cmd_mailbox(dev, inbox); - mlx4_free_cmd_mailbox(dev, outbox); - return ret; -} -EXPORT_SYMBOL(mlx4_get_module_info); Property changes on: stable/11/sys/ofed/drivers/net/mlx4/port.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/eq.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/eq.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/eq.c (nonexistent) @@ -1,1408 +0,0 @@ -/* - * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include -#include - -#include - -#include "mlx4.h" -#include "fw.h" - -enum { - MLX4_IRQNAME_SIZE = 32 -}; - -enum { - MLX4_NUM_ASYNC_EQE = 0x100, - MLX4_NUM_SPARE_EQE = 0x80, - MLX4_EQ_ENTRY_SIZE = 0x20 -}; - -#define MLX4_EQ_STATUS_OK ( 0 << 28) -#define MLX4_EQ_STATUS_WRITE_FAIL (10 << 28) -#define MLX4_EQ_OWNER_SW ( 0 << 24) -#define MLX4_EQ_OWNER_HW ( 1 << 24) -#define MLX4_EQ_FLAG_EC ( 1 << 18) -#define MLX4_EQ_FLAG_OI ( 1 << 17) -#define MLX4_EQ_STATE_ARMED ( 9 << 8) -#define MLX4_EQ_STATE_FIRED (10 << 8) -#define MLX4_EQ_STATE_ALWAYS_ARMED (11 << 8) - -#define MLX4_ASYNC_EVENT_MASK ((1ull << MLX4_EVENT_TYPE_PATH_MIG) | \ - (1ull << MLX4_EVENT_TYPE_COMM_EST) | \ - (1ull << MLX4_EVENT_TYPE_SQ_DRAINED) | \ - (1ull << MLX4_EVENT_TYPE_CQ_ERROR) | \ - (1ull << MLX4_EVENT_TYPE_WQ_CATAS_ERROR) | \ - (1ull << MLX4_EVENT_TYPE_EEC_CATAS_ERROR) | \ - (1ull << MLX4_EVENT_TYPE_PATH_MIG_FAILED) | \ - (1ull << MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \ - (1ull << MLX4_EVENT_TYPE_WQ_ACCESS_ERROR) | \ - (1ull << MLX4_EVENT_TYPE_PORT_CHANGE) | \ - (1ull << MLX4_EVENT_TYPE_ECC_DETECT) | \ - (1ull << MLX4_EVENT_TYPE_SRQ_CATAS_ERROR) | \ - (1ull << MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE) | \ - (1ull << MLX4_EVENT_TYPE_SRQ_LIMIT) | \ - (1ull << MLX4_EVENT_TYPE_CMD) | \ - (1ull << MLX4_EVENT_TYPE_OP_REQUIRED) | \ - (1ull << MLX4_EVENT_TYPE_COMM_CHANNEL) | \ - (1ull << MLX4_EVENT_TYPE_FLR_EVENT) | \ - (1ull << MLX4_EVENT_TYPE_FATAL_WARNING)) - -static u64 get_async_ev_mask(struct mlx4_dev *dev) -{ - u64 async_ev_mask = MLX4_ASYNC_EVENT_MASK; - if (dev->caps.flags & MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV) - async_ev_mask |= (1ull << MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT); - if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RECOVERABLE_ERROR_EVENT) - async_ev_mask |= (1ull << MLX4_EVENT_TYPE_RECOVERABLE_ERROR_EVENT); - - return async_ev_mask; -} - -static void eq_set_ci(struct mlx4_eq *eq, int req_not) -{ - __raw_writel((__force u32) cpu_to_be32((eq->cons_index & 0xffffff) | - req_not << 31), - eq->doorbell); - /* We still want ordering, just not swabbing, so add a barrier */ - mb(); -} - -static struct mlx4_eqe *get_eqe(struct mlx4_eq *eq, u32 entry, u8 eqe_factor) -{ - /* (entry & (eq->nent - 1)) gives us a cyclic array */ - unsigned long offset = (entry & (eq->nent - 1)) * (MLX4_EQ_ENTRY_SIZE << eqe_factor); - /* CX3 is capable of extending the EQE from 32 to 64 bytes. - * When this feature is enabled, the first (in the lower addresses) - * 32 bytes in the 64 byte EQE are reserved and the next 32 bytes - * contain the legacy EQE information. - */ - return eq->page_list[offset / PAGE_SIZE].buf + (offset + (eqe_factor ? MLX4_EQ_ENTRY_SIZE : 0)) % PAGE_SIZE; -} - -static struct mlx4_eqe *next_eqe_sw(struct mlx4_eq *eq, u8 eqe_factor) -{ - struct mlx4_eqe *eqe = get_eqe(eq, eq->cons_index, eqe_factor); - return !!(eqe->owner & 0x80) ^ !!(eq->cons_index & eq->nent) ? NULL : eqe; -} - -static struct mlx4_eqe *next_slave_event_eqe(struct mlx4_slave_event_eq *slave_eq) -{ - struct mlx4_eqe *eqe = - &slave_eq->event_eqe[slave_eq->cons & (SLAVE_EVENT_EQ_SIZE - 1)]; - return (!!(eqe->owner & 0x80) ^ - !!(slave_eq->cons & SLAVE_EVENT_EQ_SIZE)) ? - eqe : NULL; -} - -void mlx4_gen_slave_eqe(struct work_struct *work) -{ - struct mlx4_mfunc_master_ctx *master = - container_of(work, struct mlx4_mfunc_master_ctx, - slave_event_work); - struct mlx4_mfunc *mfunc = - container_of(master, struct mlx4_mfunc, master); - struct mlx4_priv *priv = container_of(mfunc, struct mlx4_priv, mfunc); - struct mlx4_dev *dev = &priv->dev; - struct mlx4_slave_event_eq *slave_eq = &mfunc->master.slave_eq; - struct mlx4_eqe *eqe; - u8 slave; - int i; - - for (eqe = next_slave_event_eqe(slave_eq); eqe; - eqe = next_slave_event_eqe(slave_eq)) { - slave = eqe->slave_id; - - /* All active slaves need to receive the event */ - if (slave == ALL_SLAVES) { - for (i = 0; i < dev->num_slaves; i++) { - if (mlx4_GEN_EQE(dev, i, eqe)) - mlx4_warn(dev, "Failed to generate " - "event for slave %d\n", i); - } - } else { - if (mlx4_GEN_EQE(dev, slave, eqe)) - mlx4_warn(dev, "Failed to generate event " - "for slave %d\n", slave); - } - ++slave_eq->cons; - } -} - - -static void slave_event(struct mlx4_dev *dev, u8 slave, struct mlx4_eqe *eqe) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_slave_event_eq *slave_eq = &priv->mfunc.master.slave_eq; - struct mlx4_eqe *s_eqe; - unsigned long flags; - - spin_lock_irqsave(&slave_eq->event_lock, flags); - s_eqe = &slave_eq->event_eqe[slave_eq->prod & (SLAVE_EVENT_EQ_SIZE - 1)]; - if ((!!(s_eqe->owner & 0x80)) ^ - (!!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE))) { - mlx4_warn(dev, "Master failed to generate an EQE for slave: %d. " - "No free EQE on slave events queue\n", slave); - spin_unlock_irqrestore(&slave_eq->event_lock, flags); - return; - } - - memcpy(s_eqe, eqe, dev->caps.eqe_size - 1); - s_eqe->slave_id = slave; - /* ensure all information is written before setting the ownersip bit */ - wmb(); - s_eqe->owner = !!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE) ? 0x0 : 0x80; - ++slave_eq->prod; - - queue_work(priv->mfunc.master.comm_wq, - &priv->mfunc.master.slave_event_work); - spin_unlock_irqrestore(&slave_eq->event_lock, flags); -} - -static void mlx4_slave_event(struct mlx4_dev *dev, int slave, - struct mlx4_eqe *eqe) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - if (slave < 0 || slave >= dev->num_slaves || - slave == dev->caps.function) - return; - - if (!priv->mfunc.master.slave_state[slave].active) - return; - - slave_event(dev, slave, eqe); -} - -int mlx4_gen_pkey_eqe(struct mlx4_dev *dev, int slave, u8 port) -{ - struct mlx4_eqe eqe; - - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_slave_state *s_slave = &priv->mfunc.master.slave_state[slave]; - - if (!s_slave->active) - return 0; - - memset(&eqe, 0, sizeof eqe); - - eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT; - eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PKEY_TABLE; - eqe.event.port_mgmt_change.port = port; - - return mlx4_GEN_EQE(dev, slave, &eqe); -} -EXPORT_SYMBOL(mlx4_gen_pkey_eqe); - -int mlx4_gen_guid_change_eqe(struct mlx4_dev *dev, int slave, u8 port) -{ - struct mlx4_eqe eqe; - - /*don't send if we don't have the that slave */ - if (dev->num_vfs < slave) - return 0; - memset(&eqe, 0, sizeof eqe); - - eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT; - eqe.subtype = MLX4_DEV_PMC_SUBTYPE_GUID_INFO; - eqe.event.port_mgmt_change.port = port; - - return mlx4_GEN_EQE(dev, slave, &eqe); -} -EXPORT_SYMBOL(mlx4_gen_guid_change_eqe); - -int mlx4_gen_port_state_change_eqe(struct mlx4_dev *dev, int slave, u8 port, - u8 port_subtype_change) -{ - struct mlx4_eqe eqe; - - /*don't send if we don't have the that slave */ - if (dev->num_vfs < slave) - return 0; - memset(&eqe, 0, sizeof eqe); - - eqe.type = MLX4_EVENT_TYPE_PORT_CHANGE; - eqe.subtype = port_subtype_change; - eqe.event.port_change.port = cpu_to_be32(port << 28); - - mlx4_dbg(dev, "%s: sending: %d to slave: %d on port: %d\n", __func__, - port_subtype_change, slave, port); - return mlx4_GEN_EQE(dev, slave, &eqe); -} -EXPORT_SYMBOL(mlx4_gen_port_state_change_eqe); - -enum slave_port_state mlx4_get_slave_port_state(struct mlx4_dev *dev, int slave, u8 port) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_slave_state *s_state = priv->mfunc.master.slave_state; - if (slave >= dev->num_slaves || port > MLX4_MAX_PORTS) { - pr_err("%s: Error: asking for slave:%d, port:%d\n", - __func__, slave, port); - return SLAVE_PORT_DOWN; - } - return s_state[slave].port_state[port]; -} -EXPORT_SYMBOL(mlx4_get_slave_port_state); - -static int mlx4_set_slave_port_state(struct mlx4_dev *dev, int slave, u8 port, - enum slave_port_state state) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_slave_state *s_state = priv->mfunc.master.slave_state; - - if (slave >= dev->num_slaves || port > MLX4_MAX_PORTS || port == 0) { - pr_err("%s: Error: asking for slave:%d, port:%d\n", - __func__, slave, port); - return -1; - } - s_state[slave].port_state[port] = state; - - return 0; -} - -static void set_all_slave_state(struct mlx4_dev *dev, u8 port, int event) -{ - int i; - enum slave_port_gen_event gen_event; - - for (i = 0; i < dev->num_slaves; i++) - set_and_calc_slave_port_state(dev, i, port, event, &gen_event); -} -/************************************************************************** - The function get as input the new event to that port, - and according to the prev state change the slave's port state. - The events are: - MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN, - MLX4_PORT_STATE_DEV_EVENT_PORT_UP - MLX4_PORT_STATE_IB_EVENT_GID_VALID - MLX4_PORT_STATE_IB_EVENT_GID_INVALID -***************************************************************************/ -int set_and_calc_slave_port_state(struct mlx4_dev *dev, int slave, - u8 port, int event, - enum slave_port_gen_event *gen_event) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_slave_state *ctx = NULL; - unsigned long flags; - int ret = -1; - enum slave_port_state cur_state = - mlx4_get_slave_port_state(dev, slave, port); - - *gen_event = SLAVE_PORT_GEN_EVENT_NONE; - - if (slave >= dev->num_slaves || port > MLX4_MAX_PORTS || port == 0) { - pr_err("%s: Error: asking for slave:%d, port:%d\n", - __func__, slave, port); - return ret; - } - - ctx = &priv->mfunc.master.slave_state[slave]; - spin_lock_irqsave(&ctx->lock, flags); - - switch (cur_state) { - case SLAVE_PORT_DOWN: - if (MLX4_PORT_STATE_DEV_EVENT_PORT_UP == event) - mlx4_set_slave_port_state(dev, slave, port, - SLAVE_PENDING_UP); - break; - case SLAVE_PENDING_UP: - if (MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN == event) - mlx4_set_slave_port_state(dev, slave, port, - SLAVE_PORT_DOWN); - else if (MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID == event) { - mlx4_set_slave_port_state(dev, slave, port, - SLAVE_PORT_UP); - *gen_event = SLAVE_PORT_GEN_EVENT_UP; - } - break; - case SLAVE_PORT_UP: - if (MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN == event) { - mlx4_set_slave_port_state(dev, slave, port, - SLAVE_PORT_DOWN); - *gen_event = SLAVE_PORT_GEN_EVENT_DOWN; - } else if (MLX4_PORT_STATE_IB_EVENT_GID_INVALID == - event) { - mlx4_set_slave_port_state(dev, slave, port, - SLAVE_PENDING_UP); - *gen_event = SLAVE_PORT_GEN_EVENT_DOWN; - } - break; - default: - pr_err("%s: BUG!!! UNKNOWN state: " - "slave:%d, port:%d\n", __func__, slave, port); - goto out; - } - ret = mlx4_get_slave_port_state(dev, slave, port); - -out: - spin_unlock_irqrestore(&ctx->lock, flags); - return ret; -} - -EXPORT_SYMBOL(set_and_calc_slave_port_state); - -int mlx4_gen_slaves_port_mgt_ev(struct mlx4_dev *dev, u8 port, int attr, u16 sm_lid, u8 sm_sl) -{ - struct mlx4_eqe eqe; - - memset(&eqe, 0, sizeof eqe); - - eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT; - eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PORT_INFO; - eqe.event.port_mgmt_change.port = port; - eqe.event.port_mgmt_change.params.port_info.changed_attr = - cpu_to_be32((u32) attr); - if (attr & MSTR_SM_CHANGE_MASK) { - eqe.event.port_mgmt_change.params.port_info.mstr_sm_lid = - cpu_to_be16(sm_lid); - eqe.event.port_mgmt_change.params.port_info.mstr_sm_sl = - sm_sl; - } - - slave_event(dev, ALL_SLAVES, &eqe); - return 0; -} -EXPORT_SYMBOL(mlx4_gen_slaves_port_mgt_ev); - -void mlx4_master_handle_slave_flr(struct work_struct *work) -{ - struct mlx4_mfunc_master_ctx *master = - container_of(work, struct mlx4_mfunc_master_ctx, - slave_flr_event_work); - struct mlx4_mfunc *mfunc = - container_of(master, struct mlx4_mfunc, master); - struct mlx4_priv *priv = - container_of(mfunc, struct mlx4_priv, mfunc); - struct mlx4_dev *dev = &priv->dev; - struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state; - int i; - int err; - unsigned long flags; - - mlx4_dbg(dev, "mlx4_handle_slave_flr\n"); - - for (i = 0 ; i < dev->num_slaves; i++) { - - if (MLX4_COMM_CMD_FLR == slave_state[i].last_cmd) { - mlx4_dbg(dev, "mlx4_handle_slave_flr: " - "clean slave: %d\n", i); - - mlx4_delete_all_resources_for_slave(dev, i); - /*return the slave to running mode*/ - spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags); - slave_state[i].last_cmd = MLX4_COMM_CMD_RESET; - slave_state[i].is_slave_going_down = 0; - spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags); - /*notify the FW:*/ - err = mlx4_cmd(dev, 0, i, 0, MLX4_CMD_INFORM_FLR_DONE, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); - if (err) - mlx4_warn(dev, "Failed to notify FW on " - "FLR done (slave:%d)\n", i); - } - } -} - -static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_eqe *eqe; - int cqn; - int eqes_found = 0; - int set_ci = 0; - int port; - int slave = 0; - int ret; - u32 flr_slave; - u8 update_slave_state; - int i; - enum slave_port_gen_event gen_event; - unsigned long flags; - struct mlx4_vport_state *s_info; - - while ((eqe = next_eqe_sw(eq, dev->caps.eqe_factor))) { - /* - * Make sure we read EQ entry contents after we've - * checked the ownership bit. - */ - rmb(); - - switch (eqe->type) { - case MLX4_EVENT_TYPE_COMP: - cqn = be32_to_cpu(eqe->event.comp.cqn) & 0xffffff; - mlx4_cq_completion(dev, cqn); - break; - - case MLX4_EVENT_TYPE_PATH_MIG: - case MLX4_EVENT_TYPE_COMM_EST: - case MLX4_EVENT_TYPE_SQ_DRAINED: - case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE: - case MLX4_EVENT_TYPE_WQ_CATAS_ERROR: - case MLX4_EVENT_TYPE_PATH_MIG_FAILED: - case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR: - case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR: - mlx4_dbg(dev, "event %d arrived\n", eqe->type); - if (mlx4_is_master(dev)) { - /* forward only to slave owning the QP */ - ret = mlx4_get_slave_from_resource_id(dev, - RES_QP, - be32_to_cpu(eqe->event.qp.qpn) - & 0xffffff, &slave); - if (ret && ret != -ENOENT) { - mlx4_dbg(dev, "QP event %02x(%02x) on " - "EQ %d at index %u: could " - "not get slave id (%d)\n", - eqe->type, eqe->subtype, - eq->eqn, eq->cons_index, ret); - break; - } - - if (!ret && slave != dev->caps.function) { - mlx4_slave_event(dev, slave, eqe); - break; - } - - } - mlx4_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & - 0xffffff, eqe->type); - break; - - case MLX4_EVENT_TYPE_SRQ_LIMIT: - mlx4_dbg(dev, "%s: MLX4_EVENT_TYPE_SRQ_LIMIT\n", - __func__); - /* fall through */ - case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR: - if (mlx4_is_master(dev)) { - /* forward only to slave owning the SRQ */ - ret = mlx4_get_slave_from_resource_id(dev, - RES_SRQ, - be32_to_cpu(eqe->event.srq.srqn) - & 0xffffff, - &slave); - if (ret && ret != -ENOENT) { - mlx4_warn(dev, "SRQ event %02x(%02x) " - "on EQ %d at index %u: could" - " not get slave id (%d)\n", - eqe->type, eqe->subtype, - eq->eqn, eq->cons_index, ret); - break; - } - mlx4_dbg(dev, "%s: slave:%d, srq_no:0x%x, event: %02x(%02x)\n", - __func__, slave, - be32_to_cpu(eqe->event.srq.srqn), - eqe->type, eqe->subtype); - - if (!ret && slave != dev->caps.function) { - mlx4_dbg(dev, "%s: sending event %02x(%02x) to slave:%d\n", - __func__, eqe->type, - eqe->subtype, slave); - mlx4_slave_event(dev, slave, eqe); - break; - } - } - mlx4_srq_event(dev, be32_to_cpu(eqe->event.srq.srqn) & - 0xffffff, eqe->type); - break; - - case MLX4_EVENT_TYPE_CMD: - mlx4_cmd_event(dev, - be16_to_cpu(eqe->event.cmd.token), - eqe->event.cmd.status, - be64_to_cpu(eqe->event.cmd.out_param)); - break; - - case MLX4_EVENT_TYPE_PORT_CHANGE: - port = be32_to_cpu(eqe->event.port_change.port) >> 28; - if (eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_DOWN) { - mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_DOWN, - port); - mlx4_priv(dev)->sense.do_sense_port[port] = 1; - if (!mlx4_is_master(dev)) - break; - for (i = 0; i < dev->num_slaves; i++) { - if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) { - if (i == mlx4_master_func_num(dev)) - continue; - mlx4_dbg(dev, "%s: Sending MLX4_PORT_CHANGE_SUBTYPE_DOWN" - " to slave: %d, port:%d\n", - __func__, i, port); - s_info = &priv->mfunc.master.vf_oper[slave].vport[port].state; - if (IFLA_VF_LINK_STATE_AUTO == s_info->link_state) - mlx4_slave_event(dev, i, eqe); - } else { /* IB port */ - set_and_calc_slave_port_state(dev, i, port, - MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN, - &gen_event); - /*we can be in pending state, then do not send port_down event*/ - if (SLAVE_PORT_GEN_EVENT_DOWN == gen_event) { - if (i == mlx4_master_func_num(dev)) - continue; - mlx4_slave_event(dev, i, eqe); - } - } - } - } else { - mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_UP, port); - - mlx4_priv(dev)->sense.do_sense_port[port] = 0; - - if (!mlx4_is_master(dev)) - break; - if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) - for (i = 0; i < dev->num_slaves; i++) { - if (i == mlx4_master_func_num(dev)) - continue; - s_info = &priv->mfunc.master.vf_oper[slave].vport[port].state; - if (IFLA_VF_LINK_STATE_AUTO == s_info->link_state) - mlx4_slave_event(dev, i, eqe); - } - else /* IB port */ - /* port-up event will be sent to a slave when the - * slave's alias-guid is set. This is done in alias_GUID.c - */ - set_all_slave_state(dev, port, MLX4_DEV_EVENT_PORT_UP); - } - break; - - case MLX4_EVENT_TYPE_CQ_ERROR: - mlx4_warn(dev, "CQ %s on CQN %06x\n", - eqe->event.cq_err.syndrome == 1 ? - "overrun" : "access violation", - be32_to_cpu(eqe->event.cq_err.cqn) & 0xffffff); - if (mlx4_is_master(dev)) { - ret = mlx4_get_slave_from_resource_id(dev, - RES_CQ, - be32_to_cpu(eqe->event.cq_err.cqn) - & 0xffffff, &slave); - if (ret && ret != -ENOENT) { - mlx4_dbg(dev, "CQ event %02x(%02x) on " - "EQ %d at index %u: could " - "not get slave id (%d)\n", - eqe->type, eqe->subtype, - eq->eqn, eq->cons_index, ret); - break; - } - - if (!ret && slave != dev->caps.function) { - mlx4_slave_event(dev, slave, eqe); - break; - } - } - mlx4_cq_event(dev, - be32_to_cpu(eqe->event.cq_err.cqn) - & 0xffffff, - eqe->type); - break; - - case MLX4_EVENT_TYPE_EQ_OVERFLOW: - mlx4_warn(dev, "EQ overrun on EQN %d\n", eq->eqn); - break; - - case MLX4_EVENT_TYPE_OP_REQUIRED: - atomic_inc(&priv->opreq_count); - /* FW commands can't be executed from interrupt context - working in deferred task */ - queue_work(mlx4_wq, &priv->opreq_task); - break; - - case MLX4_EVENT_TYPE_COMM_CHANNEL: - if (!mlx4_is_master(dev)) { - mlx4_warn(dev, "Received comm channel event " - "for non master device\n"); - break; - } - - memcpy(&priv->mfunc.master.comm_arm_bit_vector, - eqe->event.comm_channel_arm.bit_vec, - sizeof eqe->event.comm_channel_arm.bit_vec); - - if (!queue_work(priv->mfunc.master.comm_wq, - &priv->mfunc.master.comm_work)) - mlx4_warn(dev, "Failed to queue comm channel work\n"); - - if (!queue_work(priv->mfunc.master.comm_wq, - &priv->mfunc.master.arm_comm_work)) - mlx4_warn(dev, "Failed to queue arm comm channel work\n"); - break; - - case MLX4_EVENT_TYPE_FLR_EVENT: - flr_slave = be32_to_cpu(eqe->event.flr_event.slave_id); - if (!mlx4_is_master(dev)) { - mlx4_warn(dev, "Non-master function received" - "FLR event\n"); - break; - } - - mlx4_dbg(dev, "FLR event for slave: %d\n", flr_slave); - - if (flr_slave >= dev->num_slaves) { - mlx4_warn(dev, - "Got FLR for unknown function: %d\n", - flr_slave); - update_slave_state = 0; - } else - update_slave_state = 1; - - spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags); - if (update_slave_state) { - priv->mfunc.master.slave_state[flr_slave].active = false; - priv->mfunc.master.slave_state[flr_slave].last_cmd = MLX4_COMM_CMD_FLR; - priv->mfunc.master.slave_state[flr_slave].is_slave_going_down = 1; - } - spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags); - queue_work(priv->mfunc.master.comm_wq, - &priv->mfunc.master.slave_flr_event_work); - break; - - case MLX4_EVENT_TYPE_FATAL_WARNING: - if (eqe->subtype == MLX4_FATAL_WARNING_SUBTYPE_WARMING) { - if (mlx4_is_master(dev)) - for (i = 0; i < dev->num_slaves; i++) { - mlx4_dbg(dev, "%s: Sending " - "MLX4_FATAL_WARNING_SUBTYPE_WARMING" - " to slave: %d\n", __func__, i); - if (i == dev->caps.function) - continue; - mlx4_slave_event(dev, i, eqe); - } - mlx4_err(dev, "Temperature Threshold was reached! " - "Threshold: %d celsius degrees; " - "Current Temperature: %d\n", - be16_to_cpu(eqe->event.warming.warning_threshold), - be16_to_cpu(eqe->event.warming.current_temperature)); - } else - mlx4_warn(dev, "Unhandled event FATAL WARNING (%02x), " - "subtype %02x on EQ %d at index %u. owner=%x, " - "nent=0x%x, slave=%x, ownership=%s\n", - eqe->type, eqe->subtype, eq->eqn, - eq->cons_index, eqe->owner, eq->nent, - eqe->slave_id, - !!(eqe->owner & 0x80) ^ - !!(eq->cons_index & eq->nent) ? "HW" : "SW"); - - break; - - case MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT: - mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_MGMT_CHANGE, - (unsigned long) eqe); - break; - - case MLX4_EVENT_TYPE_RECOVERABLE_ERROR_EVENT: - switch (eqe->subtype) { - case MLX4_RECOVERABLE_ERROR_EVENT_SUBTYPE_BAD_CABLE: - mlx4_warn(dev, "Bad cable detected on port %u\n", - eqe->event.bad_cable.port); - break; - case MLX4_RECOVERABLE_ERROR_EVENT_SUBTYPE_UNSUPPORTED_CABLE: - mlx4_warn(dev, "Unsupported cable detected\n"); - break; - default: - mlx4_dbg(dev, "Unhandled recoverable error event " - "detected: %02x(%02x) on EQ %d at index %u. " - "owner=%x, nent=0x%x, ownership=%s\n", - eqe->type, eqe->subtype, eq->eqn, - eq->cons_index, eqe->owner, eq->nent, - !!(eqe->owner & 0x80) ^ - !!(eq->cons_index & eq->nent) ? "HW" : "SW"); - break; - } - break; - - case MLX4_EVENT_TYPE_EEC_CATAS_ERROR: - case MLX4_EVENT_TYPE_ECC_DETECT: - default: - mlx4_warn(dev, "Unhandled event %02x(%02x) on EQ %d at " - "index %u. owner=%x, nent=0x%x, slave=%x, " - "ownership=%s\n", - eqe->type, eqe->subtype, eq->eqn, - eq->cons_index, eqe->owner, eq->nent, - eqe->slave_id, - !!(eqe->owner & 0x80) ^ - !!(eq->cons_index & eq->nent) ? "HW" : "SW"); - break; - } - - ++eq->cons_index; - eqes_found = 1; - ++set_ci; - - /* - * The HCA will think the queue has overflowed if we - * don't tell it we've been processing events. We - * create our EQs with MLX4_NUM_SPARE_EQE extra - * entries, so we must update our consumer index at - * least that often. - */ - if (unlikely(set_ci >= MLX4_NUM_SPARE_EQE)) { - eq_set_ci(eq, 0); - set_ci = 0; - } - } - - eq_set_ci(eq, 1); - - return eqes_found; -} - -static irqreturn_t mlx4_interrupt(int irq, void *dev_ptr) -{ - struct mlx4_dev *dev = dev_ptr; - struct mlx4_priv *priv = mlx4_priv(dev); - int work = 0; - int i; - - writel(priv->eq_table.clr_mask, priv->eq_table.clr_int); - - for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) - work |= mlx4_eq_int(dev, &priv->eq_table.eq[i]); - - return IRQ_RETVAL(work); -} - -static irqreturn_t mlx4_msi_x_interrupt(int irq, void *eq_ptr) -{ - struct mlx4_eq *eq = eq_ptr; - struct mlx4_dev *dev = eq->dev; - - mlx4_eq_int(dev, eq); - - /* MSI-X vectors always belong to us */ - return IRQ_HANDLED; -} - -int mlx4_MAP_EQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_slave_event_eq_info *event_eq = - priv->mfunc.master.slave_state[slave].event_eq; - u32 in_modifier = vhcr->in_modifier; - u32 eqn = in_modifier & 0x3FF; - u64 in_param = vhcr->in_param; - int err = 0; - int i; - - if (slave == dev->caps.function) - err = mlx4_cmd(dev, in_param, (in_modifier & 0x80000000) | eqn, - 0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_NATIVE); - if (!err) - for (i = 0; i < MLX4_EVENT_TYPES_NUM; ++i) - if (in_param & (1LL << i)) - event_eq[i].eqn = in_modifier >> 31 ? -1 : eqn; - - return err; -} - -static int mlx4_MAP_EQ(struct mlx4_dev *dev, u64 event_mask, int unmap, - int eq_num) -{ - return mlx4_cmd(dev, event_mask, (unmap << 31) | eq_num, - 0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_WRAPPED); -} - -static int mlx4_SW2HW_EQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, - int eq_num) -{ - return mlx4_cmd(dev, mailbox->dma, eq_num, 0, - MLX4_CMD_SW2HW_EQ, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_WRAPPED); -} - -static int mlx4_HW2SW_EQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, - int eq_num) -{ - return mlx4_cmd_box(dev, 0, mailbox->dma, eq_num, - 0, MLX4_CMD_HW2SW_EQ, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_WRAPPED); -} - -static int mlx4_num_eq_uar(struct mlx4_dev *dev) -{ - /* - * Each UAR holds 4 EQ doorbells. To figure out how many UARs - * we need to map, take the difference of highest index and - * the lowest index we'll use and add 1. - */ - return (dev->caps.num_comp_vectors + 1 + dev->caps.reserved_eqs + - dev->caps.comp_pool)/4 - dev->caps.reserved_eqs/4 + 1; -} - -static void __iomem *mlx4_get_eq_uar(struct mlx4_dev *dev, struct mlx4_eq *eq) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int index; - - index = eq->eqn / 4 - dev->caps.reserved_eqs / 4; - - if (!priv->eq_table.uar_map[index]) { - priv->eq_table.uar_map[index] = - ioremap(pci_resource_start(dev->pdev, 2) + - ((eq->eqn / 4) << PAGE_SHIFT), - PAGE_SIZE); - if (!priv->eq_table.uar_map[index]) { - mlx4_err(dev, "Couldn't map EQ doorbell for EQN 0x%06x\n", - eq->eqn); - return NULL; - } - } - - return priv->eq_table.uar_map[index] + 0x800 + 8 * (eq->eqn % 4); -} - -static void mlx4_unmap_uar(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int i; - - for (i = 0; i < mlx4_num_eq_uar(dev); ++i) - if (priv->eq_table.uar_map[i]) { - iounmap(priv->eq_table.uar_map[i]); - priv->eq_table.uar_map[i] = NULL; - } -} - -static int mlx4_create_eq(struct mlx4_dev *dev, int nent, - u8 intr, struct mlx4_eq *eq) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_eq_context *eq_context; - int npages; - u64 *dma_list = NULL; - dma_addr_t t; - u64 mtt_addr; - int err = -ENOMEM; - int i; - - eq->dev = dev; - eq->nent = roundup_pow_of_two(max(nent, 2)); - /* CX3 is capable of extending the CQE\EQE from 32 to 64 bytes */ - npages = PAGE_ALIGN(eq->nent * (MLX4_EQ_ENTRY_SIZE << dev->caps.eqe_factor)) / PAGE_SIZE; - - eq->page_list = kmalloc(npages * sizeof *eq->page_list, - GFP_KERNEL); - if (!eq->page_list) - goto err_out; - - for (i = 0; i < npages; ++i) - eq->page_list[i].buf = NULL; - - dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL); - if (!dma_list) - goto err_out_free; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - goto err_out_free; - eq_context = mailbox->buf; - - for (i = 0; i < npages; ++i) { - eq->page_list[i].buf = dma_alloc_coherent(&dev->pdev->dev, - PAGE_SIZE, &t, GFP_KERNEL); - if (!eq->page_list[i].buf) - goto err_out_free_pages; - - dma_list[i] = t; - eq->page_list[i].map = t; - - memset(eq->page_list[i].buf, 0, PAGE_SIZE); - } - - eq->eqn = mlx4_bitmap_alloc(&priv->eq_table.bitmap); - if (eq->eqn == -1) - goto err_out_free_pages; - - eq->doorbell = mlx4_get_eq_uar(dev, eq); - if (!eq->doorbell) { - err = -ENOMEM; - goto err_out_free_eq; - } - - err = mlx4_mtt_init(dev, npages, PAGE_SHIFT, &eq->mtt); - if (err) - goto err_out_free_eq; - - err = mlx4_write_mtt(dev, &eq->mtt, 0, npages, dma_list); - if (err) - goto err_out_free_mtt; - - memset(eq_context, 0, sizeof *eq_context); - eq_context->flags = cpu_to_be32(MLX4_EQ_STATUS_OK | - MLX4_EQ_STATE_ARMED); - eq_context->log_eq_size = ilog2(eq->nent); - eq_context->intr = intr; - eq_context->log_page_size = PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT; - - mtt_addr = mlx4_mtt_addr(dev, &eq->mtt); - eq_context->mtt_base_addr_h = mtt_addr >> 32; - eq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); - - err = mlx4_SW2HW_EQ(dev, mailbox, eq->eqn); - if (err) { - mlx4_warn(dev, "SW2HW_EQ failed (%d)\n", err); - goto err_out_free_mtt; - } - - kfree(dma_list); - mlx4_free_cmd_mailbox(dev, mailbox); - - eq->cons_index = 0; - - return err; - -err_out_free_mtt: - mlx4_mtt_cleanup(dev, &eq->mtt); - -err_out_free_eq: - mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn, MLX4_USE_RR); - -err_out_free_pages: - for (i = 0; i < npages; ++i) - if (eq->page_list[i].buf) - dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, - eq->page_list[i].buf, - eq->page_list[i].map); - - mlx4_free_cmd_mailbox(dev, mailbox); - -err_out_free: - kfree(eq->page_list); - kfree(dma_list); - -err_out: - return err; -} - -static void mlx4_free_eq(struct mlx4_dev *dev, - struct mlx4_eq *eq) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - struct mlx4_cmd_mailbox *mailbox; - int err; - int i; - /* CX3 is capable of extending the CQE\EQE from 32 to 64 bytes */ - int npages = PAGE_ALIGN((MLX4_EQ_ENTRY_SIZE << dev->caps.eqe_factor) * eq->nent) / PAGE_SIZE; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return; - - err = mlx4_HW2SW_EQ(dev, mailbox, eq->eqn); - if (err) - mlx4_warn(dev, "HW2SW_EQ failed (%d)\n", err); - - if (0) { - mlx4_dbg(dev, "Dumping EQ context %02x:\n", eq->eqn); - for (i = 0; i < sizeof (struct mlx4_eq_context) / 4; ++i) { - if (i % 4 == 0) - pr_cont("[%02x] ", i * 4); - pr_cont(" %08x", be32_to_cpup(mailbox->buf + i * 4)); - if ((i + 1) % 4 == 0) - pr_cont("\n"); - } - } - - mlx4_mtt_cleanup(dev, &eq->mtt); - for (i = 0; i < npages; ++i) - dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, - eq->page_list[i].buf, - eq->page_list[i].map); - - kfree(eq->page_list); - mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn, MLX4_USE_RR); - mlx4_free_cmd_mailbox(dev, mailbox); -} - -static void mlx4_free_irqs(struct mlx4_dev *dev) -{ - struct mlx4_eq_table *eq_table = &mlx4_priv(dev)->eq_table; - struct mlx4_priv *priv = mlx4_priv(dev); - int i, vec; - - if (eq_table->have_irq) - free_irq(dev->pdev->irq, dev); - - for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) - if (eq_table->eq[i].have_irq) { - free_irq(eq_table->eq[i].irq, eq_table->eq + i); - eq_table->eq[i].have_irq = 0; - } - - for (i = 0; i < dev->caps.comp_pool; i++) { - /* - * Freeing the assigned irq's - * all bits should be 0, but we need to validate - */ - if (priv->msix_ctl.pool_bm & 1ULL << i) { - /* NO need protecting*/ - vec = dev->caps.num_comp_vectors + 1 + i; - free_irq(priv->eq_table.eq[vec].irq, - &priv->eq_table.eq[vec]); - } - } - - - kfree(eq_table->irq_names); -} - -static int mlx4_map_clr_int(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - priv->clr_base = ioremap(pci_resource_start(dev->pdev, priv->fw.clr_int_bar) + - priv->fw.clr_int_base, MLX4_CLR_INT_SIZE); - if (!priv->clr_base) { - mlx4_err(dev, "Couldn't map interrupt clear register, aborting.\n"); - return -ENOMEM; - } - - return 0; -} - -static void mlx4_unmap_clr_int(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - iounmap(priv->clr_base); -} - -int mlx4_alloc_eq_table(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - - priv->eq_table.eq = kcalloc(dev->caps.num_eqs - dev->caps.reserved_eqs, - sizeof *priv->eq_table.eq, GFP_KERNEL); - if (!priv->eq_table.eq) - return -ENOMEM; - - return 0; -} - -void mlx4_free_eq_table(struct mlx4_dev *dev) -{ - kfree(mlx4_priv(dev)->eq_table.eq); -} - -int mlx4_init_eq_table(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int err; - int i; - - priv->eq_table.uar_map = kcalloc(mlx4_num_eq_uar(dev), - sizeof *priv->eq_table.uar_map, - GFP_KERNEL); - if (!priv->eq_table.uar_map) { - err = -ENOMEM; - goto err_out_free; - } - - err = mlx4_bitmap_init(&priv->eq_table.bitmap, - roundup_pow_of_two(dev->caps.num_eqs), - dev->caps.num_eqs - 1, - dev->caps.reserved_eqs, - roundup_pow_of_two(dev->caps.num_eqs) - - dev->caps.num_eqs); - if (err) - goto err_out_free; - - for (i = 0; i < mlx4_num_eq_uar(dev); ++i) - priv->eq_table.uar_map[i] = NULL; - - if (!mlx4_is_slave(dev)) { - err = mlx4_map_clr_int(dev); - if (err) - goto err_out_bitmap; - - priv->eq_table.clr_mask = - swab32(1 << (priv->eq_table.inta_pin & 31)); - priv->eq_table.clr_int = priv->clr_base + - (priv->eq_table.inta_pin < 32 ? 4 : 0); - } - - priv->eq_table.irq_names = - kmalloc(MLX4_IRQNAME_SIZE * (dev->caps.num_comp_vectors + 1 + - dev->caps.comp_pool), - GFP_KERNEL); - if (!priv->eq_table.irq_names) { - err = -ENOMEM; - goto err_out_clr_int; - } - - for (i = 0; i < dev->caps.num_comp_vectors; ++i) { - err = mlx4_create_eq(dev, dev->quotas.cq + - MLX4_NUM_SPARE_EQE, - (dev->flags & MLX4_FLAG_MSI_X) ? i : 0, - &priv->eq_table.eq[i]); - if (err) { - --i; - goto err_out_unmap; - } - } - - err = mlx4_create_eq(dev, MLX4_NUM_ASYNC_EQE + MLX4_NUM_SPARE_EQE, - (dev->flags & MLX4_FLAG_MSI_X) ? dev->caps.num_comp_vectors : 0, - &priv->eq_table.eq[dev->caps.num_comp_vectors]); - if (err) - goto err_out_comp; - - /*if additional completion vectors poolsize is 0 this loop will not run*/ - for (i = dev->caps.num_comp_vectors + 1; - i < dev->caps.num_comp_vectors + dev->caps.comp_pool + 1; ++i) { - - err = mlx4_create_eq(dev, dev->quotas.cq + - MLX4_NUM_SPARE_EQE, - (dev->flags & MLX4_FLAG_MSI_X) ? i : 0, - &priv->eq_table.eq[i]); - if (err) { - --i; - goto err_out_unmap; - } - } - - - if (dev->flags & MLX4_FLAG_MSI_X) { - const char *eq_name; - - for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) { - if (i < dev->caps.num_comp_vectors) { - snprintf(priv->eq_table.irq_names + - i * MLX4_IRQNAME_SIZE, - MLX4_IRQNAME_SIZE, - "mlx4-comp-%d@pci:%s", i, - pci_name(dev->pdev)); - } else { - snprintf(priv->eq_table.irq_names + - i * MLX4_IRQNAME_SIZE, - MLX4_IRQNAME_SIZE, - "mlx4-async@pci:%s", - pci_name(dev->pdev)); - } - - eq_name = priv->eq_table.irq_names + - i * MLX4_IRQNAME_SIZE; - err = request_irq(priv->eq_table.eq[i].irq, - mlx4_msi_x_interrupt, 0, eq_name, - priv->eq_table.eq + i); - if (err) - goto err_out_async; - - priv->eq_table.eq[i].have_irq = 1; - } - } else { - snprintf(priv->eq_table.irq_names, - MLX4_IRQNAME_SIZE, - DRV_NAME "@pci:%s", - pci_name(dev->pdev)); - err = request_irq(dev->pdev->irq, mlx4_interrupt, - IRQF_SHARED, priv->eq_table.irq_names, dev); - if (err) - goto err_out_async; - - priv->eq_table.have_irq = 1; - } - - err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, - priv->eq_table.eq[dev->caps.num_comp_vectors].eqn); - if (err) - mlx4_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n", - priv->eq_table.eq[dev->caps.num_comp_vectors].eqn, err); - - for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) - eq_set_ci(&priv->eq_table.eq[i], 1); - - return 0; - -err_out_async: - mlx4_free_eq(dev, &priv->eq_table.eq[dev->caps.num_comp_vectors]); - -err_out_comp: - i = dev->caps.num_comp_vectors - 1; - -err_out_unmap: - while (i >= 0) { - mlx4_free_eq(dev, &priv->eq_table.eq[i]); - --i; - } - mlx4_free_irqs(dev); - -err_out_clr_int: - if (!mlx4_is_slave(dev)) - mlx4_unmap_clr_int(dev); - -err_out_bitmap: - mlx4_unmap_uar(dev); - mlx4_bitmap_cleanup(&priv->eq_table.bitmap); - -err_out_free: - kfree(priv->eq_table.uar_map); - - return err; -} - -void mlx4_cleanup_eq_table(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int i; - - mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 1, - priv->eq_table.eq[dev->caps.num_comp_vectors].eqn); - - mlx4_free_irqs(dev); - - for (i = 0; i < dev->caps.num_comp_vectors + dev->caps.comp_pool + 1; ++i) - mlx4_free_eq(dev, &priv->eq_table.eq[i]); - - if (!mlx4_is_slave(dev)) - mlx4_unmap_clr_int(dev); - - mlx4_unmap_uar(dev); - mlx4_bitmap_cleanup(&priv->eq_table.bitmap); - - kfree(priv->eq_table.uar_map); -} - -/* A test that verifies that we can accept interrupts on all - * the irq vectors of the device. - * Interrupts are checked using the NOP command. - */ -int mlx4_test_interrupts(struct mlx4_dev *dev) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int i; - int err; - - err = mlx4_NOP(dev); - /* When not in MSI_X, there is only one irq to check */ - if (!(dev->flags & MLX4_FLAG_MSI_X) || mlx4_is_slave(dev)) - return err; - - /* A loop over all completion vectors, for each vector we will check - * whether it works by mapping command completions to that vector - * and performing a NOP command - */ - for(i = 0; !err && (i < dev->caps.num_comp_vectors); ++i) { - /* Temporary use polling for command completions */ - mlx4_cmd_use_polling(dev); - - /* Map the new eq to handle all asyncronous events */ - err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, - priv->eq_table.eq[i].eqn); - if (err) { - mlx4_warn(dev, "Failed mapping eq for interrupt test\n"); - mlx4_cmd_use_events(dev); - break; - } - - /* Go back to using events */ - mlx4_cmd_use_events(dev); - err = mlx4_NOP(dev); - } - - /* Return to default */ - mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, - priv->eq_table.eq[dev->caps.num_comp_vectors].eqn); - return err; -} -EXPORT_SYMBOL(mlx4_test_interrupts); - -int mlx4_assign_eq(struct mlx4_dev *dev, char* name, int * vector) -{ - - struct mlx4_priv *priv = mlx4_priv(dev); - int vec = 0, err = 0, i; - - mutex_lock(&priv->msix_ctl.pool_lock); - for (i = 0; !vec && i < dev->caps.comp_pool; i++) { - if (~priv->msix_ctl.pool_bm & 1ULL << i) { - priv->msix_ctl.pool_bm |= 1ULL << i; - vec = dev->caps.num_comp_vectors + 1 + i; - snprintf(priv->eq_table.irq_names + - vec * MLX4_IRQNAME_SIZE, - MLX4_IRQNAME_SIZE, "%s", name); - err = request_irq(priv->eq_table.eq[vec].irq, - mlx4_msi_x_interrupt, 0, - &priv->eq_table.irq_names[vec<<5], - priv->eq_table.eq + vec); - if (err) { - /*zero out bit by fliping it*/ - priv->msix_ctl.pool_bm ^= 1 << i; - vec = 0; - continue; - /*we dont want to break here*/ - } - eq_set_ci(&priv->eq_table.eq[vec], 1); - } - } - mutex_unlock(&priv->msix_ctl.pool_lock); - - if (vec) { - *vector = vec; - } else { - *vector = 0; - err = (i == dev->caps.comp_pool) ? -ENOSPC : err; - } - return err; -} -EXPORT_SYMBOL(mlx4_assign_eq); - -void mlx4_release_eq(struct mlx4_dev *dev, int vec) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - /*bm index*/ - int i = vec - dev->caps.num_comp_vectors - 1; - - if (likely(i >= 0)) { - /*sanity check , making sure were not trying to free irq's - Belonging to a legacy EQ*/ - mutex_lock(&priv->msix_ctl.pool_lock); - if (priv->msix_ctl.pool_bm & 1ULL << i) { - free_irq(priv->eq_table.eq[vec].irq, - &priv->eq_table.eq[vec]); - priv->msix_ctl.pool_bm &= ~(1ULL << i); - } - mutex_unlock(&priv->msix_ctl.pool_lock); - } - -} -EXPORT_SYMBOL(mlx4_release_eq); - Property changes on: stable/11/sys/ofed/drivers/net/mlx4/eq.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/fw.c =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/fw.c (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/fw.c (nonexistent) @@ -1,2056 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#define LINUXKPI_PARAM_PREFIX mlx4_ - -#include -#include -#include -#include - -#include "fw.h" -#include "icm.h" - -enum { - MLX4_COMMAND_INTERFACE_MIN_REV = 2, - MLX4_COMMAND_INTERFACE_MAX_REV = 3, - MLX4_COMMAND_INTERFACE_NEW_PORT_CMDS = 3, -}; - -extern void __buggy_use_of_MLX4_GET(void); -extern void __buggy_use_of_MLX4_PUT(void); - -static u8 enable_qos; -module_param(enable_qos, byte, 0444); -MODULE_PARM_DESC(enable_qos, "Enable Quality of Service support in the HCA (default: off)"); - -#define MLX4_GET(dest, source, offset) \ - do { \ - void *__p = (char *) (source) + (offset); \ - switch (sizeof (dest)) { \ - case 1: (dest) = *(u8 *) __p; break; \ - case 2: (dest) = be16_to_cpup(__p); break; \ - case 4: (dest) = be32_to_cpup(__p); break; \ - case 8: (dest) = be64_to_cpup(__p); break; \ - default: __buggy_use_of_MLX4_GET(); \ - } \ - } while (0) - -#define MLX4_PUT(dest, source, offset) \ - do { \ - void *__d = ((char *) (dest) + (offset)); \ - switch (sizeof(source)) { \ - case 1: *(u8 *) __d = (source); break; \ - case 2: *(__be16 *) __d = cpu_to_be16(source); break; \ - case 4: *(__be32 *) __d = cpu_to_be32(source); break; \ - case 8: *(__be64 *) __d = cpu_to_be64(source); break; \ - default: __buggy_use_of_MLX4_PUT(); \ - } \ - } while (0) - -static void dump_dev_cap_flags(struct mlx4_dev *dev, u64 flags) -{ - static const char *fname[] = { - [ 0] = "RC transport", - [ 1] = "UC transport", - [ 2] = "UD transport", - [ 3] = "XRC transport", - [ 4] = "reliable multicast", - [ 5] = "FCoIB support", - [ 6] = "SRQ support", - [ 7] = "IPoIB checksum offload", - [ 8] = "P_Key violation counter", - [ 9] = "Q_Key violation counter", - [10] = "VMM", - [12] = "DPDP", - [15] = "Big LSO headers", - [16] = "MW support", - [17] = "APM support", - [18] = "Atomic ops support", - [19] = "Raw multicast support", - [20] = "Address vector port checking support", - [21] = "UD multicast support", - [24] = "Demand paging support", - [25] = "Router support", - [30] = "IBoE support", - [32] = "Unicast loopback support", - [34] = "FCS header control", - [38] = "Wake On LAN support", - [40] = "UDP RSS support", - [41] = "Unicast VEP steering support", - [42] = "Multicast VEP steering support", - [44] = "Cross-channel (sync_qp) operations support", - [48] = "Counters support", - [59] = "Port management change event support", - [60] = "eSwitch support", - [61] = "64 byte EQE support", - [62] = "64 byte CQE support", - }; - int i; - - mlx4_dbg(dev, "DEV_CAP flags:\n"); - for (i = 0; i < ARRAY_SIZE(fname); ++i) - if (fname[i] && (flags & (1LL << i))) - mlx4_dbg(dev, " %s\n", fname[i]); -} - -static void dump_dev_cap_flags2(struct mlx4_dev *dev, u64 flags) -{ - static const char * const fname[] = { - [0] = "RSS support", - [1] = "RSS Toeplitz Hash Function support", - [2] = "RSS XOR Hash Function support", - [3] = "Device manage flow steering support", - [4] = "FSM (MAC unti-spoofing) support", - [5] = "VST (control vlan insertion/stripping) support", - [6] = "Dynamic QP updates support", - [7] = "Loopback source checks support", - [8] = "Device managed flow steering IPoIB support", - [9] = "ETS configuration support", - [10] = "ETH backplane autoneg report", - [11] = "Ethernet Flow control statistics support", - [12] = "Recoverable error events support", - [13] = "Time stamping support", - [14] = "Report driver version to FW support" - }; - int i; - - for (i = 0; i < ARRAY_SIZE(fname); ++i) - if (fname[i] && (flags & (1LL << i))) - mlx4_dbg(dev, " %s\n", fname[i]); -} - -int mlx4_MOD_STAT_CFG(struct mlx4_dev *dev, struct mlx4_mod_stat_cfg *cfg) -{ - struct mlx4_cmd_mailbox *mailbox; - u32 *inbox; - int err = 0; - -#define MOD_STAT_CFG_IN_SIZE 0x100 - -#define MOD_STAT_CFG_PG_SZ_M_OFFSET 0x002 -#define MOD_STAT_CFG_PG_SZ_OFFSET 0x003 - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - inbox = mailbox->buf; - - memset(inbox, 0, MOD_STAT_CFG_IN_SIZE); - - MLX4_PUT(inbox, cfg->log_pg_sz, MOD_STAT_CFG_PG_SZ_OFFSET); - MLX4_PUT(inbox, cfg->log_pg_sz_m, MOD_STAT_CFG_PG_SZ_M_OFFSET); - - err = mlx4_cmd(dev, mailbox->dma, 0, 0, MLX4_CMD_MOD_STAT_CFG, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - -int mlx4_QUERY_FUNC(struct mlx4_dev *dev, struct mlx4_func *func, int slave) -{ - struct mlx4_cmd_mailbox *mailbox; - u32 *outbox; - u8 in_modifier; - u8 field; - u16 field16; - int err; - -#define QUERY_FUNC_BUS_OFFSET 0x00 -#define QUERY_FUNC_DEVICE_OFFSET 0x01 -#define QUERY_FUNC_FUNCTION_OFFSET 0x01 -#define QUERY_FUNC_PHYSICAL_FUNCTION_OFFSET 0x03 -#define QUERY_FUNC_RSVD_EQS_OFFSET 0x04 -#define QUERY_FUNC_MAX_EQ_OFFSET 0x06 -#define QUERY_FUNC_RSVD_UARS_OFFSET 0x0b - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - outbox = mailbox->buf; - - in_modifier = slave; - - err = mlx4_cmd_box(dev, 0, mailbox->dma, in_modifier, 0, - MLX4_CMD_QUERY_FUNC, - MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - if (err) - goto out; - - MLX4_GET(field, outbox, QUERY_FUNC_BUS_OFFSET); - func->bus = field & 0xf; - MLX4_GET(field, outbox, QUERY_FUNC_DEVICE_OFFSET); - func->device = field & 0xf1; - MLX4_GET(field, outbox, QUERY_FUNC_FUNCTION_OFFSET); - func->function = field & 0x7; - MLX4_GET(field, outbox, QUERY_FUNC_PHYSICAL_FUNCTION_OFFSET); - func->physical_function = field & 0xf; - MLX4_GET(field16, outbox, QUERY_FUNC_RSVD_EQS_OFFSET); - func->rsvd_eqs = field16 & 0xffff; - MLX4_GET(field16, outbox, QUERY_FUNC_MAX_EQ_OFFSET); - func->max_eq = field16 & 0xffff; - MLX4_GET(field, outbox, QUERY_FUNC_RSVD_UARS_OFFSET); - func->rsvd_uars = field & 0x0f; - - mlx4_dbg(dev, "Bus: %d, Device: %d, Function: %d, Physical function: %d, Max EQs: %d, Reserved EQs: %d, Reserved UARs: %d\n", - func->bus, func->device, func->function, func->physical_function, - func->max_eq, func->rsvd_eqs, func->rsvd_uars); -out: - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - -int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - u8 field, port; - u32 size; - int err = 0; - struct mlx4_func func; - -#define QUERY_FUNC_CAP_FLAGS_OFFSET 0x0 -#define QUERY_FUNC_CAP_NUM_PORTS_OFFSET 0x1 -#define QUERY_FUNC_CAP_PF_BHVR_OFFSET 0x4 -#define QUERY_FUNC_CAP_FMR_OFFSET 0x8 -#define QUERY_FUNC_CAP_QP_QUOTA_OFFSET_DEP 0x10 -#define QUERY_FUNC_CAP_CQ_QUOTA_OFFSET_DEP 0x14 -#define QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET_DEP 0x18 -#define QUERY_FUNC_CAP_MPT_QUOTA_OFFSET_DEP 0x20 -#define QUERY_FUNC_CAP_MTT_QUOTA_OFFSET_DEP 0x24 -#define QUERY_FUNC_CAP_MCG_QUOTA_OFFSET_DEP 0x28 -#define QUERY_FUNC_CAP_MAX_EQ_OFFSET 0x2c -#define QUERY_FUNC_CAP_RESERVED_EQ_OFFSET 0x30 - -#define QUERY_FUNC_CAP_QP_QUOTA_OFFSET 0x50 -#define QUERY_FUNC_CAP_CQ_QUOTA_OFFSET 0x54 -#define QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET 0x58 -#define QUERY_FUNC_CAP_MPT_QUOTA_OFFSET 0x60 -#define QUERY_FUNC_CAP_MTT_QUOTA_OFFSET 0x64 -#define QUERY_FUNC_CAP_MCG_QUOTA_OFFSET 0x68 - -#define QUERY_FUNC_CAP_EXTRA_FLAGS_OFFSET 0x6c - -#define QUERY_FUNC_CAP_FMR_FLAG 0x80 -#define QUERY_FUNC_CAP_FLAG_RDMA 0x40 -#define QUERY_FUNC_CAP_FLAG_ETH 0x80 -#define QUERY_FUNC_CAP_FLAG_QUOTAS 0x10 -#define QUERY_FUNC_CAP_FLAG_VALID_MAILBOX 0x04 - -#define QUERY_FUNC_CAP_EXTRA_FLAGS_BF_QP_ALLOC_FLAG (1UL << 31) - -/* when opcode modifier = 1 */ -#define QUERY_FUNC_CAP_PHYS_PORT_OFFSET 0x3 -#define QUERY_FUNC_CAP_FLAGS0_OFFSET 0x8 -#define QUERY_FUNC_CAP_FLAGS1_OFFSET 0xc -#define QUERY_FUNC_CAP_COUNTER_INDEX_OFFSET 0xd - -#define QUERY_FUNC_CAP_QP0_TUNNEL 0x10 -#define QUERY_FUNC_CAP_QP0_PROXY 0x14 -#define QUERY_FUNC_CAP_QP1_TUNNEL 0x18 -#define QUERY_FUNC_CAP_QP1_PROXY 0x1c - -#define QUERY_FUNC_CAP_ETH_PROPS_FORCE_MAC 0x40 -#define QUERY_FUNC_CAP_ETH_PROPS_FORCE_VLAN 0x80 -#define QUERY_FUNC_CAP_PROPS_DEF_COUNTER 0x20 - -#define QUERY_FUNC_CAP_RDMA_PROPS_FORCE_PHY_WQE_GID 0x80 -#define QUERY_FUNC_CAP_SUPPORTS_NON_POWER_OF_2_NUM_EQS (1 << 31) - - if (vhcr->op_modifier == 1) { - port = vhcr->in_modifier; /* phys-port = logical-port */ - MLX4_PUT(outbox->buf, port, QUERY_FUNC_CAP_PHYS_PORT_OFFSET); - - field = 0; - /* ensure that phy_wqe_gid bit is not set */ - MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FLAGS0_OFFSET); - - /* ensure force vlan and force mac bits are not set - * and that default counter bit is set - */ - field = QUERY_FUNC_CAP_PROPS_DEF_COUNTER; /* def counter */ - MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FLAGS1_OFFSET); - - /* There is always default counter legal or sink counter */ - field = mlx4_get_default_counter_index(dev, slave, vhcr->in_modifier); - MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_COUNTER_INDEX_OFFSET); - - /* size is now the QP number */ - size = dev->phys_caps.base_tunnel_sqpn + 8 * slave + port - 1; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP0_TUNNEL); - - size += 2; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP1_TUNNEL); - - size = dev->phys_caps.base_proxy_sqpn + 8 * slave + port - 1; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP0_PROXY); - - size += 2; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP1_PROXY); - - } else if (vhcr->op_modifier == 0) { - /* enable rdma and ethernet interfaces, and new quota locations */ - field = (QUERY_FUNC_CAP_FLAG_ETH | QUERY_FUNC_CAP_FLAG_RDMA | - QUERY_FUNC_CAP_FLAG_QUOTAS | QUERY_FUNC_CAP_FLAG_VALID_MAILBOX); - MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FLAGS_OFFSET); - - field = dev->caps.num_ports; - MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_NUM_PORTS_OFFSET); - - size = dev->caps.function_caps; /* set PF behaviours */ - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_PF_BHVR_OFFSET); - - field = 0; /* protected FMR support not available as yet */ - MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FMR_OFFSET); - - size = priv->mfunc.master.res_tracker.res_alloc[RES_QP].quota[slave]; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP_QUOTA_OFFSET); - size = dev->caps.num_qps; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP_QUOTA_OFFSET_DEP); - - size = priv->mfunc.master.res_tracker.res_alloc[RES_SRQ].quota[slave]; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET); - size = dev->caps.num_srqs; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET_DEP); - - size = priv->mfunc.master.res_tracker.res_alloc[RES_CQ].quota[slave]; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET); - size = dev->caps.num_cqs; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET_DEP); - - if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) || - mlx4_QUERY_FUNC(dev, &func, slave)) { - size = vhcr->in_modifier & - QUERY_FUNC_CAP_SUPPORTS_NON_POWER_OF_2_NUM_EQS ? - dev->caps.num_eqs : - rounddown_pow_of_two(dev->caps.num_eqs); - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MAX_EQ_OFFSET); - size = dev->caps.reserved_eqs; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_RESERVED_EQ_OFFSET); - } else { - size = vhcr->in_modifier & - QUERY_FUNC_CAP_SUPPORTS_NON_POWER_OF_2_NUM_EQS ? - func.max_eq : - rounddown_pow_of_two(func.max_eq); - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MAX_EQ_OFFSET); - size = func.rsvd_eqs; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_RESERVED_EQ_OFFSET); - } - - size = priv->mfunc.master.res_tracker.res_alloc[RES_MPT].quota[slave]; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET); - size = dev->caps.num_mpts; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET_DEP); - - size = priv->mfunc.master.res_tracker.res_alloc[RES_MTT].quota[slave]; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET); - size = dev->caps.num_mtts; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET_DEP); - - size = dev->caps.num_mgms + dev->caps.num_amgms; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET); - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET_DEP); - - size = QUERY_FUNC_CAP_EXTRA_FLAGS_BF_QP_ALLOC_FLAG; - MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_EXTRA_FLAGS_OFFSET); - } else - err = -EINVAL; - - return err; -} - -int mlx4_QUERY_FUNC_CAP(struct mlx4_dev *dev, u32 gen_or_port, - struct mlx4_func_cap *func_cap) -{ - struct mlx4_cmd_mailbox *mailbox; - u32 *outbox; - u8 field, op_modifier; - u32 size; - int err = 0, quotas = 0; - u32 in_modifier; - - op_modifier = !!gen_or_port; /* 0 = general, 1 = logical port */ - in_modifier = op_modifier ? gen_or_port : - QUERY_FUNC_CAP_SUPPORTS_NON_POWER_OF_2_NUM_EQS; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - err = mlx4_cmd_box(dev, 0, mailbox->dma, in_modifier, op_modifier, - MLX4_CMD_QUERY_FUNC_CAP, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); - if (err) - goto out; - - outbox = mailbox->buf; - - if (!op_modifier) { - MLX4_GET(field, outbox, QUERY_FUNC_CAP_FLAGS_OFFSET); - if (!(field & (QUERY_FUNC_CAP_FLAG_ETH | QUERY_FUNC_CAP_FLAG_RDMA))) { - mlx4_err(dev, "The host supports neither eth nor rdma interfaces\n"); - err = -EPROTONOSUPPORT; - goto out; - } - func_cap->flags = field; - quotas = !!(func_cap->flags & QUERY_FUNC_CAP_FLAG_QUOTAS); - - MLX4_GET(field, outbox, QUERY_FUNC_CAP_NUM_PORTS_OFFSET); - func_cap->num_ports = field; - - MLX4_GET(size, outbox, QUERY_FUNC_CAP_PF_BHVR_OFFSET); - func_cap->pf_context_behaviour = size; - - if (quotas) { - MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP_QUOTA_OFFSET); - func_cap->qp_quota = size & 0xFFFFFF; - - MLX4_GET(size, outbox, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET); - func_cap->srq_quota = size & 0xFFFFFF; - - MLX4_GET(size, outbox, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET); - func_cap->cq_quota = size & 0xFFFFFF; - - MLX4_GET(size, outbox, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET); - func_cap->mpt_quota = size & 0xFFFFFF; - - MLX4_GET(size, outbox, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET); - func_cap->mtt_quota = size & 0xFFFFFF; - - MLX4_GET(size, outbox, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET); - func_cap->mcg_quota = size & 0xFFFFFF; - - } else { - MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP_QUOTA_OFFSET_DEP); - func_cap->qp_quota = size & 0xFFFFFF; - - MLX4_GET(size, outbox, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET_DEP); - func_cap->srq_quota = size & 0xFFFFFF; - - MLX4_GET(size, outbox, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET_DEP); - func_cap->cq_quota = size & 0xFFFFFF; - - MLX4_GET(size, outbox, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET_DEP); - func_cap->mpt_quota = size & 0xFFFFFF; - - MLX4_GET(size, outbox, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET_DEP); - func_cap->mtt_quota = size & 0xFFFFFF; - - MLX4_GET(size, outbox, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET_DEP); - func_cap->mcg_quota = size & 0xFFFFFF; - } - MLX4_GET(size, outbox, QUERY_FUNC_CAP_MAX_EQ_OFFSET); - func_cap->max_eq = size & 0xFFFFFF; - - MLX4_GET(size, outbox, QUERY_FUNC_CAP_RESERVED_EQ_OFFSET); - func_cap->reserved_eq = size & 0xFFFFFF; - - func_cap->extra_flags = 0; - - /* Mailbox data from 0x6c and onward should only be treated if - * QUERY_FUNC_CAP_FLAG_VALID_MAILBOX is set in func_cap->flags - */ - if (func_cap->flags & QUERY_FUNC_CAP_FLAG_VALID_MAILBOX) { - MLX4_GET(size, outbox, QUERY_FUNC_CAP_EXTRA_FLAGS_OFFSET); - if (size & QUERY_FUNC_CAP_EXTRA_FLAGS_BF_QP_ALLOC_FLAG) - func_cap->extra_flags |= MLX4_QUERY_FUNC_FLAGS_BF_RES_QP; - } - - goto out; - } - - /* logical port query */ - if (gen_or_port > dev->caps.num_ports) { - err = -EINVAL; - goto out; - } - - if (dev->caps.port_type[gen_or_port] == MLX4_PORT_TYPE_ETH) { - MLX4_GET(field, outbox, QUERY_FUNC_CAP_FLAGS1_OFFSET); - if (field & QUERY_FUNC_CAP_ETH_PROPS_FORCE_VLAN) { - mlx4_err(dev, "VLAN is enforced on this port\n"); - err = -EPROTONOSUPPORT; - goto out; - } - - if (field & QUERY_FUNC_CAP_ETH_PROPS_FORCE_MAC) { - mlx4_err(dev, "Force mac is enabled on this port\n"); - err = -EPROTONOSUPPORT; - goto out; - } - } else if (dev->caps.port_type[gen_or_port] == MLX4_PORT_TYPE_IB) { - MLX4_GET(field, outbox, QUERY_FUNC_CAP_FLAGS0_OFFSET); - if (field & QUERY_FUNC_CAP_RDMA_PROPS_FORCE_PHY_WQE_GID) { - mlx4_err(dev, "phy_wqe_gid is " - "enforced on this ib port\n"); - err = -EPROTONOSUPPORT; - goto out; - } - } - - MLX4_GET(field, outbox, QUERY_FUNC_CAP_PHYS_PORT_OFFSET); - func_cap->physical_port = field; - if (func_cap->physical_port != gen_or_port) { - err = -ENOSYS; - goto out; - } - - MLX4_GET(field, outbox, QUERY_FUNC_CAP_FLAGS1_OFFSET); - if (field & QUERY_FUNC_CAP_PROPS_DEF_COUNTER) { - MLX4_GET(field, outbox, QUERY_FUNC_CAP_COUNTER_INDEX_OFFSET); - func_cap->def_counter_index = field; - } else { - func_cap->def_counter_index = MLX4_SINK_COUNTER_INDEX; - } - - MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP0_TUNNEL); - func_cap->qp0_tunnel_qpn = size & 0xFFFFFF; - - MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP0_PROXY); - func_cap->qp0_proxy_qpn = size & 0xFFFFFF; - - MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP1_TUNNEL); - func_cap->qp1_tunnel_qpn = size & 0xFFFFFF; - - MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP1_PROXY); - func_cap->qp1_proxy_qpn = size & 0xFFFFFF; - - /* All other resources are allocated by the master, but we still report - * 'num' and 'reserved' capabilities as follows: - * - num remains the maximum resource index - * - 'num - reserved' is the total available objects of a resource, but - * resource indices may be less than 'reserved' - * TODO: set per-resource quotas */ - -out: - mlx4_free_cmd_mailbox(dev, mailbox); - - return err; -} - -int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) -{ - struct mlx4_cmd_mailbox *mailbox; - u32 *outbox; - u8 field; - u32 field32, flags, ext_flags; - u16 size; - u16 stat_rate; - int err; - int i; - -#define QUERY_DEV_CAP_OUT_SIZE 0x100 -#define QUERY_DEV_CAP_MAX_SRQ_SZ_OFFSET 0x10 -#define QUERY_DEV_CAP_MAX_QP_SZ_OFFSET 0x11 -#define QUERY_DEV_CAP_RSVD_QP_OFFSET 0x12 -#define QUERY_DEV_CAP_MAX_QP_OFFSET 0x13 -#define QUERY_DEV_CAP_RSVD_SRQ_OFFSET 0x14 -#define QUERY_DEV_CAP_MAX_SRQ_OFFSET 0x15 -#define QUERY_DEV_CAP_RSVD_EEC_OFFSET 0x16 -#define QUERY_DEV_CAP_MAX_EEC_OFFSET 0x17 -#define QUERY_DEV_CAP_MAX_CQ_SZ_OFFSET 0x19 -#define QUERY_DEV_CAP_RSVD_CQ_OFFSET 0x1a -#define QUERY_DEV_CAP_MAX_CQ_OFFSET 0x1b -#define QUERY_DEV_CAP_MAX_MPT_OFFSET 0x1d -#define QUERY_DEV_CAP_RSVD_EQ_OFFSET 0x1e -#define QUERY_DEV_CAP_MAX_EQ_OFFSET 0x1f -#define QUERY_DEV_CAP_RSVD_MTT_OFFSET 0x20 -#define QUERY_DEV_CAP_MAX_MRW_SZ_OFFSET 0x21 -#define QUERY_DEV_CAP_RSVD_MRW_OFFSET 0x22 -#define QUERY_DEV_CAP_MAX_MTT_SEG_OFFSET 0x23 -#define QUERY_DEV_CAP_NUM_SYS_EQ_OFFSET 0x26 -#define QUERY_DEV_CAP_MAX_AV_OFFSET 0x27 -#define QUERY_DEV_CAP_MAX_REQ_QP_OFFSET 0x29 -#define QUERY_DEV_CAP_MAX_RES_QP_OFFSET 0x2b -#define QUERY_DEV_CAP_MAX_GSO_OFFSET 0x2d -#define QUERY_DEV_CAP_RSS_OFFSET 0x2e -#define QUERY_DEV_CAP_MAX_RDMA_OFFSET 0x2f -#define QUERY_DEV_CAP_RSZ_SRQ_OFFSET 0x33 -#define QUERY_DEV_CAP_ACK_DELAY_OFFSET 0x35 -#define QUERY_DEV_CAP_MTU_WIDTH_OFFSET 0x36 -#define QUERY_DEV_CAP_VL_PORT_OFFSET 0x37 -#define QUERY_DEV_CAP_MAX_MSG_SZ_OFFSET 0x38 -#define QUERY_DEV_CAP_MAX_GID_OFFSET 0x3b -#define QUERY_DEV_CAP_RATE_SUPPORT_OFFSET 0x3c -#define QUERY_DEV_CAP_CQ_TS_SUPPORT_OFFSET 0x3e -#define QUERY_DEV_CAP_MAX_PKEY_OFFSET 0x3f -#define QUERY_DEV_CAP_EXT_FLAGS_OFFSET 0x40 -#define QUERY_DEV_CAP_SYNC_QP_OFFSET 0x42 -#define QUERY_DEV_CAP_FLAGS_OFFSET 0x44 -#define QUERY_DEV_CAP_RSVD_UAR_OFFSET 0x48 -#define QUERY_DEV_CAP_UAR_SZ_OFFSET 0x49 -#define QUERY_DEV_CAP_PAGE_SZ_OFFSET 0x4b -#define QUERY_DEV_CAP_BF_OFFSET 0x4c -#define QUERY_DEV_CAP_LOG_BF_REG_SZ_OFFSET 0x4d -#define QUERY_DEV_CAP_LOG_MAX_BF_REGS_PER_PAGE_OFFSET 0x4e -#define QUERY_DEV_CAP_LOG_MAX_BF_PAGES_OFFSET 0x4f -#define QUERY_DEV_CAP_MAX_SG_SQ_OFFSET 0x51 -#define QUERY_DEV_CAP_MAX_DESC_SZ_SQ_OFFSET 0x52 -#define QUERY_DEV_CAP_MAX_SG_RQ_OFFSET 0x55 -#define QUERY_DEV_CAP_MAX_DESC_SZ_RQ_OFFSET 0x56 -#define QUERY_DEV_CAP_MAX_QP_MCG_OFFSET 0x61 -#define QUERY_DEV_CAP_RSVD_MCG_OFFSET 0x62 -#define QUERY_DEV_CAP_MAX_MCG_OFFSET 0x63 -#define QUERY_DEV_CAP_RSVD_PD_OFFSET 0x64 -#define QUERY_DEV_CAP_MAX_PD_OFFSET 0x65 -#define QUERY_DEV_CAP_RSVD_XRC_OFFSET 0x66 -#define QUERY_DEV_CAP_MAX_XRC_OFFSET 0x67 -#define QUERY_DEV_CAP_MAX_BASIC_COUNTERS_OFFSET 0x68 -#define QUERY_DEV_CAP_MAX_EXTENDED_COUNTERS_OFFSET 0x6c -#define QUERY_DEV_CAP_PORT_FLOWSTATS_COUNTERS_OFFSET 0x70 -#define QUERY_DEV_CAP_FLOW_STEERING_RANGE_EN_OFFSET 0x76 -#define QUERY_DEV_CAP_EXT_2_FLAGS_OFFSET 0x70 -#define QUERY_DEV_CAP_FLOW_STEERING_IPOIB_OFFSET 0x74 -#define QUERY_DEV_CAP_FLOW_STEERING_MAX_QP_OFFSET 0x77 -#define QUERY_DEV_CAP_RDMARC_ENTRY_SZ_OFFSET 0x80 -#define QUERY_DEV_CAP_QPC_ENTRY_SZ_OFFSET 0x82 -#define QUERY_DEV_CAP_AUX_ENTRY_SZ_OFFSET 0x84 -#define QUERY_DEV_CAP_ALTC_ENTRY_SZ_OFFSET 0x86 -#define QUERY_DEV_CAP_EQC_ENTRY_SZ_OFFSET 0x88 -#define QUERY_DEV_CAP_CQC_ENTRY_SZ_OFFSET 0x8a -#define QUERY_DEV_CAP_SRQ_ENTRY_SZ_OFFSET 0x8c -#define QUERY_DEV_CAP_C_MPT_ENTRY_SZ_OFFSET 0x8e -#define QUERY_DEV_CAP_MTT_ENTRY_SZ_OFFSET 0x90 -#define QUERY_DEV_CAP_D_MPT_ENTRY_SZ_OFFSET 0x92 -#define QUERY_DEV_CAP_BMME_FLAGS_OFFSET 0x94 -#define QUERY_DEV_CAP_RSVD_LKEY_OFFSET 0x98 -#define QUERY_DEV_CAP_ETS_CFG_OFFSET 0x9c -#define QUERY_DEV_CAP_MAX_ICM_SZ_OFFSET 0xa0 - - dev_cap->flags2 = 0; - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - outbox = mailbox->buf; - - err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, MLX4_CMD_QUERY_DEV_CAP, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); - if (err) - goto out; - - MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_QP_OFFSET); - dev_cap->reserved_qps = 1 << (field & 0xf); - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_QP_OFFSET); - dev_cap->max_qps = 1 << (field & 0x1f); - MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_SRQ_OFFSET); - dev_cap->reserved_srqs = 1 << (field >> 4); - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_SRQ_OFFSET); - dev_cap->max_srqs = 1 << (field & 0x1f); - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_CQ_SZ_OFFSET); - dev_cap->max_cq_sz = 1 << field; - MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_CQ_OFFSET); - dev_cap->reserved_cqs = 1 << (field & 0xf); - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_CQ_OFFSET); - dev_cap->max_cqs = 1 << (field & 0x1f); - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_MPT_OFFSET); - dev_cap->max_mpts = 1 << (field & 0x3f); - MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_EQ_OFFSET); - dev_cap->reserved_eqs = field & 0xf; - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_EQ_OFFSET); - dev_cap->max_eqs = 1 << (field & 0xf); - MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_MTT_OFFSET); - dev_cap->reserved_mtts = 1 << (field >> 4); - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_MRW_SZ_OFFSET); - dev_cap->max_mrw_sz = 1 << field; - MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_MRW_OFFSET); - dev_cap->reserved_mrws = 1 << (field & 0xf); - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_MTT_SEG_OFFSET); - dev_cap->max_mtt_seg = 1 << (field & 0x3f); - MLX4_GET(size, outbox, QUERY_DEV_CAP_NUM_SYS_EQ_OFFSET); - dev_cap->num_sys_eqs = size & 0xfff; - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_REQ_QP_OFFSET); - dev_cap->max_requester_per_qp = 1 << (field & 0x3f); - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_RES_QP_OFFSET); - dev_cap->max_responder_per_qp = 1 << (field & 0x3f); - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_GSO_OFFSET); - field &= 0x1f; - if (!field) - dev_cap->max_gso_sz = 0; - else - dev_cap->max_gso_sz = 1 << field; - - MLX4_GET(field, outbox, QUERY_DEV_CAP_RSS_OFFSET); - if (field & 0x20) - dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_RSS_XOR; - if (field & 0x10) - dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_RSS_TOP; - field &= 0xf; - if (field) { - dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_RSS; - dev_cap->max_rss_tbl_sz = 1 << field; - } else - dev_cap->max_rss_tbl_sz = 0; - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_RDMA_OFFSET); - dev_cap->max_rdma_global = 1 << (field & 0x3f); - MLX4_GET(field, outbox, QUERY_DEV_CAP_ACK_DELAY_OFFSET); - dev_cap->local_ca_ack_delay = field & 0x1f; - MLX4_GET(field, outbox, QUERY_DEV_CAP_VL_PORT_OFFSET); - dev_cap->num_ports = field & 0xf; - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_MSG_SZ_OFFSET); - dev_cap->max_msg_sz = 1 << (field & 0x1f); - MLX4_GET(field, outbox, QUERY_DEV_CAP_PORT_FLOWSTATS_COUNTERS_OFFSET); - if (field & 0x10) - dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN; - MLX4_GET(field, outbox, QUERY_DEV_CAP_FLOW_STEERING_RANGE_EN_OFFSET); - if (field & 0x80) - dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_FS_EN; - MLX4_GET(field, outbox, QUERY_DEV_CAP_FLOW_STEERING_IPOIB_OFFSET); - if (field & 0x80) - dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_DMFS_IPOIB; - dev_cap->fs_log_max_ucast_qp_range_size = field & 0x1f; - MLX4_GET(field, outbox, QUERY_DEV_CAP_FLOW_STEERING_MAX_QP_OFFSET); - dev_cap->fs_max_num_qp_per_entry = field; - MLX4_GET(stat_rate, outbox, QUERY_DEV_CAP_RATE_SUPPORT_OFFSET); - dev_cap->stat_rate_support = stat_rate; - MLX4_GET(field, outbox, QUERY_DEV_CAP_CQ_TS_SUPPORT_OFFSET); - if (field & 0x80) - dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_TS; - MLX4_GET(ext_flags, outbox, QUERY_DEV_CAP_EXT_FLAGS_OFFSET); - MLX4_GET(flags, outbox, QUERY_DEV_CAP_FLAGS_OFFSET); - dev_cap->flags = flags | (u64)ext_flags << 32; - MLX4_GET(field, outbox, QUERY_DEV_CAP_SYNC_QP_OFFSET); - dev_cap->sync_qp = field & 0x10; - MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_UAR_OFFSET); - dev_cap->reserved_uars = field >> 4; - MLX4_GET(field, outbox, QUERY_DEV_CAP_UAR_SZ_OFFSET); - dev_cap->uar_size = 1 << ((field & 0x3f) + 20); - MLX4_GET(field, outbox, QUERY_DEV_CAP_PAGE_SZ_OFFSET); - dev_cap->min_page_sz = 1 << field; - - MLX4_GET(field, outbox, QUERY_DEV_CAP_BF_OFFSET); - if (field & 0x80) { - MLX4_GET(field, outbox, QUERY_DEV_CAP_LOG_BF_REG_SZ_OFFSET); - dev_cap->bf_reg_size = 1 << (field & 0x1f); - MLX4_GET(field, outbox, QUERY_DEV_CAP_LOG_MAX_BF_REGS_PER_PAGE_OFFSET); - if ((1 << (field & 0x3f)) > (PAGE_SIZE / dev_cap->bf_reg_size)) - field = 3; - dev_cap->bf_regs_per_page = 1 << (field & 0x3f); - mlx4_dbg(dev, "BlueFlame available (reg size %d, regs/page %d)\n", - dev_cap->bf_reg_size, dev_cap->bf_regs_per_page); - } else { - dev_cap->bf_reg_size = 0; - mlx4_dbg(dev, "BlueFlame not available\n"); - } - - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_SG_SQ_OFFSET); - dev_cap->max_sq_sg = field; - MLX4_GET(size, outbox, QUERY_DEV_CAP_MAX_DESC_SZ_SQ_OFFSET); - dev_cap->max_sq_desc_sz = size; - - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_QP_MCG_OFFSET); - dev_cap->max_qp_per_mcg = 1 << field; - MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_MCG_OFFSET); - dev_cap->reserved_mgms = field & 0xf; - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_MCG_OFFSET); - dev_cap->max_mcgs = 1 << field; - MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_PD_OFFSET); - dev_cap->reserved_pds = field >> 4; - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_PD_OFFSET); - dev_cap->max_pds = 1 << (field & 0x3f); - MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_XRC_OFFSET); - dev_cap->reserved_xrcds = field >> 4; - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_XRC_OFFSET); - dev_cap->max_xrcds = 1 << (field & 0x1f); - - MLX4_GET(size, outbox, QUERY_DEV_CAP_RDMARC_ENTRY_SZ_OFFSET); - dev_cap->rdmarc_entry_sz = size; - MLX4_GET(size, outbox, QUERY_DEV_CAP_QPC_ENTRY_SZ_OFFSET); - dev_cap->qpc_entry_sz = size; - MLX4_GET(size, outbox, QUERY_DEV_CAP_AUX_ENTRY_SZ_OFFSET); - dev_cap->aux_entry_sz = size; - MLX4_GET(size, outbox, QUERY_DEV_CAP_ALTC_ENTRY_SZ_OFFSET); - dev_cap->altc_entry_sz = size; - MLX4_GET(size, outbox, QUERY_DEV_CAP_EQC_ENTRY_SZ_OFFSET); - dev_cap->eqc_entry_sz = size; - MLX4_GET(size, outbox, QUERY_DEV_CAP_CQC_ENTRY_SZ_OFFSET); - dev_cap->cqc_entry_sz = size; - MLX4_GET(size, outbox, QUERY_DEV_CAP_SRQ_ENTRY_SZ_OFFSET); - dev_cap->srq_entry_sz = size; - MLX4_GET(size, outbox, QUERY_DEV_CAP_C_MPT_ENTRY_SZ_OFFSET); - dev_cap->cmpt_entry_sz = size; - MLX4_GET(size, outbox, QUERY_DEV_CAP_MTT_ENTRY_SZ_OFFSET); - dev_cap->mtt_entry_sz = size; - MLX4_GET(size, outbox, QUERY_DEV_CAP_D_MPT_ENTRY_SZ_OFFSET); - dev_cap->dmpt_entry_sz = size; - - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_SRQ_SZ_OFFSET); - dev_cap->max_srq_sz = 1 << field; - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_QP_SZ_OFFSET); - dev_cap->max_qp_sz = 1 << field; - MLX4_GET(field, outbox, QUERY_DEV_CAP_RSZ_SRQ_OFFSET); - dev_cap->resize_srq = field & 1; - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_SG_RQ_OFFSET); - dev_cap->max_rq_sg = field; - MLX4_GET(size, outbox, QUERY_DEV_CAP_MAX_DESC_SZ_RQ_OFFSET); - dev_cap->max_rq_desc_sz = size; - - MLX4_GET(dev_cap->bmme_flags, outbox, - QUERY_DEV_CAP_BMME_FLAGS_OFFSET); - MLX4_GET(dev_cap->reserved_lkey, outbox, - QUERY_DEV_CAP_RSVD_LKEY_OFFSET); - MLX4_GET(field32, outbox, QUERY_DEV_CAP_ETS_CFG_OFFSET); - if (field32 & (1 << 0)) - dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_ETH_BACKPL_AN_REP; - if (field32 & (1 << 7)) - dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_RECOVERABLE_ERROR_EVENT; - if (field32 & (1 << 8)) - dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_DRIVER_VERSION_TO_FW; - if (field32 & (1 << 13)) - dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_ETS_CFG; - - MLX4_GET(dev_cap->max_icm_sz, outbox, - QUERY_DEV_CAP_MAX_ICM_SZ_OFFSET); - if (dev_cap->flags & MLX4_DEV_CAP_FLAG_COUNTERS) - MLX4_GET(dev_cap->max_basic_counters, outbox, - QUERY_DEV_CAP_MAX_BASIC_COUNTERS_OFFSET); - /* FW reports 256 however real value is 255 */ - dev_cap->max_basic_counters = min_t(u32, dev_cap->max_basic_counters, 255); - if (dev_cap->flags & MLX4_DEV_CAP_FLAG_COUNTERS_EXT) - MLX4_GET(dev_cap->max_extended_counters, outbox, - QUERY_DEV_CAP_MAX_EXTENDED_COUNTERS_OFFSET); - - MLX4_GET(field32, outbox, QUERY_DEV_CAP_EXT_2_FLAGS_OFFSET); - if (field32 & (1 << 16)) - dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_UPDATE_QP; - if (field32 & (1 << 19)) - dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_LB_SRC_CHK; - if (field32 & (1 << 20)) - dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_FSM; - if (field32 & (1 << 26)) - dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_VLAN_CONTROL; - - if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { - for (i = 1; i <= dev_cap->num_ports; ++i) { - MLX4_GET(field, outbox, QUERY_DEV_CAP_VL_PORT_OFFSET); - dev_cap->max_vl[i] = field >> 4; - MLX4_GET(field, outbox, QUERY_DEV_CAP_MTU_WIDTH_OFFSET); - dev_cap->ib_mtu[i] = field >> 4; - dev_cap->max_port_width[i] = field & 0xf; - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_GID_OFFSET); - dev_cap->max_gids[i] = 1 << (field & 0xf); - MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_PKEY_OFFSET); - dev_cap->max_pkeys[i] = 1 << (field & 0xf); - } - } else { -#define QUERY_PORT_SUPPORTED_TYPE_OFFSET 0x00 -#define QUERY_PORT_MTU_OFFSET 0x01 -#define QUERY_PORT_ETH_MTU_OFFSET 0x02 -#define QUERY_PORT_WIDTH_OFFSET 0x06 -#define QUERY_PORT_MAX_GID_PKEY_OFFSET 0x07 -#define QUERY_PORT_MAX_MACVLAN_OFFSET 0x0a -#define QUERY_PORT_MAX_VL_OFFSET 0x0b -#define QUERY_PORT_MAC_OFFSET 0x10 -#define QUERY_PORT_TRANS_VENDOR_OFFSET 0x18 -#define QUERY_PORT_WAVELENGTH_OFFSET 0x1c -#define QUERY_PORT_TRANS_CODE_OFFSET 0x20 - - for (i = 1; i <= dev_cap->num_ports; ++i) { - err = mlx4_cmd_box(dev, 0, mailbox->dma, i, 0, MLX4_CMD_QUERY_PORT, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); - if (err) - goto out; - - MLX4_GET(field, outbox, QUERY_PORT_SUPPORTED_TYPE_OFFSET); - dev_cap->supported_port_types[i] = field & 3; - dev_cap->suggested_type[i] = (field >> 3) & 1; - dev_cap->default_sense[i] = (field >> 4) & 1; - MLX4_GET(field, outbox, QUERY_PORT_MTU_OFFSET); - dev_cap->ib_mtu[i] = field & 0xf; - MLX4_GET(field, outbox, QUERY_PORT_WIDTH_OFFSET); - dev_cap->max_port_width[i] = field & 0xf; - MLX4_GET(field, outbox, QUERY_PORT_MAX_GID_PKEY_OFFSET); - dev_cap->max_gids[i] = 1 << (field >> 4); - dev_cap->max_pkeys[i] = 1 << (field & 0xf); - MLX4_GET(field, outbox, QUERY_PORT_MAX_VL_OFFSET); - dev_cap->max_vl[i] = field & 0xf; - MLX4_GET(field, outbox, QUERY_PORT_MAX_MACVLAN_OFFSET); - dev_cap->log_max_macs[i] = field & 0xf; - dev_cap->log_max_vlans[i] = field >> 4; - MLX4_GET(dev_cap->eth_mtu[i], outbox, QUERY_PORT_ETH_MTU_OFFSET); - MLX4_GET(dev_cap->def_mac[i], outbox, QUERY_PORT_MAC_OFFSET); - MLX4_GET(field32, outbox, QUERY_PORT_TRANS_VENDOR_OFFSET); - dev_cap->trans_type[i] = field32 >> 24; - dev_cap->vendor_oui[i] = field32 & 0xffffff; - MLX4_GET(dev_cap->wavelength[i], outbox, QUERY_PORT_WAVELENGTH_OFFSET); - MLX4_GET(dev_cap->trans_code[i], outbox, QUERY_PORT_TRANS_CODE_OFFSET); - } - } - - mlx4_dbg(dev, "Base MM extensions: flags %08x, rsvd L_Key %08x\n", - dev_cap->bmme_flags, dev_cap->reserved_lkey); - - /* - * Each UAR has 4 EQ doorbells; so if a UAR is reserved, then - * we can't use any EQs whose doorbell falls on that page, - * even if the EQ itself isn't reserved. - */ - if (dev_cap->num_sys_eqs == 0) - dev_cap->reserved_eqs = max(dev_cap->reserved_uars * 4, - dev_cap->reserved_eqs); - else - dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_SYS_EQS; - - mlx4_dbg(dev, "Max ICM size %lld MB\n", - (unsigned long long) dev_cap->max_icm_sz >> 20); - mlx4_dbg(dev, "Max QPs: %d, reserved QPs: %d, entry size: %d\n", - dev_cap->max_qps, dev_cap->reserved_qps, dev_cap->qpc_entry_sz); - mlx4_dbg(dev, "Max SRQs: %d, reserved SRQs: %d, entry size: %d\n", - dev_cap->max_srqs, dev_cap->reserved_srqs, dev_cap->srq_entry_sz); - mlx4_dbg(dev, "Max CQs: %d, reserved CQs: %d, entry size: %d\n", - dev_cap->max_cqs, dev_cap->reserved_cqs, dev_cap->cqc_entry_sz); - mlx4_dbg(dev, "Num sys EQs: %d, max EQs: %d, reserved EQs: %d, entry size: %d\n", - dev_cap->num_sys_eqs, dev_cap->max_eqs, dev_cap->reserved_eqs, - dev_cap->eqc_entry_sz); - mlx4_dbg(dev, "reserved MPTs: %d, reserved MTTs: %d\n", - dev_cap->reserved_mrws, dev_cap->reserved_mtts); - mlx4_dbg(dev, "Max PDs: %d, reserved PDs: %d, reserved UARs: %d\n", - dev_cap->max_pds, dev_cap->reserved_pds, dev_cap->reserved_uars); - mlx4_dbg(dev, "Max QP/MCG: %d, reserved MGMs: %d\n", - dev_cap->max_pds, dev_cap->reserved_mgms); - mlx4_dbg(dev, "Max CQEs: %d, max WQEs: %d, max SRQ WQEs: %d\n", - dev_cap->max_cq_sz, dev_cap->max_qp_sz, dev_cap->max_srq_sz); - mlx4_dbg(dev, "Local CA ACK delay: %d, max MTU: %d, port width cap: %d\n", - dev_cap->local_ca_ack_delay, 128 << dev_cap->ib_mtu[1], - dev_cap->max_port_width[1]); - mlx4_dbg(dev, "Max SQ desc size: %d, max SQ S/G: %d\n", - dev_cap->max_sq_desc_sz, dev_cap->max_sq_sg); - mlx4_dbg(dev, "Max RQ desc size: %d, max RQ S/G: %d\n", - dev_cap->max_rq_desc_sz, dev_cap->max_rq_sg); - mlx4_dbg(dev, "Max GSO size: %d\n", dev_cap->max_gso_sz); - mlx4_dbg(dev, "Max basic counters: %d\n", dev_cap->max_basic_counters); - mlx4_dbg(dev, "Max extended counters: %d\n", dev_cap->max_extended_counters); - mlx4_dbg(dev, "Max RSS Table size: %d\n", dev_cap->max_rss_tbl_sz); - - dump_dev_cap_flags(dev, dev_cap->flags); - dump_dev_cap_flags2(dev, dev_cap->flags2); - -out: - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - -int mlx4_QUERY_DEV_CAP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - u64 flags; - int err = 0; - u8 field; - - err = mlx4_cmd_box(dev, 0, outbox->dma, 0, 0, MLX4_CMD_QUERY_DEV_CAP, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); - if (err) - return err; - - /* add port mng change event capability unconditionally to slaves */ - MLX4_GET(flags, outbox->buf, QUERY_DEV_CAP_EXT_FLAGS_OFFSET); - flags |= MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV; - MLX4_PUT(outbox->buf, flags, QUERY_DEV_CAP_EXT_FLAGS_OFFSET); - - /* For guests, report Blueflame disabled */ - MLX4_GET(field, outbox->buf, QUERY_DEV_CAP_BF_OFFSET); - field &= 0x7f; - MLX4_PUT(outbox->buf, field, QUERY_DEV_CAP_BF_OFFSET); - - /* turn off device-managed steering capability if not enabled */ - if (dev->caps.steering_mode != MLX4_STEERING_MODE_DEVICE_MANAGED) { - MLX4_GET(field, outbox->buf, - QUERY_DEV_CAP_FLOW_STEERING_RANGE_EN_OFFSET); - field &= 0x7f; - MLX4_PUT(outbox->buf, field, - QUERY_DEV_CAP_FLOW_STEERING_RANGE_EN_OFFSET); - } - return 0; -} - -int mlx4_QUERY_PORT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - u64 def_mac; - u8 port_type; - u16 short_field; - int err; - int admin_link_state; - -#define MLX4_VF_PORT_NO_LINK_SENSE_MASK 0xE0 -#define MLX4_PORT_LINK_UP_MASK 0x80 -#define QUERY_PORT_CUR_MAX_PKEY_OFFSET 0x0c -#define QUERY_PORT_CUR_MAX_GID_OFFSET 0x0e - - err = mlx4_cmd_box(dev, 0, outbox->dma, vhcr->in_modifier, 0, - MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_NATIVE); - - if (!err && dev->caps.function != slave) { - /* set slave default_mac address to be zero MAC */ - def_mac = priv->mfunc.master.vf_oper[slave].vport[vhcr->in_modifier].state.mac; - MLX4_PUT(outbox->buf, def_mac, QUERY_PORT_MAC_OFFSET); - - /* get port type - currently only eth is enabled */ - MLX4_GET(port_type, outbox->buf, - QUERY_PORT_SUPPORTED_TYPE_OFFSET); - - /* No link sensing allowed */ - port_type &= MLX4_VF_PORT_NO_LINK_SENSE_MASK; - /* set port type to currently operating port type */ - port_type |= (dev->caps.port_type[vhcr->in_modifier] & 0x3); - - admin_link_state = priv->mfunc.master.vf_oper[slave].vport[vhcr->in_modifier].state.link_state; - if (IFLA_VF_LINK_STATE_ENABLE == admin_link_state) - port_type |= MLX4_PORT_LINK_UP_MASK; - else if (IFLA_VF_LINK_STATE_DISABLE == admin_link_state) - port_type &= ~MLX4_PORT_LINK_UP_MASK; - - MLX4_PUT(outbox->buf, port_type, - QUERY_PORT_SUPPORTED_TYPE_OFFSET); - - if (dev->caps.port_type[vhcr->in_modifier] == MLX4_PORT_TYPE_ETH) - short_field = mlx4_get_slave_num_gids(dev, slave); - else - short_field = 1; /* slave max gids */ - MLX4_PUT(outbox->buf, short_field, - QUERY_PORT_CUR_MAX_GID_OFFSET); - - short_field = dev->caps.pkey_table_len[vhcr->in_modifier]; - MLX4_PUT(outbox->buf, short_field, - QUERY_PORT_CUR_MAX_PKEY_OFFSET); - } - - return err; -} - -int mlx4_get_slave_pkey_gid_tbl_len(struct mlx4_dev *dev, u8 port, - int *gid_tbl_len, int *pkey_tbl_len) -{ - struct mlx4_cmd_mailbox *mailbox; - u32 *outbox; - u16 field; - int err; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - err = mlx4_cmd_box(dev, 0, mailbox->dma, port, 0, - MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_WRAPPED); - if (err) - goto out; - - outbox = mailbox->buf; - - MLX4_GET(field, outbox, QUERY_PORT_CUR_MAX_GID_OFFSET); - *gid_tbl_len = field; - - MLX4_GET(field, outbox, QUERY_PORT_CUR_MAX_PKEY_OFFSET); - *pkey_tbl_len = field; - -out: - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} -EXPORT_SYMBOL(mlx4_get_slave_pkey_gid_tbl_len); - -int mlx4_map_cmd(struct mlx4_dev *dev, u16 op, struct mlx4_icm *icm, u64 virt) -{ - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_icm_iter iter; - __be64 *pages; - int lg; - int nent = 0; - int i; - int err = 0; - int ts = 0, tc = 0; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - memset(mailbox->buf, 0, MLX4_MAILBOX_SIZE); - pages = mailbox->buf; - - for (mlx4_icm_first(icm, &iter); - !mlx4_icm_last(&iter); - mlx4_icm_next(&iter)) { - /* - * We have to pass pages that are aligned to their - * size, so find the least significant 1 in the - * address or size and use that as our log2 size. - */ - lg = ffs(mlx4_icm_addr(&iter) | mlx4_icm_size(&iter)) - 1; - if (lg < MLX4_ICM_PAGE_SHIFT) { - mlx4_warn(dev, "Got FW area not aligned to %d (%llx/%lx).\n", - MLX4_ICM_PAGE_SIZE, - (unsigned long long) mlx4_icm_addr(&iter), - mlx4_icm_size(&iter)); - err = -EINVAL; - goto out; - } - - for (i = 0; i < mlx4_icm_size(&iter) >> lg; ++i) { - if (virt != -1) { - pages[nent * 2] = cpu_to_be64(virt); - virt += 1 << lg; - } - - pages[nent * 2 + 1] = - cpu_to_be64((mlx4_icm_addr(&iter) + (i << lg)) | - (lg - MLX4_ICM_PAGE_SHIFT)); - ts += 1 << (lg - 10); - ++tc; - - if (++nent == MLX4_MAILBOX_SIZE / 16) { - err = mlx4_cmd(dev, mailbox->dma, nent, 0, op, - MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_NATIVE); - if (err) - goto out; - nent = 0; - } - } - } - - if (nent) - err = mlx4_cmd(dev, mailbox->dma, nent, 0, op, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); - if (err) - goto out; - - switch (op) { - case MLX4_CMD_MAP_FA: - mlx4_dbg(dev, "Mapped %d chunks/%d KB for FW.\n", tc, ts); - break; - case MLX4_CMD_MAP_ICM_AUX: - mlx4_dbg(dev, "Mapped %d chunks/%d KB for ICM aux.\n", tc, ts); - break; - case MLX4_CMD_MAP_ICM: - mlx4_dbg(dev, "Mapped %d chunks/%d KB at %llx for ICM.\n", - tc, ts, (unsigned long long) virt - (ts << 10)); - break; - } - -out: - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - -int mlx4_MAP_FA(struct mlx4_dev *dev, struct mlx4_icm *icm) -{ - return mlx4_map_cmd(dev, MLX4_CMD_MAP_FA, icm, -1); -} - -int mlx4_UNMAP_FA(struct mlx4_dev *dev) -{ - return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_UNMAP_FA, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); -} - - -int mlx4_RUN_FW(struct mlx4_dev *dev) -{ - return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_RUN_FW, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); -} - -int mlx4_QUERY_FW(struct mlx4_dev *dev) -{ - struct mlx4_fw *fw = &mlx4_priv(dev)->fw; - struct mlx4_cmd *cmd = &mlx4_priv(dev)->cmd; - struct mlx4_cmd_mailbox *mailbox; - u32 *outbox; - int err = 0; - u64 fw_ver; - u16 cmd_if_rev; - u8 lg; - -#define QUERY_FW_OUT_SIZE 0x100 -#define QUERY_FW_VER_OFFSET 0x00 -#define QUERY_FW_PPF_ID 0x09 -#define QUERY_FW_CMD_IF_REV_OFFSET 0x0a -#define QUERY_FW_MAX_CMD_OFFSET 0x0f -#define QUERY_FW_ERR_START_OFFSET 0x30 -#define QUERY_FW_ERR_SIZE_OFFSET 0x38 -#define QUERY_FW_ERR_BAR_OFFSET 0x3c - -#define QUERY_FW_SIZE_OFFSET 0x00 -#define QUERY_FW_CLR_INT_BASE_OFFSET 0x20 -#define QUERY_FW_CLR_INT_BAR_OFFSET 0x28 - -#define QUERY_FW_COMM_BASE_OFFSET 0x40 -#define QUERY_FW_COMM_BAR_OFFSET 0x48 - -#define QUERY_FW_CLOCK_OFFSET 0x50 -#define QUERY_FW_CLOCK_BAR 0x58 - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - outbox = mailbox->buf; - - err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, MLX4_CMD_QUERY_FW, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); - if (err) - goto out; - - MLX4_GET(fw_ver, outbox, QUERY_FW_VER_OFFSET); - /* - * FW subminor version is at more significant bits than minor - * version, so swap here. - */ - dev->caps.fw_ver = (fw_ver & 0xffff00000000ull) | - ((fw_ver & 0xffff0000ull) >> 16) | - ((fw_ver & 0x0000ffffull) << 16); - - MLX4_GET(lg, outbox, QUERY_FW_PPF_ID); - dev->caps.function = lg; - - if (mlx4_is_slave(dev)) - goto out; - - - MLX4_GET(cmd_if_rev, outbox, QUERY_FW_CMD_IF_REV_OFFSET); - if (cmd_if_rev < MLX4_COMMAND_INTERFACE_MIN_REV || - cmd_if_rev > MLX4_COMMAND_INTERFACE_MAX_REV) { - mlx4_err(dev, "Installed FW has unsupported " - "command interface revision %d.\n", - cmd_if_rev); - mlx4_err(dev, "(Installed FW version is %d.%d.%03d)\n", - (int) (dev->caps.fw_ver >> 32), - (int) (dev->caps.fw_ver >> 16) & 0xffff, - (int) dev->caps.fw_ver & 0xffff); - mlx4_err(dev, "This driver version supports only revisions %d to %d.\n", - MLX4_COMMAND_INTERFACE_MIN_REV, MLX4_COMMAND_INTERFACE_MAX_REV); - err = -ENODEV; - goto out; - } - - if (cmd_if_rev < MLX4_COMMAND_INTERFACE_NEW_PORT_CMDS) - dev->flags |= MLX4_FLAG_OLD_PORT_CMDS; - - MLX4_GET(lg, outbox, QUERY_FW_MAX_CMD_OFFSET); - cmd->max_cmds = 1 << lg; - - mlx4_dbg(dev, "FW version %d.%d.%03d (cmd intf rev %d), max commands %d\n", - (int) (dev->caps.fw_ver >> 32), - (int) (dev->caps.fw_ver >> 16) & 0xffff, - (int) dev->caps.fw_ver & 0xffff, - cmd_if_rev, cmd->max_cmds); - - MLX4_GET(fw->catas_offset, outbox, QUERY_FW_ERR_START_OFFSET); - MLX4_GET(fw->catas_size, outbox, QUERY_FW_ERR_SIZE_OFFSET); - MLX4_GET(fw->catas_bar, outbox, QUERY_FW_ERR_BAR_OFFSET); - fw->catas_bar = (fw->catas_bar >> 6) * 2; - - mlx4_dbg(dev, "Catastrophic error buffer at 0x%llx, size 0x%x, BAR %d\n", - (unsigned long long) fw->catas_offset, fw->catas_size, fw->catas_bar); - - MLX4_GET(fw->fw_pages, outbox, QUERY_FW_SIZE_OFFSET); - MLX4_GET(fw->clr_int_base, outbox, QUERY_FW_CLR_INT_BASE_OFFSET); - MLX4_GET(fw->clr_int_bar, outbox, QUERY_FW_CLR_INT_BAR_OFFSET); - fw->clr_int_bar = (fw->clr_int_bar >> 6) * 2; - - MLX4_GET(fw->comm_base, outbox, QUERY_FW_COMM_BASE_OFFSET); - MLX4_GET(fw->comm_bar, outbox, QUERY_FW_COMM_BAR_OFFSET); - fw->comm_bar = (fw->comm_bar >> 6) * 2; - mlx4_dbg(dev, "Communication vector bar:%d offset:0x%llx\n", - fw->comm_bar, (unsigned long long)fw->comm_base); - mlx4_dbg(dev, "FW size %d KB\n", fw->fw_pages >> 2); - - MLX4_GET(fw->clock_offset, outbox, QUERY_FW_CLOCK_OFFSET); - MLX4_GET(fw->clock_bar, outbox, QUERY_FW_CLOCK_BAR); - fw->clock_bar = (fw->clock_bar >> 6) * 2; - mlx4_dbg(dev, "Internal clock bar:%d offset:0x%llx\n", - fw->comm_bar, (unsigned long long)fw->comm_base); - - /* - * Round up number of system pages needed in case - * MLX4_ICM_PAGE_SIZE < PAGE_SIZE. - */ - fw->fw_pages = - ALIGN(fw->fw_pages, PAGE_SIZE / MLX4_ICM_PAGE_SIZE) >> - (PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT); - - mlx4_dbg(dev, "Clear int @ %llx, BAR %d\n", - (unsigned long long) fw->clr_int_base, fw->clr_int_bar); - -out: - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - -int mlx4_QUERY_FW_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - u8 *outbuf; - int err; - - outbuf = outbox->buf; - err = mlx4_cmd_box(dev, 0, outbox->dma, 0, 0, MLX4_CMD_QUERY_FW, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); - if (err) - return err; - - /* for slaves, set pci PPF ID to invalid and zero out everything - * else except FW version */ - outbuf[0] = outbuf[1] = 0; - memset(&outbuf[8], 0, QUERY_FW_OUT_SIZE - 8); - outbuf[QUERY_FW_PPF_ID] = MLX4_INVALID_SLAVE_ID; - - return 0; -} - -static void get_board_id(void *vsd, char *board_id, char *vsdstr) -{ - int i; - -#define VSD_OFFSET_SIG1 0x00 -#define VSD_OFFSET_SIG2 0xde -#define VSD_OFFSET_MLX_BOARD_ID 0xd0 -#define VSD_OFFSET_TS_BOARD_ID 0x20 -#define VSD_LEN 0xd0 - -#define VSD_SIGNATURE_TOPSPIN 0x5ad - - memset(vsdstr, 0, MLX4_VSD_LEN); - - for (i = 0; i < VSD_LEN / 4; i++) - ((u32 *)vsdstr)[i] = - swab32(*(u32 *)(vsd + i * 4)); - - memset(board_id, 0, MLX4_BOARD_ID_LEN); - - if (be16_to_cpup(vsd + VSD_OFFSET_SIG1) == VSD_SIGNATURE_TOPSPIN && - be16_to_cpup(vsd + VSD_OFFSET_SIG2) == VSD_SIGNATURE_TOPSPIN) { - strlcpy(board_id, vsd + VSD_OFFSET_TS_BOARD_ID, MLX4_BOARD_ID_LEN); - } else { - /* - * The board ID is a string but the firmware byte - * swaps each 4-byte word before passing it back to - * us. Therefore we need to swab it before printing. - */ - for (i = 0; i < 4; ++i) - ((u32 *) board_id)[i] = - swab32(*(u32 *) (vsd + VSD_OFFSET_MLX_BOARD_ID + i * 4)); - } -} - -int mlx4_QUERY_ADAPTER(struct mlx4_dev *dev, struct mlx4_adapter *adapter) -{ - struct mlx4_cmd_mailbox *mailbox; - u32 *outbox; - int err; - -#define QUERY_ADAPTER_OUT_SIZE 0x100 -#define QUERY_ADAPTER_INTA_PIN_OFFSET 0x10 -#define QUERY_ADAPTER_VSD_OFFSET 0x20 -#define QUERY_ADAPTER_VSD_VENDOR_ID_OFFSET 0x1e - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - outbox = mailbox->buf; - - err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, MLX4_CMD_QUERY_ADAPTER, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); - if (err) - goto out; - - MLX4_GET(adapter->inta_pin, outbox, QUERY_ADAPTER_INTA_PIN_OFFSET); - - adapter->vsd_vendor_id = be16_to_cpup((u16 *)outbox + - QUERY_ADAPTER_VSD_VENDOR_ID_OFFSET / 2); - - get_board_id(outbox + QUERY_ADAPTER_VSD_OFFSET / 4, - adapter->board_id, adapter->vsd); - -out: - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - -int mlx4_INIT_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param) -{ - struct mlx4_cmd_mailbox *mailbox; - __be32 *inbox; - u32 mw_enable; - int err; - -#define INIT_HCA_IN_SIZE 0x200 -#define INIT_HCA_DRV_NAME_FOR_FW_MAX_SIZE 64 -#define INIT_HCA_VERSION_OFFSET 0x000 -#define INIT_HCA_VERSION 2 -#define INIT_HCA_CACHELINE_SZ_OFFSET 0x0e -#define INIT_HCA_FLAGS_OFFSET 0x014 -#define INIT_HCA_RECOVERABLE_ERROR_EVENT_OFFSET 0x018 -#define INIT_HCA_QPC_OFFSET 0x020 -#define INIT_HCA_QPC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x10) -#define INIT_HCA_LOG_QP_OFFSET (INIT_HCA_QPC_OFFSET + 0x17) -#define INIT_HCA_SRQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x28) -#define INIT_HCA_LOG_SRQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x2f) -#define INIT_HCA_CQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x30) -#define INIT_HCA_LOG_CQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x37) -#define INIT_HCA_EQE_CQE_OFFSETS (INIT_HCA_QPC_OFFSET + 0x38) -#define INIT_HCA_ALTC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x40) -#define INIT_HCA_AUXC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x50) -#define INIT_HCA_EQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x60) -#define INIT_HCA_LOG_EQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x67) -#define INIT_HCA_NUM_SYS_EQS_OFFSET (INIT_HCA_QPC_OFFSET + 0x6a) -#define INIT_HCA_RDMARC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x70) -#define INIT_HCA_LOG_RD_OFFSET (INIT_HCA_QPC_OFFSET + 0x77) -#define INIT_HCA_MCAST_OFFSET 0x0c0 -#define INIT_HCA_MC_BASE_OFFSET (INIT_HCA_MCAST_OFFSET + 0x00) -#define INIT_HCA_LOG_MC_ENTRY_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x12) -#define INIT_HCA_LOG_MC_HASH_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x16) -#define INIT_HCA_UC_STEERING_OFFSET (INIT_HCA_MCAST_OFFSET + 0x18) -#define INIT_HCA_LOG_MC_TABLE_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x1b) -#define INIT_HCA_DEVICE_MANAGED_FLOW_STEERING_EN 0x6 -#define INIT_HCA_DRIVER_VERSION_OFFSET 0x140 -#define INIT_HCA_FS_PARAM_OFFSET 0x1d0 -#define INIT_HCA_FS_BASE_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x00) -#define INIT_HCA_FS_LOG_ENTRY_SZ_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x12) -#define INIT_HCA_FS_LOG_TABLE_SZ_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x1b) -#define INIT_HCA_FS_ETH_BITS_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x21) -#define INIT_HCA_FS_ETH_NUM_ADDRS_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x22) -#define INIT_HCA_FS_IB_BITS_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x25) -#define INIT_HCA_FS_IB_NUM_ADDRS_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x26) -#define INIT_HCA_TPT_OFFSET 0x0f0 -#define INIT_HCA_DMPT_BASE_OFFSET (INIT_HCA_TPT_OFFSET + 0x00) -#define INIT_HCA_TPT_MW_OFFSET (INIT_HCA_TPT_OFFSET + 0x08) -#define INIT_HCA_TPT_MW_ENABLE (1 << 31) -#define INIT_HCA_LOG_MPT_SZ_OFFSET (INIT_HCA_TPT_OFFSET + 0x0b) -#define INIT_HCA_MTT_BASE_OFFSET (INIT_HCA_TPT_OFFSET + 0x10) -#define INIT_HCA_CMPT_BASE_OFFSET (INIT_HCA_TPT_OFFSET + 0x18) -#define INIT_HCA_UAR_OFFSET 0x120 -#define INIT_HCA_LOG_UAR_SZ_OFFSET (INIT_HCA_UAR_OFFSET + 0x0a) -#define INIT_HCA_UAR_PAGE_SZ_OFFSET (INIT_HCA_UAR_OFFSET + 0x0b) - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - inbox = mailbox->buf; - - memset(inbox, 0, INIT_HCA_IN_SIZE); - - *((u8 *) mailbox->buf + INIT_HCA_VERSION_OFFSET) = INIT_HCA_VERSION; - - *((u8 *) mailbox->buf + INIT_HCA_CACHELINE_SZ_OFFSET) = - ((ilog2(cache_line_size()) - 4) << 5) | (1 << 4); - -#if defined(__LITTLE_ENDIAN) - *(inbox + INIT_HCA_FLAGS_OFFSET / 4) &= ~cpu_to_be32(1 << 1); -#elif defined(__BIG_ENDIAN) - *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 1); -#else -#error Host endianness not defined -#endif - /* Check port for UD address vector: */ - *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1); - - /* Enable IPoIB checksumming if we can: */ - if (dev->caps.flags & MLX4_DEV_CAP_FLAG_IPOIB_CSUM) - *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 3); - - /* Enable QoS support if module parameter set */ - if (enable_qos) - *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 2); - - /* Enable fast drop performance optimization */ - if (dev->caps.fast_drop) - *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 7); - - /* enable counters */ - if (dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS) - *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 4); - - /* CX3 is capable of extending CQEs\EQEs from 32 to 64 bytes */ - if (dev->caps.flags & MLX4_DEV_CAP_FLAG_64B_EQE) { - *(inbox + INIT_HCA_EQE_CQE_OFFSETS / 4) |= cpu_to_be32(1 << 29); - dev->caps.eqe_size = 64; - dev->caps.eqe_factor = 1; - } else { - dev->caps.eqe_size = 32; - dev->caps.eqe_factor = 0; - } - - if (dev->caps.flags & MLX4_DEV_CAP_FLAG_64B_CQE) { - *(inbox + INIT_HCA_EQE_CQE_OFFSETS / 4) |= cpu_to_be32(1 << 30); - dev->caps.cqe_size = 64; - dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_64B_CQE; - } else { - dev->caps.cqe_size = 32; - } - - if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RECOVERABLE_ERROR_EVENT) - *(inbox + INIT_HCA_RECOVERABLE_ERROR_EVENT_OFFSET / 4) |= cpu_to_be32(1 << 31); - - if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_DRIVER_VERSION_TO_FW) { - strncpy((u8 *)mailbox->buf + INIT_HCA_DRIVER_VERSION_OFFSET, - DRV_NAME_FOR_FW, - INIT_HCA_DRV_NAME_FOR_FW_MAX_SIZE - 1); - mlx4_dbg(dev, "Reporting Driver Version to FW: %s\n", - (u8 *)mailbox->buf + INIT_HCA_DRIVER_VERSION_OFFSET); - } - - /* QPC/EEC/CQC/EQC/RDMARC attributes */ - - MLX4_PUT(inbox, param->qpc_base, INIT_HCA_QPC_BASE_OFFSET); - MLX4_PUT(inbox, param->log_num_qps, INIT_HCA_LOG_QP_OFFSET); - MLX4_PUT(inbox, param->srqc_base, INIT_HCA_SRQC_BASE_OFFSET); - MLX4_PUT(inbox, param->log_num_srqs, INIT_HCA_LOG_SRQ_OFFSET); - MLX4_PUT(inbox, param->cqc_base, INIT_HCA_CQC_BASE_OFFSET); - MLX4_PUT(inbox, param->log_num_cqs, INIT_HCA_LOG_CQ_OFFSET); - MLX4_PUT(inbox, param->altc_base, INIT_HCA_ALTC_BASE_OFFSET); - MLX4_PUT(inbox, param->auxc_base, INIT_HCA_AUXC_BASE_OFFSET); - MLX4_PUT(inbox, param->eqc_base, INIT_HCA_EQC_BASE_OFFSET); - MLX4_PUT(inbox, param->log_num_eqs, INIT_HCA_LOG_EQ_OFFSET); - MLX4_PUT(inbox, param->num_sys_eqs, INIT_HCA_NUM_SYS_EQS_OFFSET); - MLX4_PUT(inbox, param->rdmarc_base, INIT_HCA_RDMARC_BASE_OFFSET); - MLX4_PUT(inbox, param->log_rd_per_qp, INIT_HCA_LOG_RD_OFFSET); - - /* steering attributes */ - if (dev->caps.steering_mode == - MLX4_STEERING_MODE_DEVICE_MANAGED) { - *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= - cpu_to_be32(1 << - INIT_HCA_DEVICE_MANAGED_FLOW_STEERING_EN); - - MLX4_PUT(inbox, param->mc_base, INIT_HCA_FS_BASE_OFFSET); - MLX4_PUT(inbox, param->log_mc_entry_sz, - INIT_HCA_FS_LOG_ENTRY_SZ_OFFSET); - MLX4_PUT(inbox, param->log_mc_table_sz, - INIT_HCA_FS_LOG_TABLE_SZ_OFFSET); - /* Enable Ethernet flow steering - * with udp unicast and tcp unicast - */ - MLX4_PUT(inbox, (u8) (MLX4_FS_UDP_UC_EN | MLX4_FS_TCP_UC_EN), - INIT_HCA_FS_ETH_BITS_OFFSET); - MLX4_PUT(inbox, (u16) MLX4_FS_NUM_OF_L2_ADDR, - INIT_HCA_FS_ETH_NUM_ADDRS_OFFSET); - /* Enable IPoIB flow steering - * with udp unicast and tcp unicast - */ - MLX4_PUT(inbox, (u8) (MLX4_FS_UDP_UC_EN | MLX4_FS_TCP_UC_EN), - INIT_HCA_FS_IB_BITS_OFFSET); - MLX4_PUT(inbox, (u16) MLX4_FS_NUM_OF_L2_ADDR, - INIT_HCA_FS_IB_NUM_ADDRS_OFFSET); - } else { - MLX4_PUT(inbox, param->mc_base, INIT_HCA_MC_BASE_OFFSET); - MLX4_PUT(inbox, param->log_mc_entry_sz, - INIT_HCA_LOG_MC_ENTRY_SZ_OFFSET); - MLX4_PUT(inbox, param->log_mc_hash_sz, - INIT_HCA_LOG_MC_HASH_SZ_OFFSET); - MLX4_PUT(inbox, param->log_mc_table_sz, - INIT_HCA_LOG_MC_TABLE_SZ_OFFSET); - if (dev->caps.steering_mode == MLX4_STEERING_MODE_B0) - MLX4_PUT(inbox, (u8) (1 << 3), - INIT_HCA_UC_STEERING_OFFSET); - } - - /* TPT attributes */ - - MLX4_PUT(inbox, param->dmpt_base, INIT_HCA_DMPT_BASE_OFFSET); - mw_enable = param->mw_enable ? INIT_HCA_TPT_MW_ENABLE : 0; - MLX4_PUT(inbox, mw_enable, INIT_HCA_TPT_MW_OFFSET); - MLX4_PUT(inbox, param->log_mpt_sz, INIT_HCA_LOG_MPT_SZ_OFFSET); - MLX4_PUT(inbox, param->mtt_base, INIT_HCA_MTT_BASE_OFFSET); - MLX4_PUT(inbox, param->cmpt_base, INIT_HCA_CMPT_BASE_OFFSET); - - /* UAR attributes */ - - MLX4_PUT(inbox, param->uar_page_sz, INIT_HCA_UAR_PAGE_SZ_OFFSET); - MLX4_PUT(inbox, param->log_uar_sz, INIT_HCA_LOG_UAR_SZ_OFFSET); - - err = mlx4_cmd(dev, mailbox->dma, 0, 0, MLX4_CMD_INIT_HCA, 10000, - MLX4_CMD_NATIVE); - - if (err) - mlx4_err(dev, "INIT_HCA returns %d\n", err); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - -int mlx4_QUERY_HCA(struct mlx4_dev *dev, - struct mlx4_init_hca_param *param) -{ - struct mlx4_cmd_mailbox *mailbox; - __be32 *outbox; - u32 dword_field; - u32 mw_enable; - int err; - u8 byte_field; - -#define QUERY_HCA_GLOBAL_CAPS_OFFSET 0x04 -#define QUERY_HCA_CORE_CLOCK_OFFSET 0x0c - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - outbox = mailbox->buf; - - err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, - MLX4_CMD_QUERY_HCA, - MLX4_CMD_TIME_CLASS_B, - !mlx4_is_slave(dev)); - if (err) - goto out; - - MLX4_GET(param->global_caps, outbox, QUERY_HCA_GLOBAL_CAPS_OFFSET); - MLX4_GET(param->hca_core_clock, outbox, QUERY_HCA_CORE_CLOCK_OFFSET); - - /* QPC/EEC/CQC/EQC/RDMARC attributes */ - - MLX4_GET(param->qpc_base, outbox, INIT_HCA_QPC_BASE_OFFSET); - MLX4_GET(param->log_num_qps, outbox, INIT_HCA_LOG_QP_OFFSET); - MLX4_GET(param->srqc_base, outbox, INIT_HCA_SRQC_BASE_OFFSET); - MLX4_GET(param->log_num_srqs, outbox, INIT_HCA_LOG_SRQ_OFFSET); - MLX4_GET(param->cqc_base, outbox, INIT_HCA_CQC_BASE_OFFSET); - MLX4_GET(param->log_num_cqs, outbox, INIT_HCA_LOG_CQ_OFFSET); - MLX4_GET(param->altc_base, outbox, INIT_HCA_ALTC_BASE_OFFSET); - MLX4_GET(param->auxc_base, outbox, INIT_HCA_AUXC_BASE_OFFSET); - MLX4_GET(param->eqc_base, outbox, INIT_HCA_EQC_BASE_OFFSET); - MLX4_GET(param->log_num_eqs, outbox, INIT_HCA_LOG_EQ_OFFSET); - MLX4_GET(param->num_sys_eqs, outbox, INIT_HCA_NUM_SYS_EQS_OFFSET); - MLX4_GET(param->rdmarc_base, outbox, INIT_HCA_RDMARC_BASE_OFFSET); - MLX4_GET(param->log_rd_per_qp, outbox, INIT_HCA_LOG_RD_OFFSET); - - MLX4_GET(dword_field, outbox, INIT_HCA_FLAGS_OFFSET); - if (dword_field & (1 << INIT_HCA_DEVICE_MANAGED_FLOW_STEERING_EN)) { - param->steering_mode = MLX4_STEERING_MODE_DEVICE_MANAGED; - } else { - MLX4_GET(byte_field, outbox, INIT_HCA_UC_STEERING_OFFSET); - if (byte_field & 0x8) - param->steering_mode = MLX4_STEERING_MODE_B0; - else - param->steering_mode = MLX4_STEERING_MODE_A0; - } - /* steering attributes */ - if (param->steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) { - MLX4_GET(param->mc_base, outbox, INIT_HCA_FS_BASE_OFFSET); - MLX4_GET(param->log_mc_entry_sz, outbox, - INIT_HCA_FS_LOG_ENTRY_SZ_OFFSET); - MLX4_GET(param->log_mc_table_sz, outbox, - INIT_HCA_FS_LOG_TABLE_SZ_OFFSET); - } else { - MLX4_GET(param->mc_base, outbox, INIT_HCA_MC_BASE_OFFSET); - MLX4_GET(param->log_mc_entry_sz, outbox, - INIT_HCA_LOG_MC_ENTRY_SZ_OFFSET); - MLX4_GET(param->log_mc_hash_sz, outbox, - INIT_HCA_LOG_MC_HASH_SZ_OFFSET); - MLX4_GET(param->log_mc_table_sz, outbox, - INIT_HCA_LOG_MC_TABLE_SZ_OFFSET); - } - - /* CX3 is capable of extending CQEs\EQEs from 32 to 64 bytes */ - MLX4_GET(byte_field, outbox, INIT_HCA_EQE_CQE_OFFSETS); - if (byte_field & 0x20) /* 64-bytes eqe enabled */ - param->dev_cap_enabled |= MLX4_DEV_CAP_64B_EQE_ENABLED; - if (byte_field & 0x40) /* 64-bytes cqe enabled */ - param->dev_cap_enabled |= MLX4_DEV_CAP_64B_CQE_ENABLED; - - /* TPT attributes */ - - MLX4_GET(param->dmpt_base, outbox, INIT_HCA_DMPT_BASE_OFFSET); - MLX4_GET(mw_enable, outbox, INIT_HCA_TPT_MW_OFFSET); - param->mw_enable = (mw_enable & INIT_HCA_TPT_MW_ENABLE) == - INIT_HCA_TPT_MW_ENABLE; - MLX4_GET(param->log_mpt_sz, outbox, INIT_HCA_LOG_MPT_SZ_OFFSET); - MLX4_GET(param->mtt_base, outbox, INIT_HCA_MTT_BASE_OFFSET); - MLX4_GET(param->cmpt_base, outbox, INIT_HCA_CMPT_BASE_OFFSET); - - /* UAR attributes */ - - MLX4_GET(param->uar_page_sz, outbox, INIT_HCA_UAR_PAGE_SZ_OFFSET); - MLX4_GET(param->log_uar_sz, outbox, INIT_HCA_LOG_UAR_SZ_OFFSET); - -out: - mlx4_free_cmd_mailbox(dev, mailbox); - - return err; -} - -/* for IB-type ports only in SRIOV mode. Checks that both proxy QP0 - * and real QP0 are active, so that the paravirtualized QP0 is ready - * to operate */ -static int check_qp0_state(struct mlx4_dev *dev, int function, int port) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - /* irrelevant if not infiniband */ - if (priv->mfunc.master.qp0_state[port].proxy_qp0_active && - priv->mfunc.master.qp0_state[port].qp0_active) - return 1; - return 0; -} - -int mlx4_INIT_PORT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int port = vhcr->in_modifier; - int err; - - if (priv->mfunc.master.slave_state[slave].init_port_mask & (1 << port)) - return 0; - - if (dev->caps.port_mask[port] != MLX4_PORT_TYPE_IB) { - /* Enable port only if it was previously disabled */ - if (!priv->mfunc.master.init_port_ref[port]) { - err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_INIT_PORT, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); - if (err) - return err; - } - priv->mfunc.master.slave_state[slave].init_port_mask |= (1 << port); - } else { - if (slave == mlx4_master_func_num(dev)) { - if (check_qp0_state(dev, slave, port) && - !priv->mfunc.master.qp0_state[port].port_active) { - err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_INIT_PORT, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); - if (err) - return err; - priv->mfunc.master.qp0_state[port].port_active = 1; - priv->mfunc.master.slave_state[slave].init_port_mask |= (1 << port); - } - } else - priv->mfunc.master.slave_state[slave].init_port_mask |= (1 << port); - } - ++priv->mfunc.master.init_port_ref[port]; - return 0; -} - -int mlx4_INIT_PORT(struct mlx4_dev *dev, int port) -{ - struct mlx4_cmd_mailbox *mailbox; - u32 *inbox; - int err; - u32 flags; - u16 field; - - if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { -#define INIT_PORT_IN_SIZE 256 -#define INIT_PORT_FLAGS_OFFSET 0x00 -#define INIT_PORT_FLAG_SIG (1 << 18) -#define INIT_PORT_FLAG_NG (1 << 17) -#define INIT_PORT_FLAG_G0 (1 << 16) -#define INIT_PORT_VL_SHIFT 4 -#define INIT_PORT_PORT_WIDTH_SHIFT 8 -#define INIT_PORT_MTU_OFFSET 0x04 -#define INIT_PORT_MAX_GID_OFFSET 0x06 -#define INIT_PORT_MAX_PKEY_OFFSET 0x0a -#define INIT_PORT_GUID0_OFFSET 0x10 -#define INIT_PORT_NODE_GUID_OFFSET 0x18 -#define INIT_PORT_SI_GUID_OFFSET 0x20 - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - inbox = mailbox->buf; - - memset(inbox, 0, INIT_PORT_IN_SIZE); - - flags = 0; - flags |= (dev->caps.vl_cap[port] & 0xf) << INIT_PORT_VL_SHIFT; - flags |= (dev->caps.port_width_cap[port] & 0xf) << INIT_PORT_PORT_WIDTH_SHIFT; - MLX4_PUT(inbox, flags, INIT_PORT_FLAGS_OFFSET); - - field = 128 << dev->caps.ib_mtu_cap[port]; - MLX4_PUT(inbox, field, INIT_PORT_MTU_OFFSET); - field = dev->caps.gid_table_len[port]; - MLX4_PUT(inbox, field, INIT_PORT_MAX_GID_OFFSET); - field = dev->caps.pkey_table_len[port]; - MLX4_PUT(inbox, field, INIT_PORT_MAX_PKEY_OFFSET); - - err = mlx4_cmd(dev, mailbox->dma, port, 0, MLX4_CMD_INIT_PORT, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); - - mlx4_free_cmd_mailbox(dev, mailbox); - } else - err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_INIT_PORT, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); - - return err; -} -EXPORT_SYMBOL_GPL(mlx4_INIT_PORT); - -int mlx4_CLOSE_PORT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - int port = vhcr->in_modifier; - int err; - - if (!(priv->mfunc.master.slave_state[slave].init_port_mask & - (1 << port))) - return 0; - - if (dev->caps.port_mask[port] != MLX4_PORT_TYPE_IB) { - if (priv->mfunc.master.init_port_ref[port] == 1) { - err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_CLOSE_PORT, - 1000, MLX4_CMD_NATIVE); - if (err) - return err; - } - priv->mfunc.master.slave_state[slave].init_port_mask &= ~(1 << port); - } else { - /* infiniband port */ - if (slave == mlx4_master_func_num(dev)) { - if (!priv->mfunc.master.qp0_state[port].qp0_active && - priv->mfunc.master.qp0_state[port].port_active) { - err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_CLOSE_PORT, - 1000, MLX4_CMD_NATIVE); - if (err) - return err; - priv->mfunc.master.slave_state[slave].init_port_mask &= ~(1 << port); - priv->mfunc.master.qp0_state[port].port_active = 0; - } - } else - priv->mfunc.master.slave_state[slave].init_port_mask &= ~(1 << port); - } - --priv->mfunc.master.init_port_ref[port]; - return 0; -} - -int mlx4_CLOSE_PORT(struct mlx4_dev *dev, int port) -{ - return mlx4_cmd(dev, 0, port, 0, MLX4_CMD_CLOSE_PORT, 1000, - MLX4_CMD_WRAPPED); -} -EXPORT_SYMBOL_GPL(mlx4_CLOSE_PORT); - -int mlx4_CLOSE_HCA(struct mlx4_dev *dev, int panic) -{ - return mlx4_cmd(dev, 0, 0, panic, MLX4_CMD_CLOSE_HCA, 1000, - MLX4_CMD_NATIVE); -} - -int mlx4_SET_ICM_SIZE(struct mlx4_dev *dev, u64 icm_size, u64 *aux_pages) -{ - int ret = mlx4_cmd_imm(dev, icm_size, aux_pages, 0, 0, - MLX4_CMD_SET_ICM_SIZE, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); - if (ret) - return ret; - - /* - * Round up number of system pages needed in case - * MLX4_ICM_PAGE_SIZE < PAGE_SIZE. - */ - *aux_pages = ALIGN(*aux_pages, PAGE_SIZE / MLX4_ICM_PAGE_SIZE) >> - (PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT); - - return 0; -} - -int mlx4_NOP(struct mlx4_dev *dev) -{ - /* Input modifier of 0x1f means "finish as soon as possible." */ - return mlx4_cmd(dev, 0, 0x1f, 0, MLX4_CMD_NOP, MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); -} - -int mlx4_query_diag_counters(struct mlx4_dev *dev, int array_length, - u8 op_modifier, u32 in_offset[], - u32 counter_out[]) -{ - struct mlx4_cmd_mailbox *mailbox; - u32 *outbox; - int ret; - int i; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - outbox = mailbox->buf; - - ret = mlx4_cmd_box(dev, 0, mailbox->dma, 0, op_modifier, - MLX4_CMD_DIAG_RPRT, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - if (ret) - goto out; - - for (i = 0; i < array_length; i++) { - if (in_offset[i] > MLX4_MAILBOX_SIZE) { - ret = -EINVAL; - goto out; - } - - MLX4_GET(counter_out[i], outbox, in_offset[i]); - } - -out: - mlx4_free_cmd_mailbox(dev, mailbox); - return ret; -} -EXPORT_SYMBOL_GPL(mlx4_query_diag_counters); - -int mlx4_MOD_STAT_CFG_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd) -{ - return -EPERM; -} - -#define MLX4_WOL_SETUP_MODE (5 << 28) -int mlx4_wol_read(struct mlx4_dev *dev, u64 *config, int port) -{ - u32 in_mod = MLX4_WOL_SETUP_MODE | port << 8; - - return mlx4_cmd_imm(dev, 0, config, in_mod, 0x3, - MLX4_CMD_MOD_STAT_CFG, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); -} -EXPORT_SYMBOL_GPL(mlx4_wol_read); - -int mlx4_wol_write(struct mlx4_dev *dev, u64 config, int port) -{ - u32 in_mod = MLX4_WOL_SETUP_MODE | port << 8; - - return mlx4_cmd(dev, config, in_mod, 0x1, MLX4_CMD_MOD_STAT_CFG, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); -} -EXPORT_SYMBOL_GPL(mlx4_wol_write); - -enum { - ADD_TO_MCG = 0x26, -}; - - -void mlx4_opreq_action(struct work_struct *work) -{ - struct mlx4_priv *priv = container_of(work, struct mlx4_priv, opreq_task); - struct mlx4_dev *dev = &priv->dev; - int num_tasks = atomic_read(&priv->opreq_count); - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_mgm *mgm; - u32 *outbox; - u32 modifier; - u16 token; - u16 type_m; - u16 type; - int err; - u32 num_qps; - struct mlx4_qp qp; - int i; - u8 rem_mcg; - u8 prot; - -#define GET_OP_REQ_MODIFIER_OFFSET 0x08 -#define GET_OP_REQ_TOKEN_OFFSET 0x14 -#define GET_OP_REQ_TYPE_OFFSET 0x1a -#define GET_OP_REQ_DATA_OFFSET 0x20 - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) { - mlx4_err(dev, "Failed to allocate mailbox for GET_OP_REQ\n"); - return; - } - outbox = mailbox->buf; - - while (num_tasks) { - err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, - MLX4_CMD_GET_OP_REQ, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - if (err) { - mlx4_err(dev, "Failed to retrieve required operation: %d\n", err); - return; - } - MLX4_GET(modifier, outbox, GET_OP_REQ_MODIFIER_OFFSET); - MLX4_GET(token, outbox, GET_OP_REQ_TOKEN_OFFSET); - MLX4_GET(type, outbox, GET_OP_REQ_TYPE_OFFSET); - type_m = type >> 12; - type &= 0xfff; - - switch (type) { - case ADD_TO_MCG: - if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) { - mlx4_warn(dev, "ADD MCG operation is not supported in " - "DEVICE_MANAGED steerign mode\n"); - err = EPERM; - break; - } - mgm = (struct mlx4_mgm *) ((u8 *) (outbox) + GET_OP_REQ_DATA_OFFSET); - num_qps = be32_to_cpu(mgm->members_count) & MGM_QPN_MASK; - rem_mcg = ((u8 *) (&mgm->members_count))[0] & 1; - prot = ((u8 *) (&mgm->members_count))[0] >> 6; - - for (i = 0; i < num_qps; i++) { - qp.qpn = be32_to_cpu(mgm->qp[i]); - if (rem_mcg) - err = mlx4_multicast_detach(dev, &qp, mgm->gid, prot, 0); - else - err = mlx4_multicast_attach(dev, &qp, mgm->gid, mgm->gid[5] ,0, prot, NULL); - if (err) - break; - } - break; - default: - mlx4_warn(dev, "Bad type for required operation\n"); - err = EINVAL; - break; - } - err = mlx4_cmd(dev, 0, ((u32) err | cpu_to_be32(token) << 16), 1, - MLX4_CMD_GET_OP_REQ, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - if (err) { - mlx4_err(dev, "Failed to acknowledge required request: %d\n", err); - goto out; - } - memset(outbox, 0, 0xffc); - num_tasks = atomic_dec_return(&priv->opreq_count); - } - -out: - mlx4_free_cmd_mailbox(dev, mailbox); -} Property changes on: stable/11/sys/ofed/drivers/net/mlx4/fw.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/fw.h =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/fw.h (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/fw.h (nonexistent) @@ -1,244 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * Copyright (c) 2006, 2007 Cisco Systems. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MLX4_FW_H -#define MLX4_FW_H - -#include "mlx4.h" -#include "icm.h" - -struct mlx4_mod_stat_cfg { - u8 log_pg_sz; - u8 log_pg_sz_m; -}; - -struct mlx4_dev_cap { - int max_srq_sz; - int max_qp_sz; - int reserved_qps; - int max_qps; - int reserved_srqs; - int max_srqs; - int max_cq_sz; - int reserved_cqs; - int max_cqs; - int max_mpts; - int reserved_eqs; - int max_eqs; - int num_sys_eqs; - int reserved_mtts; - int max_mrw_sz; - int reserved_mrws; - int max_mtt_seg; - int max_requester_per_qp; - int max_responder_per_qp; - int max_rdma_global; - int local_ca_ack_delay; - int num_ports; - u32 max_msg_sz; - int ib_mtu[MLX4_MAX_PORTS + 1]; - int max_port_width[MLX4_MAX_PORTS + 1]; - int max_vl[MLX4_MAX_PORTS + 1]; - int max_gids[MLX4_MAX_PORTS + 1]; - int max_pkeys[MLX4_MAX_PORTS + 1]; - u64 def_mac[MLX4_MAX_PORTS + 1]; - u16 eth_mtu[MLX4_MAX_PORTS + 1]; - int trans_type[MLX4_MAX_PORTS + 1]; - int vendor_oui[MLX4_MAX_PORTS + 1]; - u16 wavelength[MLX4_MAX_PORTS + 1]; - u64 trans_code[MLX4_MAX_PORTS + 1]; - u16 stat_rate_support; - int fs_log_max_ucast_qp_range_size; - int fs_max_num_qp_per_entry; - u64 flags; - u64 flags2; - int reserved_uars; - int uar_size; - int min_page_sz; - int bf_reg_size; - int bf_regs_per_page; - int max_sq_sg; - int max_sq_desc_sz; - int max_rq_sg; - int max_rq_desc_sz; - int max_qp_per_mcg; - int reserved_mgms; - int max_mcgs; - int reserved_pds; - int max_pds; - int reserved_xrcds; - int max_xrcds; - int qpc_entry_sz; - int rdmarc_entry_sz; - int altc_entry_sz; - int aux_entry_sz; - int srq_entry_sz; - int cqc_entry_sz; - int eqc_entry_sz; - int dmpt_entry_sz; - int cmpt_entry_sz; - int mtt_entry_sz; - int resize_srq; - u32 bmme_flags; - u32 reserved_lkey; - u64 max_icm_sz; - int max_gso_sz; - int max_rss_tbl_sz; - u8 supported_port_types[MLX4_MAX_PORTS + 1]; - u8 suggested_type[MLX4_MAX_PORTS + 1]; - u8 default_sense[MLX4_MAX_PORTS + 1]; - u8 log_max_macs[MLX4_MAX_PORTS + 1]; - u8 log_max_vlans[MLX4_MAX_PORTS + 1]; - u32 max_basic_counters; - u32 sync_qp; - u8 timestamp_support; - u32 max_extended_counters; -}; - -struct mlx4_func_cap { - u8 num_ports; - u8 flags; - u32 pf_context_behaviour; - int qp_quota; - int cq_quota; - int srq_quota; - int mpt_quota; - int mtt_quota; - int max_eq; - int reserved_eq; - int mcg_quota; - u32 qp0_tunnel_qpn; - u32 qp0_proxy_qpn; - u32 qp1_tunnel_qpn; - u32 qp1_proxy_qpn; - u8 physical_port; - u8 port_flags; - u8 def_counter_index; - u8 extra_flags; -}; - -struct mlx4_func { - int bus; - int device; - int function; - int physical_function; - int rsvd_eqs; - int max_eq; - int rsvd_uars; -}; - -struct mlx4_adapter { - u16 vsd_vendor_id; - char board_id[MLX4_BOARD_ID_LEN]; - char vsd[MLX4_VSD_LEN]; - u8 inta_pin; -}; - -struct mlx4_init_hca_param { - u64 qpc_base; - u64 rdmarc_base; - u64 auxc_base; - u64 altc_base; - u64 srqc_base; - u64 cqc_base; - u64 eqc_base; - u64 mc_base; - u64 dmpt_base; - u64 cmpt_base; - u64 mtt_base; - u64 global_caps; - u16 log_mc_entry_sz; - u16 log_mc_hash_sz; - u16 hca_core_clock; - u8 log_num_qps; - u8 log_num_srqs; - u8 log_num_cqs; - u8 log_num_eqs; - u16 num_sys_eqs; - u8 log_rd_per_qp; - u8 log_mc_table_sz; - u8 log_mpt_sz; - u8 log_uar_sz; - u8 uar_page_sz; /* log pg sz in 4k chunks */ - u8 mw_enable; /* Enable memory windows */ - u8 fs_hash_enable_bits; - u8 steering_mode; /* for QUERY_HCA */ - u64 dev_cap_enabled; -}; - -struct mlx4_init_ib_param { - int port_width; - int vl_cap; - int mtu_cap; - u16 gid_cap; - u16 pkey_cap; - int set_guid0; - u64 guid0; - int set_node_guid; - u64 node_guid; - int set_si_guid; - u64 si_guid; -}; - -struct mlx4_set_ib_param { - int set_si_guid; - int reset_qkey_viol; - u64 si_guid; - u32 cap_mask; -}; - -int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap); -int mlx4_QUERY_FUNC_CAP(struct mlx4_dev *dev, u32 gen_or_port, - struct mlx4_func_cap *func_cap); -int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_MAP_FA(struct mlx4_dev *dev, struct mlx4_icm *icm); -int mlx4_UNMAP_FA(struct mlx4_dev *dev); -int mlx4_RUN_FW(struct mlx4_dev *dev); -int mlx4_QUERY_FW(struct mlx4_dev *dev); -int mlx4_QUERY_ADAPTER(struct mlx4_dev *dev, struct mlx4_adapter *adapter); -int mlx4_INIT_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param); -int mlx4_QUERY_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param); -int mlx4_CLOSE_HCA(struct mlx4_dev *dev, int panic); -int mlx4_map_cmd(struct mlx4_dev *dev, u16 op, struct mlx4_icm *icm, u64 virt); -int mlx4_SET_ICM_SIZE(struct mlx4_dev *dev, u64 icm_size, u64 *aux_pages); -int mlx4_NOP(struct mlx4_dev *dev); -int mlx4_MOD_STAT_CFG(struct mlx4_dev *dev, struct mlx4_mod_stat_cfg *cfg); -int mlx4_QUERY_FUNC(struct mlx4_dev *dev, struct mlx4_func *func, int slave); -void mlx4_opreq_action(struct work_struct *work); - -#endif /* MLX4_FW_H */ Property changes on: stable/11/sys/ofed/drivers/net/mlx4/fw.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4/mlx4.h =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4/mlx4.h (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4/mlx4.h (nonexistent) @@ -1,1328 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. - * Copyright (c) 2004 Voltaire, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MLX4_H -#define MLX4_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define DRV_NAME "mlx4_core" -#define PFX DRV_NAME ": " -#define DRV_VERSION "2.1.6" -#define DRV_RELDATE __DATE__ - -#define DRV_STACK_NAME "Linux-MLNX_OFED" -#define DRV_STACK_VERSION "2.1" -#define DRV_NAME_FOR_FW DRV_STACK_NAME","DRV_STACK_VERSION - -#define MLX4_FS_UDP_UC_EN (1 << 1) -#define MLX4_FS_TCP_UC_EN (1 << 2) -#define MLX4_FS_NUM_OF_L2_ADDR 8 -#define MLX4_FS_MGM_LOG_ENTRY_SIZE 7 -#define MLX4_FS_NUM_MCG (1 << 17) - -struct mlx4_set_port_prio2tc_context { - u8 prio2tc[4]; -}; - -struct mlx4_port_scheduler_tc_cfg_be { - __be16 pg; - __be16 bw_precentage; - __be16 max_bw_units; /* 3-100Mbps, 4-1Gbps, other values - reserved */ - __be16 max_bw_value; -}; - -struct mlx4_set_port_scheduler_context { - struct mlx4_port_scheduler_tc_cfg_be tc[MLX4_NUM_TC]; -}; - -enum { - MLX4_HCR_BASE = 0x80680, - MLX4_HCR_SIZE = 0x0001c, - MLX4_CLR_INT_SIZE = 0x00008, - MLX4_SLAVE_COMM_BASE = 0x0, - MLX4_COMM_PAGESIZE = 0x1000, - MLX4_CLOCK_SIZE = 0x00008 -}; - -enum { - MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE = 10, - MLX4_MIN_MGM_LOG_ENTRY_SIZE = 7, - MLX4_MAX_MGM_LOG_ENTRY_SIZE = 12, - MLX4_MAX_QP_PER_MGM = 4 * ((1 << MLX4_MAX_MGM_LOG_ENTRY_SIZE)/16 - 2), -}; - -enum { - MLX4_NUM_PDS = 1 << 15 -}; - -enum { - MLX4_CMPT_TYPE_QP = 0, - MLX4_CMPT_TYPE_SRQ = 1, - MLX4_CMPT_TYPE_CQ = 2, - MLX4_CMPT_TYPE_EQ = 3, - MLX4_CMPT_NUM_TYPE -}; - -enum { - MLX4_CMPT_SHIFT = 24, - MLX4_NUM_CMPTS = MLX4_CMPT_NUM_TYPE << MLX4_CMPT_SHIFT -}; - -enum mlx4_mpt_state { - MLX4_MPT_DISABLED = 0, - MLX4_MPT_EN_HW, - MLX4_MPT_EN_SW -}; - -#define MLX4_COMM_TIME 10000 -enum { - MLX4_COMM_CMD_RESET, - MLX4_COMM_CMD_VHCR0, - MLX4_COMM_CMD_VHCR1, - MLX4_COMM_CMD_VHCR2, - MLX4_COMM_CMD_VHCR_EN, - MLX4_COMM_CMD_VHCR_POST, - MLX4_COMM_CMD_FLR = 254 -}; - -/*The flag indicates that the slave should delay the RESET cmd*/ -#define MLX4_DELAY_RESET_SLAVE 0xbbbbbbb -/*indicates how many retries will be done if we are in the middle of FLR*/ -#define NUM_OF_RESET_RETRIES 10 -#define SLEEP_TIME_IN_RESET (2 * 1000) -enum mlx4_resource { - RES_QP, - RES_CQ, - RES_SRQ, - RES_XRCD, - RES_MPT, - RES_MTT, - RES_MAC, - RES_VLAN, - RES_NPORT_ID, - RES_COUNTER, - RES_FS_RULE, - RES_EQ, - MLX4_NUM_OF_RESOURCE_TYPE -}; - -enum mlx4_alloc_mode { - RES_OP_RESERVE, - RES_OP_RESERVE_AND_MAP, - RES_OP_MAP_ICM, -}; - -enum mlx4_res_tracker_free_type { - RES_TR_FREE_ALL, - RES_TR_FREE_SLAVES_ONLY, - RES_TR_FREE_STRUCTS_ONLY, -}; - -/* - *Virtual HCR structures. - * mlx4_vhcr is the sw representation, in machine endianness - * - * mlx4_vhcr_cmd is the formalized structure, the one that is passed - * to FW to go through communication channel. - * It is big endian, and has the same structure as the physical HCR - * used by command interface - */ -struct mlx4_vhcr { - u64 in_param; - u64 out_param; - u32 in_modifier; - u32 errno; - u16 op; - u16 token; - u8 op_modifier; - u8 e_bit; -}; - -struct mlx4_vhcr_cmd { - __be64 in_param; - __be32 in_modifier; - u32 reserved1; - __be64 out_param; - __be16 token; - u16 reserved; - u8 status; - u8 flags; - __be16 opcode; -} __packed; - -struct mlx4_cmd_info { - u16 opcode; - bool has_inbox; - bool has_outbox; - bool out_is_imm; - bool encode_slave_id; - bool skip_err_print; - int (*verify)(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox); - int (*wrapper)(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -}; - -enum { - MLX4_DEBUG_MASK_CMD_TIME = 0x100, -}; - -#ifdef CONFIG_MLX4_DEBUG -extern int mlx4_debug_level; -#else /* CONFIG_MLX4_DEBUG */ -#define mlx4_debug_level (0) -#endif /* CONFIG_MLX4_DEBUG */ - -#define mlx4_dbg(mdev, format, arg...) \ -do { \ - if (mlx4_debug_level) \ - dev_printk(KERN_DEBUG, &mdev->pdev->dev, format, ##arg); \ -} while (0) - -#define mlx4_err(mdev, format, arg...) \ - dev_err(&mdev->pdev->dev, format, ##arg) -#define mlx4_info(mdev, format, arg...) \ - dev_info(&mdev->pdev->dev, format, ##arg) -#define mlx4_warn(mdev, format, arg...) \ - dev_warn(&mdev->pdev->dev, format, ##arg) - -extern int mlx4_log_num_mgm_entry_size; -extern int log_mtts_per_seg; -extern int mlx4_blck_lb; -extern int mlx4_set_4k_mtu; - -#define MLX4_MAX_NUM_SLAVES (MLX4_MAX_NUM_PF + MLX4_MAX_NUM_VF) -#define ALL_SLAVES 0xff - -struct mlx4_bitmap { - u32 last; - u32 top; - u32 max; - u32 reserved_top; - u32 mask; - u32 avail; - spinlock_t lock; - unsigned long *table; -}; - -struct mlx4_buddy { - unsigned long **bits; - unsigned int *num_free; - u32 max_order; - spinlock_t lock; -}; - -struct mlx4_icm; - -struct mlx4_icm_table { - u64 virt; - int num_icm; - u32 num_obj; - int obj_size; - int lowmem; - int coherent; - struct mutex mutex; - struct mlx4_icm **icm; -}; - -#define MLX4_MPT_FLAG_SW_OWNS (0xfUL << 28) -#define MLX4_MPT_FLAG_FREE (0x3UL << 28) -#define MLX4_MPT_FLAG_MIO (1 << 17) -#define MLX4_MPT_FLAG_BIND_ENABLE (1 << 15) -#define MLX4_MPT_FLAG_PHYSICAL (1 << 9) -#define MLX4_MPT_FLAG_REGION (1 << 8) - -#define MLX4_MPT_PD_FLAG_FAST_REG (1 << 27) -#define MLX4_MPT_PD_FLAG_RAE (1 << 28) -#define MLX4_MPT_PD_FLAG_EN_INV (3 << 24) - -#define MLX4_MPT_QP_FLAG_BOUND_QP (1 << 7) - -#define MLX4_MPT_STATUS_SW 0xF0 -#define MLX4_MPT_STATUS_HW 0x00 - -/* - * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits. - */ -struct mlx4_mpt_entry { - __be32 flags; - __be32 qpn; - __be32 key; - __be32 pd_flags; - __be64 start; - __be64 length; - __be32 lkey; - __be32 win_cnt; - u8 reserved1[3]; - u8 mtt_rep; - __be64 mtt_addr; - __be32 mtt_sz; - __be32 entity_size; - __be32 first_byte_offset; -} __packed; - -/* - * Must be packed because start is 64 bits but only aligned to 32 bits. - */ -struct mlx4_eq_context { - __be32 flags; - u16 reserved1[3]; - __be16 page_offset; - u8 log_eq_size; - u8 reserved2[4]; - u8 eq_period; - u8 reserved3; - u8 eq_max_count; - u8 reserved4[3]; - u8 intr; - u8 log_page_size; - u8 reserved5[2]; - u8 mtt_base_addr_h; - __be32 mtt_base_addr_l; - u32 reserved6[2]; - __be32 consumer_index; - __be32 producer_index; - u32 reserved7[4]; -}; - -struct mlx4_cq_context { - __be32 flags; - u16 reserved1[3]; - __be16 page_offset; - __be32 logsize_usrpage; - __be16 cq_period; - __be16 cq_max_count; - u8 reserved2[3]; - u8 comp_eqn; - u8 log_page_size; - u8 reserved3[2]; - u8 mtt_base_addr_h; - __be32 mtt_base_addr_l; - __be32 last_notified_index; - __be32 solicit_producer_index; - __be32 consumer_index; - __be32 producer_index; - u32 reserved4[2]; - __be64 db_rec_addr; -}; - -struct mlx4_srq_context { - __be32 state_logsize_srqn; - u8 logstride; - u8 reserved1; - __be16 xrcd; - __be32 pg_offset_cqn; - u32 reserved2; - u8 log_page_size; - u8 reserved3[2]; - u8 mtt_base_addr_h; - __be32 mtt_base_addr_l; - __be32 pd; - __be16 limit_watermark; - __be16 wqe_cnt; - u16 reserved4; - __be16 wqe_counter; - u32 reserved5; - __be64 db_rec_addr; -}; - -struct mlx4_eq { - struct mlx4_dev *dev; - void __iomem *doorbell; - int eqn; - u32 cons_index; - u16 irq; - u16 have_irq; - int nent; - struct mlx4_buf_list *page_list; - struct mlx4_mtt mtt; -}; - -struct mlx4_slave_eqe { - u8 type; - u8 port; - u32 param; -}; - -struct mlx4_slave_event_eq_info { - int eqn; - u16 token; -}; - -struct mlx4_profile { - int num_qp; - int rdmarc_per_qp; - int num_srq; - int num_cq; - int num_mcg; - int num_mpt; - unsigned num_mtt_segs; -}; - -struct mlx4_fw { - u64 clr_int_base; - u64 catas_offset; - u64 comm_base; - u64 clock_offset; - struct mlx4_icm *fw_icm; - struct mlx4_icm *aux_icm; - u32 catas_size; - u16 fw_pages; - u8 clr_int_bar; - u8 catas_bar; - u8 comm_bar; - u8 clock_bar; -}; - -struct mlx4_comm { - u32 slave_write; - u32 slave_read; -}; - -enum { - MLX4_MCAST_CONFIG = 0, - MLX4_MCAST_DISABLE = 1, - MLX4_MCAST_ENABLE = 2, -}; - -#define VLAN_FLTR_SIZE 128 - -struct mlx4_vlan_fltr { - __be32 entry[VLAN_FLTR_SIZE]; -}; - -struct mlx4_mcast_entry { - struct list_head list; - u64 addr; -}; - -struct mlx4_promisc_qp { - struct list_head list; - u32 qpn; -}; - -struct mlx4_steer_index { - struct list_head list; - unsigned int index; - struct list_head duplicates; -}; - -#define MLX4_EVENT_TYPES_NUM 64 - -struct mlx4_slave_state { - u8 comm_toggle; - u8 last_cmd; - u8 init_port_mask; - bool active; - bool old_vlan_api; - u8 function; - dma_addr_t vhcr_dma; - u16 mtu[MLX4_MAX_PORTS + 1]; - __be32 ib_cap_mask[MLX4_MAX_PORTS + 1]; - struct mlx4_slave_eqe eq[MLX4_MFUNC_MAX_EQES]; - struct list_head mcast_filters[MLX4_MAX_PORTS + 1]; - struct mlx4_vlan_fltr *vlan_filter[MLX4_MAX_PORTS + 1]; - /* event type to eq number lookup */ - struct mlx4_slave_event_eq_info event_eq[MLX4_EVENT_TYPES_NUM]; - u16 eq_pi; - u16 eq_ci; - spinlock_t lock; - /*initialized via the kzalloc*/ - u8 is_slave_going_down; - u32 cookie; - enum slave_port_state port_state[MLX4_MAX_PORTS + 1]; -}; - -#define MLX4_VGT 4095 -#define NO_INDX (-1) - - -struct mlx4_vport_state { - u64 mac; - u16 default_vlan; - u8 default_qos; - u32 tx_rate; - bool spoofchk; - u32 link_state; -}; - -struct mlx4_vf_admin_state { - struct mlx4_vport_state vport[MLX4_MAX_PORTS + 1]; -}; - -struct mlx4_vport_oper_state { - struct mlx4_vport_state state; - int mac_idx; - int vlan_idx; -}; -struct mlx4_vf_oper_state { - struct mlx4_vport_oper_state vport[MLX4_MAX_PORTS + 1]; -}; - -struct slave_list { - struct mutex mutex; - struct list_head res_list[MLX4_NUM_OF_RESOURCE_TYPE]; -}; - -struct resource_allocator { - spinlock_t alloc_lock; - union { - int res_reserved; - int res_port_rsvd[MLX4_MAX_PORTS]; - }; - union { - int res_free; - int res_port_free[MLX4_MAX_PORTS]; - }; - int *quota; - int *allocated; - int *guaranteed; -}; - -struct mlx4_resource_tracker { - spinlock_t lock; - /* tree for each resources */ - struct rb_root res_tree[MLX4_NUM_OF_RESOURCE_TYPE]; - /* num_of_slave's lists, one per slave */ - struct slave_list *slave_list; - struct resource_allocator res_alloc[MLX4_NUM_OF_RESOURCE_TYPE]; -}; - -#define SLAVE_EVENT_EQ_SIZE 128 -struct mlx4_slave_event_eq { - u32 eqn; - u32 cons; - u32 prod; - spinlock_t event_lock; - struct mlx4_eqe event_eqe[SLAVE_EVENT_EQ_SIZE]; -}; - -struct mlx4_master_qp0_state { - int proxy_qp0_active; - int qp0_active; - int port_active; -}; - -struct mlx4_mfunc_master_ctx { - struct mlx4_slave_state *slave_state; - struct mlx4_vf_admin_state *vf_admin; - struct mlx4_vf_oper_state *vf_oper; - struct mlx4_master_qp0_state qp0_state[MLX4_MAX_PORTS + 1]; - int init_port_ref[MLX4_MAX_PORTS + 1]; - u16 max_mtu[MLX4_MAX_PORTS + 1]; - int disable_mcast_ref[MLX4_MAX_PORTS + 1]; - struct mlx4_resource_tracker res_tracker; - struct workqueue_struct *comm_wq; - struct work_struct comm_work; - struct work_struct arm_comm_work; - struct work_struct slave_event_work; - struct work_struct slave_flr_event_work; - spinlock_t slave_state_lock; - __be32 comm_arm_bit_vector[4]; - struct mlx4_eqe cmd_eqe; - struct mlx4_slave_event_eq slave_eq; - struct mutex gen_eqe_mutex[MLX4_MFUNC_MAX]; -}; - -struct mlx4_mfunc { - struct mlx4_comm __iomem *comm; - struct mlx4_vhcr_cmd *vhcr; - dma_addr_t vhcr_dma; - - struct mlx4_mfunc_master_ctx master; -}; - -#define MGM_QPN_MASK 0x00FFFFFF -#define MGM_BLCK_LB_BIT 30 - -struct mlx4_mgm { - __be32 next_gid_index; - __be32 members_count; - u32 reserved[2]; - u8 gid[16]; - __be32 qp[MLX4_MAX_QP_PER_MGM]; -}; - -struct mlx4_cmd { - struct pci_pool *pool; - void __iomem *hcr; - struct mutex hcr_mutex; - struct mutex slave_cmd_mutex; - struct semaphore poll_sem; - struct semaphore event_sem; - int max_cmds; - spinlock_t context_lock; - int free_head; - struct mlx4_cmd_context *context; - u16 token_mask; - u8 use_events; - u8 toggle; - u8 comm_toggle; -}; - -enum { - MLX4_VF_IMMED_VLAN_FLAG_VLAN = 1 << 0, - MLX4_VF_IMMED_VLAN_FLAG_QOS = 1 << 1, -}; -struct mlx4_vf_immed_vlan_work { - struct work_struct work; - struct mlx4_priv *priv; - int flags; - int slave; - int vlan_ix; - int orig_vlan_ix; - u8 port; - u8 qos; - u16 vlan_id; - u16 orig_vlan_id; -}; - - -struct mlx4_uar_table { - struct mlx4_bitmap bitmap; -}; - -struct mlx4_mr_table { - struct mlx4_bitmap mpt_bitmap; - struct mlx4_buddy mtt_buddy; - u64 mtt_base; - u64 mpt_base; - struct mlx4_icm_table mtt_table; - struct mlx4_icm_table dmpt_table; -}; - -struct mlx4_cq_table { - struct mlx4_bitmap bitmap; - spinlock_t lock; - rwlock_t cq_table_lock; - struct radix_tree_root tree; - struct mlx4_icm_table table; - struct mlx4_icm_table cmpt_table; -}; - -struct mlx4_eq_table { - struct mlx4_bitmap bitmap; - char *irq_names; - void __iomem *clr_int; - void __iomem **uar_map; - u32 clr_mask; - struct mlx4_eq *eq; - struct mlx4_icm_table table; - struct mlx4_icm_table cmpt_table; - int have_irq; - u8 inta_pin; -}; - -struct mlx4_srq_table { - struct mlx4_bitmap bitmap; - spinlock_t lock; - struct radix_tree_root tree; - struct mlx4_icm_table table; - struct mlx4_icm_table cmpt_table; -}; - -struct mlx4_qp_table { - struct mlx4_bitmap bitmap; - u32 rdmarc_base; - int rdmarc_shift; - spinlock_t lock; - struct mlx4_icm_table qp_table; - struct mlx4_icm_table auxc_table; - struct mlx4_icm_table altc_table; - struct mlx4_icm_table rdmarc_table; - struct mlx4_icm_table cmpt_table; -}; - -struct mlx4_mcg_table { - struct mutex mutex; - struct mlx4_bitmap bitmap; - struct mlx4_icm_table table; -}; - -struct mlx4_catas_err { - u32 __iomem *map; - struct timer_list timer; - struct list_head list; -}; - -#define MLX4_MAX_MAC_NUM 128 -#define MLX4_MAC_TABLE_SIZE (MLX4_MAX_MAC_NUM << 3) - -struct mlx4_mac_table { - __be64 entries[MLX4_MAX_MAC_NUM]; - int refs[MLX4_MAX_MAC_NUM]; - struct mutex mutex; - int total; - int max; -}; - -#define MLX4_MAX_VLAN_NUM 128 -#define MLX4_VLAN_TABLE_SIZE (MLX4_MAX_VLAN_NUM << 2) - -struct mlx4_vlan_table { - __be32 entries[MLX4_MAX_VLAN_NUM]; - int refs[MLX4_MAX_VLAN_NUM]; - struct mutex mutex; - int total; - int max; -}; - -#define SET_PORT_GEN_ALL_VALID 0x7 -#define SET_PORT_PROMISC_SHIFT 31 -#define SET_PORT_MC_PROMISC_SHIFT 30 - -enum { - MCAST_DIRECT_ONLY = 0, - MCAST_DIRECT = 1, - MCAST_DEFAULT = 2 -}; - - -struct mlx4_set_port_general_context { - u8 reserved[3]; - u8 flags; - u16 reserved2; - __be16 mtu; - u8 pptx; - u8 pfctx; - u16 reserved3; - u8 pprx; - u8 pfcrx; - u16 reserved4; -}; - -struct mlx4_set_port_rqp_calc_context { - __be32 base_qpn; - u8 rererved; - u8 n_mac; - u8 n_vlan; - u8 n_prio; - u8 reserved2[3]; - u8 mac_miss; - u8 intra_no_vlan; - u8 no_vlan; - u8 intra_vlan_miss; - u8 vlan_miss; - u8 reserved3[3]; - u8 no_vlan_prio; - __be32 promisc; - __be32 mcast; -}; - -struct mlx4_hca_info { - struct mlx4_dev *dev; - struct device_attribute firmware_attr; - struct device_attribute hca_attr; - struct device_attribute board_attr; -}; - -struct mlx4_port_info { - struct mlx4_dev *dev; - int port; - char dev_name[16]; - struct device_attribute port_attr; - enum mlx4_port_type tmp_type; - char dev_mtu_name[16]; - struct device_attribute port_mtu_attr; - struct mlx4_mac_table mac_table; - struct mlx4_vlan_table vlan_table; - int base_qpn; -}; - -struct mlx4_sense { - struct mlx4_dev *dev; - u8 do_sense_port[MLX4_MAX_PORTS + 1]; - u8 sense_allowed[MLX4_MAX_PORTS + 1]; - struct delayed_work sense_poll; -}; - -struct mlx4_msix_ctl { - u64 pool_bm; - struct mutex pool_lock; -}; - -struct mlx4_steer { - struct list_head promisc_qps[MLX4_NUM_STEERS]; - struct list_head steer_entries[MLX4_NUM_STEERS]; -}; - -enum { - MLX4_PCI_DEV_IS_VF = 1 << 0, - MLX4_PCI_DEV_FORCE_SENSE_PORT = 1 << 1, -}; - -struct mlx4_roce_gid_entry { - u8 raw[16]; -}; - -struct counter_index { - struct list_head list; - u32 index; -}; - -struct mlx4_counters { - struct mlx4_bitmap bitmap; - struct list_head global_port_list[MLX4_MAX_PORTS]; - struct list_head vf_list[MLX4_MAX_NUM_VF][MLX4_MAX_PORTS]; - struct mutex mutex; -}; - -enum { - MLX4_NO_RR = 0, - MLX4_USE_RR = 1, -}; - -struct mlx4_priv { - struct mlx4_dev dev; - - struct list_head dev_list; - struct list_head ctx_list; - spinlock_t ctx_lock; - - int pci_dev_data; - - struct list_head pgdir_list; - struct mutex pgdir_mutex; - - struct mlx4_fw fw; - struct mlx4_cmd cmd; - struct mlx4_mfunc mfunc; - - struct mlx4_bitmap pd_bitmap; - struct mlx4_bitmap xrcd_bitmap; - struct mlx4_uar_table uar_table; - struct mlx4_mr_table mr_table; - struct mlx4_cq_table cq_table; - struct mlx4_eq_table eq_table; - struct mlx4_srq_table srq_table; - struct mlx4_qp_table qp_table; - struct mlx4_mcg_table mcg_table; - struct mlx4_counters counters_table; - - struct mlx4_catas_err catas_err; - - void __iomem *clr_base; - - struct mlx4_uar driver_uar; - void __iomem *kar; - struct mlx4_port_info port[MLX4_MAX_PORTS + 1]; - struct mlx4_hca_info hca_info; - struct mlx4_sense sense; - struct mutex port_mutex; - struct mlx4_msix_ctl msix_ctl; - struct mlx4_steer *steer; - struct list_head bf_list; - struct mutex bf_mutex; - struct io_mapping *bf_mapping; - void __iomem *clock_mapping; - int reserved_mtts; - int fs_hash_mode; - u8 virt2phys_pkey[MLX4_MFUNC_MAX][MLX4_MAX_PORTS][MLX4_MAX_PORT_PKEYS]; - __be64 slave_node_guids[MLX4_MFUNC_MAX]; - struct mlx4_roce_gid_entry roce_gids[MLX4_MAX_PORTS][128]; - atomic_t opreq_count; - struct work_struct opreq_task; -}; - -static inline struct mlx4_priv *mlx4_priv(struct mlx4_dev *dev) -{ - return container_of(dev, struct mlx4_priv, dev); -} - -#define MLX4_SENSE_RANGE (HZ * 3) - -extern struct workqueue_struct *mlx4_wq; - -u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap); -void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr); -u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, - int align, u32 skip_mask); -void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt, - int use_rr); -u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap); -int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, - u32 reserved_bot, u32 resetrved_top); -void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap); - -int mlx4_reset(struct mlx4_dev *dev); - -int mlx4_alloc_eq_table(struct mlx4_dev *dev); -void mlx4_free_eq_table(struct mlx4_dev *dev); - -int mlx4_init_pd_table(struct mlx4_dev *dev); -int mlx4_init_xrcd_table(struct mlx4_dev *dev); -int mlx4_init_uar_table(struct mlx4_dev *dev); -int mlx4_init_mr_table(struct mlx4_dev *dev); -int mlx4_init_eq_table(struct mlx4_dev *dev); -int mlx4_init_cq_table(struct mlx4_dev *dev); -int mlx4_init_qp_table(struct mlx4_dev *dev); -int mlx4_init_srq_table(struct mlx4_dev *dev); -int mlx4_init_mcg_table(struct mlx4_dev *dev); - -void mlx4_cleanup_pd_table(struct mlx4_dev *dev); -void mlx4_cleanup_xrcd_table(struct mlx4_dev *dev); -void mlx4_cleanup_uar_table(struct mlx4_dev *dev); -void mlx4_cleanup_mr_table(struct mlx4_dev *dev); -void mlx4_cleanup_eq_table(struct mlx4_dev *dev); -void mlx4_cleanup_cq_table(struct mlx4_dev *dev); -void mlx4_cleanup_qp_table(struct mlx4_dev *dev); -void mlx4_cleanup_srq_table(struct mlx4_dev *dev); -void mlx4_cleanup_mcg_table(struct mlx4_dev *dev); -int __mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn); -void __mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn); -int __mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn); -void __mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn); -int __mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn); -void __mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn); -int __mlx4_mpt_reserve(struct mlx4_dev *dev); -void __mlx4_mpt_release(struct mlx4_dev *dev, u32 index); -int __mlx4_mpt_alloc_icm(struct mlx4_dev *dev, u32 index); -void __mlx4_mpt_free_icm(struct mlx4_dev *dev, u32 index); -u32 __mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order); -void __mlx4_free_mtt_range(struct mlx4_dev *dev, u32 first_seg, int order); - -int mlx4_WRITE_MTT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_SYNC_TPT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_SW2HW_MPT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_HW2SW_MPT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_QUERY_MPT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_SW2HW_EQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_DMA_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int __mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, - int *base, u8 flags); -void __mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt); -int __mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac); -void __mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, u64 mac); -int __mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, - int start_index, int npages, u64 *page_list); -int __mlx4_counter_alloc(struct mlx4_dev *dev, int slave, int port, u32 *idx); -void __mlx4_counter_free(struct mlx4_dev *dev, int slave, int port, u32 idx); - -int __mlx4_slave_counters_free(struct mlx4_dev *dev, int slave); -int __mlx4_clear_if_stat(struct mlx4_dev *dev, - u8 counter_index); -u8 mlx4_get_default_counter_index(struct mlx4_dev *dev, int slave, int port); - -int __mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn); -void __mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn); - -void mlx4_start_catas_poll(struct mlx4_dev *dev); -void mlx4_stop_catas_poll(struct mlx4_dev *dev); -void mlx4_catas_init(void); -int mlx4_restart_one(struct pci_dev *pdev); -int mlx4_register_device(struct mlx4_dev *dev); -void mlx4_unregister_device(struct mlx4_dev *dev); -void mlx4_dispatch_event(struct mlx4_dev *dev, enum mlx4_dev_event type, - unsigned long param); - -struct mlx4_dev_cap; -struct mlx4_init_hca_param; - -u64 mlx4_make_profile(struct mlx4_dev *dev, - struct mlx4_profile *request, - struct mlx4_dev_cap *dev_cap, - struct mlx4_init_hca_param *init_hca); -void mlx4_master_comm_channel(struct work_struct *work); -void mlx4_master_arm_comm_channel(struct work_struct *work); -void mlx4_gen_slave_eqe(struct work_struct *work); -void mlx4_master_handle_slave_flr(struct work_struct *work); - -int mlx4_ALLOC_RES_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_FREE_RES_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_MAP_EQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_COMM_INT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_HW2SW_EQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_QUERY_EQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_SW2HW_CQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_HW2SW_CQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_QUERY_CQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_MODIFY_CQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_SW2HW_SRQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_HW2SW_SRQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_QUERY_SRQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_ARM_SRQ_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_GEN_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_RST2INIT_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_INIT2INIT_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_INIT2RTR_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_RTR2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_RTS2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_SQERR2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_2ERR_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_RTS2SQD_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_SQD2SQD_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_SQD2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_2RST_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_QUERY_QP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); - -int mlx4_GEN_EQE(struct mlx4_dev *dev, int slave, struct mlx4_eqe *eqe); - -int mlx4_cmd_init(struct mlx4_dev *dev); -void mlx4_cmd_cleanup(struct mlx4_dev *dev); -int mlx4_multi_func_init(struct mlx4_dev *dev); -void mlx4_multi_func_cleanup(struct mlx4_dev *dev); -void mlx4_cmd_event(struct mlx4_dev *dev, u16 token, u8 status, u64 out_param); -int mlx4_cmd_use_events(struct mlx4_dev *dev); -void mlx4_cmd_use_polling(struct mlx4_dev *dev); - -int mlx4_comm_cmd(struct mlx4_dev *dev, u8 cmd, u16 param, - unsigned long timeout); - -void mlx4_cq_completion(struct mlx4_dev *dev, u32 cqn); -void mlx4_cq_event(struct mlx4_dev *dev, u32 cqn, int event_type); - -void mlx4_qp_event(struct mlx4_dev *dev, u32 qpn, int event_type); - -void mlx4_srq_event(struct mlx4_dev *dev, u32 srqn, int event_type); - -void mlx4_handle_catas_err(struct mlx4_dev *dev); - -int mlx4_SENSE_PORT(struct mlx4_dev *dev, int port, - enum mlx4_port_type *type); -void mlx4_do_sense_ports(struct mlx4_dev *dev, - enum mlx4_port_type *stype, - enum mlx4_port_type *defaults); -void mlx4_start_sense(struct mlx4_dev *dev); -void mlx4_stop_sense(struct mlx4_dev *dev); -void mlx4_sense_init(struct mlx4_dev *dev); -int mlx4_check_port_params(struct mlx4_dev *dev, - enum mlx4_port_type *port_type); -int mlx4_change_port_types(struct mlx4_dev *dev, - enum mlx4_port_type *port_types); - -void mlx4_init_mac_table(struct mlx4_dev *dev, struct mlx4_mac_table *table); -void mlx4_init_vlan_table(struct mlx4_dev *dev, struct mlx4_vlan_table *table); -void __mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, u16 vlan); -int __mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, int *index); - -int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port, int pkey_tbl_sz); -/* resource tracker functions*/ -int mlx4_get_slave_from_resource_id(struct mlx4_dev *dev, - enum mlx4_resource resource_type, - u64 resource_id, int *slave); -void mlx4_delete_all_resources_for_slave(struct mlx4_dev *dev, int slave_id); -int mlx4_init_resource_tracker(struct mlx4_dev *dev); - -void mlx4_free_resource_tracker(struct mlx4_dev *dev, - enum mlx4_res_tracker_free_type type); - -int mlx4_QUERY_FW_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_SET_PORT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_INIT_PORT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_CLOSE_PORT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_QUERY_DEV_CAP_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_QUERY_PORT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_get_port_ib_caps(struct mlx4_dev *dev, u8 port, __be32 *caps); - -int mlx4_get_slave_pkey_gid_tbl_len(struct mlx4_dev *dev, u8 port, - int *gid_tbl_len, int *pkey_tbl_len); - -int mlx4_QP_ATTACH_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); - -int mlx4_PROMISC_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_qp_detach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], - enum mlx4_protocol prot, enum mlx4_steer_type steer); -int mlx4_qp_attach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], - int block_mcast_loopback, enum mlx4_protocol prot, - enum mlx4_steer_type steer); -int mlx4_trans_to_dmfs_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, - u8 gid[16], u8 port, - int block_mcast_loopback, - enum mlx4_protocol prot, u64 *reg_id); -int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, u64 mac, u64 clear, u8 mode); -int mlx4_SET_MCAST_FLTR_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_SET_VLAN_FLTR_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_common_set_vlan_fltr(struct mlx4_dev *dev, int function, - int port, void *buf); -int mlx4_DUMP_ETH_STATS_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_PKEY_TABLE_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_QUERY_IF_STAT_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_QP_FLOW_STEERING_ATTACH_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_QP_FLOW_STEERING_DETACH_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); -int mlx4_MOD_STAT_CFG_wrapper(struct mlx4_dev *dev, int slave, - struct mlx4_vhcr *vhcr, - struct mlx4_cmd_mailbox *inbox, - struct mlx4_cmd_mailbox *outbox, - struct mlx4_cmd_info *cmd); - -int mlx4_get_mgm_entry_size(struct mlx4_dev *dev); -int mlx4_get_qp_per_mgm(struct mlx4_dev *dev); - -static inline void set_param_l(u64 *arg, u32 val) -{ - *arg = (*arg & 0xffffffff00000000ULL) | (u64) val; -} - -static inline void set_param_h(u64 *arg, u32 val) -{ - *arg = (*arg & 0xffffffff) | ((u64) val << 32); -} - -static inline u32 get_param_l(u64 *arg) -{ - return (u32) (*arg & 0xffffffff); -} - -static inline u32 get_param_h(u64 *arg) -{ - return (u32)(*arg >> 32); -} - -static inline spinlock_t *mlx4_tlock(struct mlx4_dev *dev) -{ - return &mlx4_priv(dev)->mfunc.master.res_tracker.lock; -} - -#define NOT_MASKED_PD_BITS 17 - -void sys_tune_init(void); -void sys_tune_fini(void); - -void mlx4_init_quotas(struct mlx4_dev *dev); - -int mlx4_get_slave_num_gids(struct mlx4_dev *dev, int slave); -int mlx4_get_base_gid_ix(struct mlx4_dev *dev, int slave); -void mlx4_vf_immed_vlan_work_handler(struct work_struct *_work); - -#endif /* MLX4_H */ Property changes on: stable/11/sys/ofed/drivers/net/mlx4/mlx4.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net/mlx4 =================================================================== --- stable/11/sys/ofed/drivers/net/mlx4 (revision 329158) +++ stable/11/sys/ofed/drivers/net/mlx4 (nonexistent) Property changes on: stable/11/sys/ofed/drivers/net/mlx4 ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/net =================================================================== --- stable/11/sys/ofed/drivers/net (revision 329158) +++ stable/11/sys/ofed/drivers/net (nonexistent) Property changes on: stable/11/sys/ofed/drivers/net ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mlx4_ib.h =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mlx4_ib.h (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mlx4_ib.h (nonexistent) @@ -1,865 +0,0 @@ -/* - * Copyright (c) 2006, 2007 Cisco Systems. All rights reserved. - * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MLX4_IB_H -#define MLX4_IB_H - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include - -#define MLX4_IB_DRV_NAME "mlx4_ib" - -#ifdef pr_fmt -#undef pr_fmt -#endif -#define pr_fmt(fmt) "<" MLX4_IB_DRV_NAME "> %s: " fmt, __func__ - -#define mlx4_ib_warn(ibdev, format, arg...) \ - dev_warn((ibdev)->dma_device, MLX4_IB_DRV_NAME ": " format, ## arg) - -#define mlx4_ib_info(ibdev, format, arg...) \ - dev_info((ibdev)->dma_device, MLX4_IB_DRV_NAME ": " format, ## arg) - -enum { - MLX4_IB_SQ_MIN_WQE_SHIFT = 6, - MLX4_IB_MAX_HEADROOM = 2048 -}; - -#define MLX4_IB_SQ_HEADROOM(shift) ((MLX4_IB_MAX_HEADROOM >> (shift)) + 1) -#define MLX4_IB_SQ_MAX_SPARE (MLX4_IB_SQ_HEADROOM(MLX4_IB_SQ_MIN_WQE_SHIFT)) - -/*module param to indicate if SM assigns the alias_GUID*/ -extern int mlx4_ib_sm_guid_assign; -extern struct proc_dir_entry *mlx4_mrs_dir_entry; - -#define MLX4_IB_UC_STEER_QPN_ALIGN 1 -#define MLX4_IB_UC_MAX_NUM_QPS (256 * 1024) - - -#define MLX4_IB_MMAP_CMD_MASK 0xFF -#define MLX4_IB_MMAP_CMD_BITS 8 - -struct mlx4_ib_ucontext { - struct ib_ucontext ibucontext; - struct mlx4_uar uar; - struct list_head db_page_list; - struct mutex db_page_mutex; -}; - -struct mlx4_ib_pd { - struct ib_pd ibpd; - u32 pdn; -}; - -struct mlx4_ib_xrcd { - struct ib_xrcd ibxrcd; - u32 xrcdn; - struct ib_pd *pd; - struct ib_cq *cq; -}; - -struct mlx4_ib_cq_buf { - struct mlx4_buf buf; - struct mlx4_mtt mtt; - int entry_size; -}; - -struct mlx4_ib_cq_resize { - struct mlx4_ib_cq_buf buf; - int cqe; -}; - -struct mlx4_shared_mr_info { - int mr_id; - struct ib_umem *umem; -}; - -struct mlx4_ib_cq { - struct ib_cq ibcq; - struct mlx4_cq mcq; - struct mlx4_ib_cq_buf buf; - struct mlx4_ib_cq_resize *resize_buf; - struct mlx4_db db; - spinlock_t lock; - struct mutex resize_mutex; - struct ib_umem *umem; - struct ib_umem *resize_umem; - int create_flags; -}; - -struct mlx4_ib_mr { - struct ib_mr ibmr; - struct mlx4_mr mmr; - struct ib_umem *umem; - struct mlx4_shared_mr_info *smr_info; - atomic_t invalidated; - struct completion invalidation_comp; -}; - -struct mlx4_ib_mw { - struct ib_mw ibmw; - struct mlx4_mw mmw; -}; - -struct mlx4_ib_fast_reg_page_list { - struct ib_fast_reg_page_list ibfrpl; - __be64 *mapped_page_list; - dma_addr_t map; -}; - -struct mlx4_ib_fmr { - struct ib_fmr ibfmr; - struct mlx4_fmr mfmr; -}; - -struct mlx4_ib_flow { - struct ib_flow ibflow; - /* translating DMFS verbs sniffer rule to FW API requires two reg IDs */ - u64 reg_id[2]; -}; - -struct mlx4_ib_wq { - u64 *wrid; - spinlock_t lock; - int wqe_cnt; - int max_post; - int max_gs; - int offset; - int wqe_shift; - unsigned head; - unsigned tail; -}; - -enum mlx4_ib_qp_flags { - MLX4_IB_QP_LSO = IB_QP_CREATE_IPOIB_UD_LSO, - MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK = IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK, - MLX4_IB_QP_CAP_CROSS_CHANNEL = IB_QP_CREATE_CROSS_CHANNEL, - MLX4_IB_QP_CAP_MANAGED_SEND = IB_QP_CREATE_MANAGED_SEND, - MLX4_IB_QP_CAP_MANAGED_RECV = IB_QP_CREATE_MANAGED_RECV, - MLX4_IB_QP_NETIF = IB_QP_CREATE_NETIF_QP, - MLX4_IB_SRIOV_TUNNEL_QP = 1 << 30, - MLX4_IB_SRIOV_SQP = 1 << 31, -}; - -struct mlx4_ib_gid_entry { - struct list_head list; - union ib_gid gid; - int added; - u8 port; -}; - -enum mlx4_ib_mmap_cmd { - MLX4_IB_MMAP_UAR_PAGE = 0, - MLX4_IB_MMAP_BLUE_FLAME_PAGE = 1, - MLX4_IB_MMAP_GET_CONTIGUOUS_PAGES = 2, - MLX4_IB_MMAP_GET_HW_CLOCK = 3, -}; - -enum mlx4_ib_qp_type { - /* - * IB_QPT_SMI and IB_QPT_GSI have to be the first two entries - * here (and in that order) since the MAD layer uses them as - * indices into a 2-entry table. - */ - MLX4_IB_QPT_SMI = IB_QPT_SMI, - MLX4_IB_QPT_GSI = IB_QPT_GSI, - - MLX4_IB_QPT_RC = IB_QPT_RC, - MLX4_IB_QPT_UC = IB_QPT_UC, - MLX4_IB_QPT_UD = IB_QPT_UD, - MLX4_IB_QPT_RAW_IPV6 = IB_QPT_RAW_IPV6, - MLX4_IB_QPT_RAW_ETHERTYPE = IB_QPT_RAW_ETHERTYPE, - MLX4_IB_QPT_RAW_PACKET = IB_QPT_RAW_PACKET, - MLX4_IB_QPT_XRC_INI = IB_QPT_XRC_INI, - MLX4_IB_QPT_XRC_TGT = IB_QPT_XRC_TGT, - - MLX4_IB_QPT_PROXY_SMI_OWNER = 1 << 16, - MLX4_IB_QPT_PROXY_SMI = 1 << 17, - MLX4_IB_QPT_PROXY_GSI = 1 << 18, - MLX4_IB_QPT_TUN_SMI_OWNER = 1 << 19, - MLX4_IB_QPT_TUN_SMI = 1 << 20, - MLX4_IB_QPT_TUN_GSI = 1 << 21, -}; - -#define MLX4_IB_QPT_ANY_SRIOV (MLX4_IB_QPT_PROXY_SMI_OWNER | \ - MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER | \ - MLX4_IB_QPT_TUN_SMI | MLX4_IB_QPT_TUN_GSI) - -enum mlx4_ib_mad_ifc_flags { - MLX4_MAD_IFC_IGNORE_MKEY = 1, - MLX4_MAD_IFC_IGNORE_BKEY = 2, - MLX4_MAD_IFC_IGNORE_KEYS = (MLX4_MAD_IFC_IGNORE_MKEY | - MLX4_MAD_IFC_IGNORE_BKEY), - MLX4_MAD_IFC_NET_VIEW = 4, -}; - -enum { - MLX4_NUM_TUNNEL_BUFS = 256, -}; - -struct mlx4_ib_tunnel_header { - struct mlx4_av av; - __be32 remote_qpn; - __be32 qkey; - __be16 vlan; - u8 mac[6]; - __be16 pkey_index; - u8 reserved[6]; -}; - -struct mlx4_ib_buf { - void *addr; - dma_addr_t map; -}; - -struct mlx4_rcv_tunnel_hdr { - __be32 flags_src_qp; /* flags[6:5] is defined for VLANs: - * 0x0 - no vlan was in the packet - * 0x01 - C-VLAN was in the packet */ - u8 g_ml_path; /* gid bit stands for ipv6/4 header in RoCE */ - u8 reserved; - __be16 pkey_index; - __be16 sl_vid; - __be16 slid_mac_47_32; - __be32 mac_31_0; -}; - -struct mlx4_ib_proxy_sqp_hdr { - struct ib_grh grh; - struct mlx4_rcv_tunnel_hdr tun; -} __packed; - -struct mlx4_roce_smac_vlan_info { - u64 smac; - int smac_index; - int smac_port; - u64 candidate_smac; - int candidate_smac_index; - int candidate_smac_port; - u16 vid; - int vlan_index; - int vlan_port; - u16 candidate_vid; - int candidate_vlan_index; - int candidate_vlan_port; - int update_vid; -}; - -struct mlx4_ib_qpg_data { - unsigned long *tss_bitmap; - unsigned long *rss_bitmap; - struct mlx4_ib_qp *qpg_parent; - int tss_qpn_base; - int rss_qpn_base; - u32 tss_child_count; - u32 rss_child_count; - u32 qpg_tss_mask_sz; -}; - -struct mlx4_ib_qp { - struct ib_qp ibqp; - struct mlx4_qp mqp; - struct mlx4_buf buf; - - struct mlx4_db db; - struct mlx4_ib_wq rq; - - u32 doorbell_qpn; - __be32 sq_signal_bits; - unsigned sq_next_wqe; - int sq_max_wqes_per_wr; - int sq_spare_wqes; - struct mlx4_ib_wq sq; - - enum mlx4_ib_qp_type mlx4_ib_qp_type; - struct ib_umem *umem; - struct mlx4_mtt mtt; - int buf_size; - struct mutex mutex; - u16 xrcdn; - u32 flags; - u8 port; - u8 alt_port; - u8 atomic_rd_en; - u8 resp_depth; - u8 sq_no_prefetch; - u8 state; - int mlx_type; - enum ib_qpg_type qpg_type; - struct mlx4_ib_qpg_data *qpg_data; - struct list_head gid_list; - struct list_head steering_rules; - struct mlx4_ib_buf *sqp_proxy_rcv; - struct mlx4_roce_smac_vlan_info pri; - struct mlx4_roce_smac_vlan_info alt; - struct list_head rules_list; - u64 reg_id; - int max_inline_data; - struct mlx4_bf bf; - - /* - * Experimental data - */ - int max_inlr_data; -}; - -struct mlx4_ib_srq { - struct ib_srq ibsrq; - struct mlx4_srq msrq; - struct mlx4_buf buf; - struct mlx4_db db; - u64 *wrid; - spinlock_t lock; - int head; - int tail; - u16 wqe_ctr; - struct ib_umem *umem; - struct mlx4_mtt mtt; - struct mutex mutex; -}; - -struct mlx4_ib_ah { - struct ib_ah ibah; - union mlx4_ext_av av; -}; - -/****************************************/ -/* alias guid support */ -/****************************************/ -#define NUM_PORT_ALIAS_GUID 2 -#define NUM_ALIAS_GUID_IN_REC 8 -#define NUM_ALIAS_GUID_REC_IN_PORT 16 -#define GUID_REC_SIZE 8 -#define NUM_ALIAS_GUID_PER_PORT 128 -#define MLX4_NOT_SET_GUID (0x00LL) -#define MLX4_GUID_FOR_DELETE_VAL (~(0x00LL)) - -/****************************************/ -/* ioctl codes */ -/****************************************/ -#define MLX4_IOC_MAGIC 'm' -#define MLX4_IOCHWCLOCKOFFSET _IOR(MLX4_IOC_MAGIC, 1, int) - -enum mlx4_guid_alias_rec_status { - MLX4_GUID_INFO_STATUS_IDLE, - MLX4_GUID_INFO_STATUS_SET, - MLX4_GUID_INFO_STATUS_PENDING, -}; - -enum mlx4_guid_alias_rec_ownership { - MLX4_GUID_DRIVER_ASSIGN, - MLX4_GUID_SYSADMIN_ASSIGN, - MLX4_GUID_NONE_ASSIGN, /*init state of each record*/ -}; - -enum mlx4_guid_alias_rec_method { - MLX4_GUID_INFO_RECORD_SET = IB_MGMT_METHOD_SET, - MLX4_GUID_INFO_RECORD_DELETE = IB_SA_METHOD_DELETE, -}; - -struct mlx4_sriov_alias_guid_info_rec_det { - u8 all_recs[GUID_REC_SIZE * NUM_ALIAS_GUID_IN_REC]; - ib_sa_comp_mask guid_indexes; /*indicates what from the 8 records are valid*/ - enum mlx4_guid_alias_rec_status status; /*indicates the administraively status of the record.*/ - u8 method; /*set or delete*/ - enum mlx4_guid_alias_rec_ownership ownership; /*indicates who assign that alias_guid record*/ -}; - -struct mlx4_sriov_alias_guid_port_rec_det { - struct mlx4_sriov_alias_guid_info_rec_det all_rec_per_port[NUM_ALIAS_GUID_REC_IN_PORT]; - struct workqueue_struct *wq; - struct delayed_work alias_guid_work; - u8 port; - struct mlx4_sriov_alias_guid *parent; - struct list_head cb_list; -}; - -struct mlx4_sriov_alias_guid { - struct mlx4_sriov_alias_guid_port_rec_det ports_guid[MLX4_MAX_PORTS]; - spinlock_t ag_work_lock; - struct ib_sa_client *sa_client; -}; - -struct mlx4_ib_demux_work { - struct work_struct work; - struct mlx4_ib_dev *dev; - int slave; - int do_init; - u8 port; - -}; - -struct mlx4_ib_tun_tx_buf { - struct mlx4_ib_buf buf; - struct ib_ah *ah; -}; - -struct mlx4_ib_demux_pv_qp { - struct ib_qp *qp; - enum ib_qp_type proxy_qpt; - struct mlx4_ib_buf *ring; - struct mlx4_ib_tun_tx_buf *tx_ring; - spinlock_t tx_lock; - unsigned tx_ix_head; - unsigned tx_ix_tail; -}; - -enum mlx4_ib_demux_pv_state { - DEMUX_PV_STATE_DOWN, - DEMUX_PV_STATE_STARTING, - DEMUX_PV_STATE_ACTIVE, - DEMUX_PV_STATE_DOWNING, -}; - -struct mlx4_ib_demux_pv_ctx { - int port; - int slave; - enum mlx4_ib_demux_pv_state state; - int has_smi; - struct ib_device *ib_dev; - struct ib_cq *cq; - struct ib_pd *pd; - struct ib_mr *mr; - struct work_struct work; - struct workqueue_struct *wq; - struct mlx4_ib_demux_pv_qp qp[2]; -}; - -struct mlx4_ib_demux_ctx { - struct ib_device *ib_dev; - int port; - struct workqueue_struct *wq; - struct workqueue_struct *ud_wq; - spinlock_t ud_lock; - __be64 subnet_prefix; - __be64 guid_cache[128]; - struct mlx4_ib_dev *dev; - /* the following lock protects both mcg_table and mcg_mgid0_list */ - struct mutex mcg_table_lock; - struct rb_root mcg_table; - struct list_head mcg_mgid0_list; - struct workqueue_struct *mcg_wq; - struct mlx4_ib_demux_pv_ctx **tun; - atomic_t tid; - int flushing; /* flushing the work queue */ -}; - -struct mlx4_ib_sriov { - struct mlx4_ib_demux_ctx demux[MLX4_MAX_PORTS]; - struct mlx4_ib_demux_pv_ctx *sqps[MLX4_MAX_PORTS]; - /* when using this spinlock you should use "irq" because - * it may be called from interrupt context.*/ - spinlock_t going_down_lock; - int is_going_down; - - struct mlx4_sriov_alias_guid alias_guid; - - /* CM paravirtualization fields */ - struct list_head cm_list; - spinlock_t id_map_lock; - struct rb_root sl_id_map; - struct idr pv_id_table; -}; - -struct mlx4_ib_iboe { - spinlock_t lock; - struct net_device *netdevs[MLX4_MAX_PORTS]; - struct net_device *masters[MLX4_MAX_PORTS]; - struct notifier_block nb; - struct notifier_block nb_inet; - union ib_gid gid_table[MLX4_MAX_PORTS][128]; -}; - -struct pkey_mgt { - u8 virt2phys_pkey[MLX4_MFUNC_MAX][MLX4_MAX_PORTS][MLX4_MAX_PORT_PKEYS]; - u16 phys_pkey_cache[MLX4_MAX_PORTS][MLX4_MAX_PORT_PKEYS]; - struct list_head pkey_port_list[MLX4_MFUNC_MAX]; - struct kobject *device_parent[MLX4_MFUNC_MAX]; -}; - -struct mlx4_ib_iov_sysfs_attr { - void *ctx; - struct kobject *kobj; - unsigned long data; - u32 entry_num; - char name[15]; - struct device_attribute dentry; - struct device *dev; -}; - -struct mlx4_ib_iov_sysfs_attr_ar { - struct mlx4_ib_iov_sysfs_attr dentries[3 * NUM_ALIAS_GUID_PER_PORT + 1]; -}; - -struct mlx4_ib_iov_port { - char name[100]; - u8 num; - struct mlx4_ib_dev *dev; - struct list_head list; - struct mlx4_ib_iov_sysfs_attr_ar *dentr_ar; - struct ib_port_attr attr; - struct kobject *cur_port; - struct kobject *admin_alias_parent; - struct kobject *gids_parent; - struct kobject *pkeys_parent; - struct kobject *mcgs_parent; - struct mlx4_ib_iov_sysfs_attr mcg_dentry; -}; - -struct mlx4_ib_counter { - int counter_index; - int status; -}; - -struct mlx4_ib_dev { - struct ib_device ib_dev; - struct mlx4_dev *dev; - int num_ports; - struct mlx4_uar priv_uar; - u32 priv_pdn; - MLX4_DECLARE_DOORBELL_LOCK(uar_lock); - - struct ib_mad_agent *send_agent[MLX4_MAX_PORTS][2]; - struct ib_ah *sm_ah[MLX4_MAX_PORTS]; - spinlock_t sm_lock; - struct mlx4_ib_sriov sriov; - - struct mutex cap_mask_mutex; - bool ib_active; - struct mlx4_ib_iboe iboe; - struct mlx4_ib_counter counters[MLX4_MAX_PORTS]; - int *eq_table; - int eq_added; - struct kobject *iov_parent; - struct kobject *ports_parent; - struct kobject *dev_ports_parent[MLX4_MFUNC_MAX]; - struct mlx4_ib_iov_port iov_ports[MLX4_MAX_PORTS]; - struct pkey_mgt pkeys; - unsigned long *ib_uc_qpns_bitmap; - int steer_qpn_count; - int steer_qpn_base; -}; - -struct ib_event_work { - struct work_struct work; - struct mlx4_ib_dev *ib_dev; - struct mlx4_eqe ib_eqe; -}; - -struct mlx4_ib_qp_tunnel_init_attr { - struct ib_qp_init_attr init_attr; - int slave; - enum ib_qp_type proxy_qp_type; - u8 port; -}; - -static inline struct mlx4_ib_dev *to_mdev(struct ib_device *ibdev) -{ - return container_of(ibdev, struct mlx4_ib_dev, ib_dev); -} - -static inline struct mlx4_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext) -{ - return container_of(ibucontext, struct mlx4_ib_ucontext, ibucontext); -} - -static inline struct mlx4_ib_pd *to_mpd(struct ib_pd *ibpd) -{ - return container_of(ibpd, struct mlx4_ib_pd, ibpd); -} - -static inline struct mlx4_ib_xrcd *to_mxrcd(struct ib_xrcd *ibxrcd) -{ - return container_of(ibxrcd, struct mlx4_ib_xrcd, ibxrcd); -} - -static inline struct mlx4_ib_cq *to_mcq(struct ib_cq *ibcq) -{ - return container_of(ibcq, struct mlx4_ib_cq, ibcq); -} - -static inline struct mlx4_ib_cq *to_mibcq(struct mlx4_cq *mcq) -{ - return container_of(mcq, struct mlx4_ib_cq, mcq); -} - -static inline struct mlx4_ib_mr *to_mmr(struct ib_mr *ibmr) -{ - return container_of(ibmr, struct mlx4_ib_mr, ibmr); -} - -static inline struct mlx4_ib_mw *to_mmw(struct ib_mw *ibmw) -{ - return container_of(ibmw, struct mlx4_ib_mw, ibmw); -} - -static inline struct mlx4_ib_fast_reg_page_list *to_mfrpl(struct ib_fast_reg_page_list *ibfrpl) -{ - return container_of(ibfrpl, struct mlx4_ib_fast_reg_page_list, ibfrpl); -} - -static inline struct mlx4_ib_fmr *to_mfmr(struct ib_fmr *ibfmr) -{ - return container_of(ibfmr, struct mlx4_ib_fmr, ibfmr); -} - -static inline struct mlx4_ib_flow *to_mflow(struct ib_flow *ibflow) -{ - return container_of(ibflow, struct mlx4_ib_flow, ibflow); -} - -static inline struct mlx4_ib_qp *to_mqp(struct ib_qp *ibqp) -{ - return container_of(ibqp, struct mlx4_ib_qp, ibqp); -} - -static inline struct mlx4_ib_qp *to_mibqp(struct mlx4_qp *mqp) -{ - return container_of(mqp, struct mlx4_ib_qp, mqp); -} - -static inline struct mlx4_ib_srq *to_msrq(struct ib_srq *ibsrq) -{ - return container_of(ibsrq, struct mlx4_ib_srq, ibsrq); -} - -static inline struct mlx4_ib_srq *to_mibsrq(struct mlx4_srq *msrq) -{ - return container_of(msrq, struct mlx4_ib_srq, msrq); -} - -static inline struct mlx4_ib_ah *to_mah(struct ib_ah *ibah) -{ - return container_of(ibah, struct mlx4_ib_ah, ibah); -} - -int mlx4_ib_init_sriov(struct mlx4_ib_dev *dev); -void mlx4_ib_close_sriov(struct mlx4_ib_dev *dev); - -int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context, unsigned long virt, - struct mlx4_db *db); -void mlx4_ib_db_unmap_user(struct mlx4_ib_ucontext *context, struct mlx4_db *db); - -struct ib_mr *mlx4_ib_get_dma_mr(struct ib_pd *pd, int acc); -int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt, - struct ib_umem *umem); -int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, - u64 start_va, - int *num_of_mtts); -struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, - u64 virt_addr, int access_flags, - struct ib_udata *udata, int mr_id); -int mlx4_ib_dereg_mr(struct ib_mr *mr); -struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type); -int mlx4_ib_bind_mw(struct ib_qp *qp, struct ib_mw *mw, - struct ib_mw_bind *mw_bind); -int mlx4_ib_dealloc_mw(struct ib_mw *mw); -struct ib_mr *mlx4_ib_alloc_fast_reg_mr(struct ib_pd *pd, - int max_page_list_len); -struct ib_fast_reg_page_list *mlx4_ib_alloc_fast_reg_page_list(struct ib_device *ibdev, - int page_list_len); -void mlx4_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list); - -int mlx4_ib_modify_cq(struct ib_cq *cq, - struct ib_cq_attr *cq_attr, - int cq_attr_mask); -int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata); -int mlx4_ib_ignore_overrun_cq(struct ib_cq *ibcq); -struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, - struct ib_cq_init_attr *attr, - struct ib_ucontext *context, - struct ib_udata *udata); -int mlx4_ib_destroy_cq(struct ib_cq *cq); -int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); -int mlx4_ib_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); -void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq); -void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq); - -struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr); -int mlx4_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr); -int mlx4_ib_destroy_ah(struct ib_ah *ah); - -struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd, - struct ib_srq_init_attr *init_attr, - struct ib_udata *udata); -int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, - enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); -int mlx4_ib_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr); -int mlx4_ib_destroy_srq(struct ib_srq *srq); -void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index); -int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, - struct ib_recv_wr **bad_wr); - -struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata); -int mlx4_ib_destroy_qp(struct ib_qp *qp); -int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, - int attr_mask, struct ib_udata *udata); -int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, - struct ib_qp_init_attr *qp_init_attr); -int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, - struct ib_send_wr **bad_wr); -int mlx4_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, - struct ib_recv_wr **bad_wr); - -int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int mad_ifc_flags, - int port, struct ib_wc *in_wc, struct ib_grh *in_grh, - void *in_mad, void *response_mad); -int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, - struct ib_wc *in_wc, struct ib_grh *in_grh, - struct ib_mad *in_mad, struct ib_mad *out_mad); -int mlx4_ib_mad_init(struct mlx4_ib_dev *dev); -void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev); - -struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int mr_access_flags, - struct ib_fmr_attr *fmr_attr); -int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, int npages, - u64 iova); -int mlx4_ib_unmap_fmr(struct list_head *fmr_list); -int mlx4_ib_fmr_dealloc(struct ib_fmr *fmr); -int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port, - struct ib_port_attr *props, int netw_view); -int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, - u16 *pkey, int netw_view); - -int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index, - union ib_gid *gid, int netw_view); - -int mlx4_ib_resolve_grh(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah_attr, - u8 *mac, int *is_mcast, u8 port); - -int mlx4_ib_query_if_stat(struct mlx4_ib_dev *dev, u32 counter_index, - union mlx4_counter *counter, u8 clear); - -static inline int mlx4_ib_ah_grh_present(struct mlx4_ib_ah *ah) -{ - u8 port = be32_to_cpu(ah->av.ib.port_pd) >> 24 & 3; - - if (rdma_port_get_link_layer(ah->ibah.device, port) == IB_LINK_LAYER_ETHERNET) - return 1; - - return !!(ah->av.ib.g_slid & 0x80); -} -static inline int mlx4_ib_qp_has_rq(struct ib_qp_init_attr *attr) -{ - if (attr->qp_type == IB_QPT_XRC_INI || attr->qp_type == IB_QPT_XRC_TGT) - return 0; - - return !attr->srq; -} - -int mlx4_ib_mcg_port_init(struct mlx4_ib_demux_ctx *ctx); -void mlx4_ib_mcg_port_cleanup(struct mlx4_ib_demux_ctx *ctx, int destroy_wq); -void clean_vf_mcast(struct mlx4_ib_demux_ctx *ctx, int slave); -int mlx4_ib_mcg_init(void); -void mlx4_ib_mcg_destroy(void); - -int mlx4_ib_find_real_gid(struct ib_device *ibdev, u8 port, __be64 guid); - -int mlx4_ib_mcg_multiplex_handler(struct ib_device *ibdev, int port, int slave, - struct ib_sa_mad *sa_mad); -int mlx4_ib_mcg_demux_handler(struct ib_device *ibdev, int port, int slave, - struct ib_sa_mad *mad); - -int mlx4_ib_add_mc(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp, - union ib_gid *gid); - -void mlx4_ib_dispatch_event(struct mlx4_ib_dev *dev, u8 port_num, - enum ib_event_type type); - -void mlx4_ib_tunnels_update_work(struct work_struct *work); - -int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port, - enum ib_qp_type qpt, struct ib_wc *wc, - struct ib_grh *grh, struct ib_mad *mad); -int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port, - enum ib_qp_type dest_qpt, u16 pkey_index, u32 remote_qpn, - u32 qkey, struct ib_ah_attr *attr, u8 *s_mac, struct ib_mad *mad); -__be64 mlx4_ib_get_new_demux_tid(struct mlx4_ib_demux_ctx *ctx); - -int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave, - struct ib_mad *mad, int is_eth); - -int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id, - struct ib_mad *mad); - -void mlx4_ib_cm_paravirt_init(struct mlx4_ib_dev *dev); -void mlx4_ib_cm_paravirt_clean(struct mlx4_ib_dev *dev, int slave_id); - -/* alias guid support */ -void mlx4_ib_init_alias_guid_work(struct mlx4_ib_dev *dev, int port); -int mlx4_ib_init_alias_guid_service(struct mlx4_ib_dev *dev); -void mlx4_ib_destroy_alias_guid_service(struct mlx4_ib_dev *dev); -void mlx4_ib_invalidate_all_guid_record(struct mlx4_ib_dev *dev, int port); - -void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev, - int block_num, - u8 port_num, u8 *p_data); - -void mlx4_ib_update_cache_on_guid_change(struct mlx4_ib_dev *dev, - int block_num, u8 port_num, - u8 *p_data); - -int add_sysfs_port_mcg_attr(struct mlx4_ib_dev *device, int port_num, - struct attribute *attr); -void del_sysfs_port_mcg_attr(struct mlx4_ib_dev *device, int port_num, - struct attribute *attr); -ib_sa_comp_mask mlx4_ib_get_aguid_comp_mask_from_ix(int index); - -int mlx4_ib_device_register_sysfs(struct mlx4_ib_dev *device) ; - -void mlx4_ib_device_unregister_sysfs(struct mlx4_ib_dev *device); - -__be64 mlx4_ib_gen_node_guid(void); - -int mlx4_ib_steer_qp_alloc(struct mlx4_ib_dev *dev, int count, int *qpn); -void mlx4_ib_steer_qp_free(struct mlx4_ib_dev *dev, u32 qpn, int count); -int mlx4_ib_steer_qp_reg(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp, - int is_attach); -int mlx4_ib_query_device(struct ib_device *ibdev, - struct ib_device_attr *props); - -#endif /* MLX4_IB_H */ Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mlx4_ib.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mad.c =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mad.c (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mad.c (nonexistent) @@ -1,2350 +0,0 @@ -/* - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "mlx4_ib.h" - -enum { - MLX4_IB_VENDOR_CLASS1 = 0x9, - MLX4_IB_VENDOR_CLASS2 = 0xa -}; - -#define MLX4_TUN_SEND_WRID_SHIFT 34 -#define MLX4_TUN_QPN_SHIFT 32 -#define MLX4_TUN_WRID_RECV (((u64) 1) << MLX4_TUN_SEND_WRID_SHIFT) -#define MLX4_TUN_SET_WRID_QPN(a) (((u64) ((a) & 0x3)) << MLX4_TUN_QPN_SHIFT) - -#define MLX4_TUN_IS_RECV(a) (((a) >> MLX4_TUN_SEND_WRID_SHIFT) & 0x1) -#define MLX4_TUN_WRID_QPN(a) (((a) >> MLX4_TUN_QPN_SHIFT) & 0x3) - - /* Port mgmt change event handling */ - -#define GET_BLK_PTR_FROM_EQE(eqe) be32_to_cpu(eqe->event.port_mgmt_change.params.tbl_change_info.block_ptr) -#define GET_MASK_FROM_EQE(eqe) be32_to_cpu(eqe->event.port_mgmt_change.params.tbl_change_info.tbl_entries_mask) -#define NUM_IDX_IN_PKEY_TBL_BLK 32 -#define GUID_TBL_ENTRY_SIZE 8 /* size in bytes */ -#define GUID_TBL_BLK_NUM_ENTRIES 8 -#define GUID_TBL_BLK_SIZE (GUID_TBL_ENTRY_SIZE * GUID_TBL_BLK_NUM_ENTRIES) - -struct mlx4_mad_rcv_buf { - struct ib_grh grh; - u8 payload[256]; -} __packed; - -struct mlx4_mad_snd_buf { - u8 payload[256]; -} __packed; - -struct mlx4_tunnel_mad { - struct ib_grh grh; - struct mlx4_ib_tunnel_header hdr; - struct ib_mad mad; -} __packed; - -struct mlx4_rcv_tunnel_mad { - struct mlx4_rcv_tunnel_hdr hdr; - struct ib_grh grh; - struct ib_mad mad; -} __packed; - -static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u8 port_num); -static void handle_lid_change_event(struct mlx4_ib_dev *dev, u8 port_num); -static void __propagate_pkey_ev(struct mlx4_ib_dev *dev, int port_num, - int block, u32 change_bitmap); - -__be64 mlx4_ib_gen_node_guid(void) -{ -#define NODE_GUID_HI ((u64) (((u64)IB_OPENIB_OUI) << 40)) - return cpu_to_be64(NODE_GUID_HI | random()); -} - -__be64 mlx4_ib_get_new_demux_tid(struct mlx4_ib_demux_ctx *ctx) -{ - return cpu_to_be64(atomic_inc_return(&ctx->tid)) | - cpu_to_be64(0xff00000000000000LL); -} - -int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int mad_ifc_flags, - int port, struct ib_wc *in_wc, struct ib_grh *in_grh, - void *in_mad, void *response_mad) -{ - struct mlx4_cmd_mailbox *inmailbox, *outmailbox; - void *inbox; - int err; - u32 in_modifier = port; - u8 op_modifier = 0; - - inmailbox = mlx4_alloc_cmd_mailbox(dev->dev); - if (IS_ERR(inmailbox)) - return PTR_ERR(inmailbox); - inbox = inmailbox->buf; - - outmailbox = mlx4_alloc_cmd_mailbox(dev->dev); - if (IS_ERR(outmailbox)) { - mlx4_free_cmd_mailbox(dev->dev, inmailbox); - return PTR_ERR(outmailbox); - } - - memcpy(inbox, in_mad, 256); - - /* - * Key check traps can't be generated unless we have in_wc to - * tell us where to send the trap. - */ - if ((mad_ifc_flags & MLX4_MAD_IFC_IGNORE_MKEY) || !in_wc) - op_modifier |= 0x1; - if ((mad_ifc_flags & MLX4_MAD_IFC_IGNORE_BKEY) || !in_wc) - op_modifier |= 0x2; - if (mlx4_is_mfunc(dev->dev) && - (mad_ifc_flags & MLX4_MAD_IFC_NET_VIEW || in_wc)) - op_modifier |= 0x8; - - if (in_wc) { - struct { - __be32 my_qpn; - u32 reserved1; - __be32 rqpn; - u8 sl; - u8 g_path; - u16 reserved2[2]; - __be16 pkey; - u32 reserved3[11]; - u8 grh[40]; - } *ext_info; - - memset(inbox + 256, 0, 256); - ext_info = inbox + 256; - - ext_info->my_qpn = cpu_to_be32(in_wc->qp->qp_num); - ext_info->rqpn = cpu_to_be32(in_wc->src_qp); - ext_info->sl = in_wc->sl << 4; - ext_info->g_path = in_wc->dlid_path_bits | - (in_wc->wc_flags & IB_WC_GRH ? 0x80 : 0); - ext_info->pkey = cpu_to_be16(in_wc->pkey_index); - - if (in_grh) - memcpy(ext_info->grh, in_grh, 40); - - op_modifier |= 0x4; - - in_modifier |= in_wc->slid << 16; - } - - err = mlx4_cmd_box(dev->dev, inmailbox->dma, outmailbox->dma, in_modifier, - mlx4_is_master(dev->dev) ? (op_modifier & ~0x8) : op_modifier, - MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C, - (op_modifier & 0x8) ? MLX4_CMD_NATIVE : MLX4_CMD_WRAPPED); - - if (!err) - memcpy(response_mad, outmailbox->buf, 256); - - mlx4_free_cmd_mailbox(dev->dev, inmailbox); - mlx4_free_cmd_mailbox(dev->dev, outmailbox); - - return err; -} - -static void update_sm_ah(struct mlx4_ib_dev *dev, u8 port_num, u16 lid, u8 sl) -{ - struct ib_ah *new_ah; - struct ib_ah_attr ah_attr; - unsigned long flags; - - if (!dev->send_agent[port_num - 1][0]) - return; - - memset(&ah_attr, 0, sizeof ah_attr); - ah_attr.dlid = lid; - ah_attr.sl = sl; - ah_attr.port_num = port_num; - - new_ah = ib_create_ah(dev->send_agent[port_num - 1][0]->qp->pd, - &ah_attr); - if (IS_ERR(new_ah)) - return; - - spin_lock_irqsave(&dev->sm_lock, flags); - if (dev->sm_ah[port_num - 1]) - ib_destroy_ah(dev->sm_ah[port_num - 1]); - dev->sm_ah[port_num - 1] = new_ah; - spin_unlock_irqrestore(&dev->sm_lock, flags); -} - -/* - * Snoop SM MADs for port info, GUID info, and P_Key table sets, so we can - * synthesize LID change, Client-Rereg, GID change, and P_Key change events. - */ -static void smp_snoop(struct ib_device *ibdev, u8 port_num, struct ib_mad *mad, - u16 prev_lid) -{ - struct ib_port_info *pinfo; - u16 lid; - __be16 *base; - u32 bn, pkey_change_bitmap; - int i; - - - struct mlx4_ib_dev *dev = to_mdev(ibdev); - if ((mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || - mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && - mad->mad_hdr.method == IB_MGMT_METHOD_SET) - switch (mad->mad_hdr.attr_id) { - case IB_SMP_ATTR_PORT_INFO: - pinfo = (struct ib_port_info *) ((struct ib_smp *) mad)->data; - lid = be16_to_cpu(pinfo->lid); - - update_sm_ah(dev, port_num, - be16_to_cpu(pinfo->sm_lid), - pinfo->neighbormtu_mastersmsl & 0xf); - - if (pinfo->clientrereg_resv_subnetto & 0x80) - handle_client_rereg_event(dev, port_num); - - if (prev_lid != lid) - handle_lid_change_event(dev, port_num); - break; - - case IB_SMP_ATTR_PKEY_TABLE: - if (!mlx4_is_mfunc(dev->dev)) { - mlx4_ib_dispatch_event(dev, port_num, - IB_EVENT_PKEY_CHANGE); - break; - } - - /* at this point, we are running in the master. - * Slaves do not receive SMPs. - */ - bn = be32_to_cpu(((struct ib_smp *)mad)->attr_mod) & 0xFFFF; - base = (__be16 *) &(((struct ib_smp *)mad)->data[0]); - pkey_change_bitmap = 0; - for (i = 0; i < 32; i++) { - pr_debug("PKEY[%d] = x%x\n", - i + bn*32, be16_to_cpu(base[i])); - if (be16_to_cpu(base[i]) != - dev->pkeys.phys_pkey_cache[port_num - 1][i + bn*32]) { - pkey_change_bitmap |= (1 << i); - dev->pkeys.phys_pkey_cache[port_num - 1][i + bn*32] = - be16_to_cpu(base[i]); - } - } - pr_debug("PKEY Change event: port=%d, " - "block=0x%x, change_bitmap=0x%x\n", - port_num, bn, pkey_change_bitmap); - - if (pkey_change_bitmap) { - mlx4_ib_dispatch_event(dev, port_num, - IB_EVENT_PKEY_CHANGE); - if (!dev->sriov.is_going_down) - __propagate_pkey_ev(dev, port_num, bn, - pkey_change_bitmap); - } - break; - - case IB_SMP_ATTR_GUID_INFO: - /* paravirtualized master's guid is guid 0 -- does not change */ - if (!mlx4_is_master(dev->dev)) - mlx4_ib_dispatch_event(dev, port_num, - IB_EVENT_GID_CHANGE); - /*if master, notify relevant slaves*/ - if (mlx4_is_master(dev->dev) && - !dev->sriov.is_going_down) { - bn = be32_to_cpu(((struct ib_smp *)mad)->attr_mod); - mlx4_ib_update_cache_on_guid_change(dev, bn, port_num, - (u8 *)(&((struct ib_smp *)mad)->data)); - mlx4_ib_notify_slaves_on_guid_change(dev, bn, port_num, - (u8 *)(&((struct ib_smp *)mad)->data)); - } - break; - - default: - break; - } -} - -static void __propagate_pkey_ev(struct mlx4_ib_dev *dev, int port_num, - int block, u32 change_bitmap) -{ - int i, ix, slave, err; - int have_event = 0; - - for (slave = 0; slave < dev->dev->caps.sqp_demux; slave++) { - if (slave == mlx4_master_func_num(dev->dev)) - continue; - if (!mlx4_is_slave_active(dev->dev, slave)) - continue; - - have_event = 0; - for (i = 0; i < 32; i++) { - if (!(change_bitmap & (1 << i))) - continue; - for (ix = 0; - ix < dev->dev->caps.pkey_table_len[port_num]; ix++) { - if (dev->pkeys.virt2phys_pkey[slave][port_num - 1] - [ix] == i + 32 * block) { - err = mlx4_gen_pkey_eqe(dev->dev, slave, port_num); - pr_debug("propagate_pkey_ev: slave %d," - " port %d, ix %d (%d)\n", - slave, port_num, ix, err); - have_event = 1; - break; - } - } - if (have_event) - break; - } - } -} - -static void node_desc_override(struct ib_device *dev, - struct ib_mad *mad) -{ - unsigned long flags; - - if ((mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || - mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && - mad->mad_hdr.method == IB_MGMT_METHOD_GET_RESP && - mad->mad_hdr.attr_id == IB_SMP_ATTR_NODE_DESC) { - spin_lock_irqsave(&to_mdev(dev)->sm_lock, flags); - memcpy(((struct ib_smp *) mad)->data, dev->node_desc, 64); - spin_unlock_irqrestore(&to_mdev(dev)->sm_lock, flags); - } -} - -static void forward_trap(struct mlx4_ib_dev *dev, u8 port_num, struct ib_mad *mad) -{ - int qpn = mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED; - struct ib_mad_send_buf *send_buf; - struct ib_mad_agent *agent = dev->send_agent[port_num - 1][qpn]; - int ret; - unsigned long flags; - - if (agent) { - send_buf = ib_create_send_mad(agent, qpn, 0, 0, IB_MGMT_MAD_HDR, - IB_MGMT_MAD_DATA, GFP_ATOMIC); - if (IS_ERR(send_buf)) - return; - /* - * We rely here on the fact that MLX QPs don't use the - * address handle after the send is posted (this is - * wrong following the IB spec strictly, but we know - * it's OK for our devices). - */ - spin_lock_irqsave(&dev->sm_lock, flags); - memcpy(send_buf->mad, mad, sizeof *mad); - if ((send_buf->ah = dev->sm_ah[port_num - 1])) - ret = ib_post_send_mad(send_buf, NULL); - else - ret = -EINVAL; - spin_unlock_irqrestore(&dev->sm_lock, flags); - - if (ret) - ib_free_send_mad(send_buf); - } -} - -static int mlx4_ib_demux_sa_handler(struct ib_device *ibdev, int port, int slave, - struct ib_sa_mad *sa_mad) -{ - int ret = 0; - - /* dispatch to different sa handlers */ - switch (be16_to_cpu(sa_mad->mad_hdr.attr_id)) { - case IB_SA_ATTR_MC_MEMBER_REC: - ret = mlx4_ib_mcg_demux_handler(ibdev, port, slave, sa_mad); - break; - default: - break; - } - return ret; -} - -int mlx4_ib_find_real_gid(struct ib_device *ibdev, u8 port, __be64 guid) -{ - struct mlx4_ib_dev *dev = to_mdev(ibdev); - int i; - - for (i = 0; i < dev->dev->caps.sqp_demux; i++) { - if (dev->sriov.demux[port - 1].guid_cache[i] == guid) - return i; - } - return -1; -} - - -static int find_slave_port_pkey_ix(struct mlx4_ib_dev *dev, int slave, - u8 port, u16 pkey, u16 *ix) -{ - int i, ret; - u8 unassigned_pkey_ix, pkey_ix, partial_ix = 0xFF; - u16 slot_pkey; - - if (slave == mlx4_master_func_num(dev->dev)) - return ib_find_cached_pkey(&dev->ib_dev, port, pkey, ix); - - unassigned_pkey_ix = dev->dev->phys_caps.pkey_phys_table_len[port] - 1; - - for (i = 0; i < dev->dev->caps.pkey_table_len[port]; i++) { - if (dev->pkeys.virt2phys_pkey[slave][port - 1][i] == unassigned_pkey_ix) - continue; - - pkey_ix = dev->pkeys.virt2phys_pkey[slave][port - 1][i]; - - ret = ib_get_cached_pkey(&dev->ib_dev, port, pkey_ix, &slot_pkey); - if (ret) - continue; - if ((slot_pkey & 0x7FFF) == (pkey & 0x7FFF)) { - if (slot_pkey & 0x8000) { - *ix = (u16) pkey_ix; - return 0; - } else { - /* take first partial pkey index found */ - if (partial_ix == 0xFF) - partial_ix = pkey_ix; - } - } - } - - if (partial_ix < 0xFF) { - *ix = (u16) partial_ix; - return 0; - } - - return -EINVAL; -} - -int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port, - enum ib_qp_type dest_qpt, struct ib_wc *wc, - struct ib_grh *grh, struct ib_mad *mad) -{ - struct ib_sge list; - struct ib_send_wr wr, *bad_wr; - struct mlx4_ib_demux_pv_ctx *tun_ctx; - struct mlx4_ib_demux_pv_qp *tun_qp; - struct mlx4_rcv_tunnel_mad *tun_mad; - struct ib_ah_attr attr; - struct ib_ah *ah; - struct ib_qp *src_qp = NULL; - unsigned tun_tx_ix = 0; - int dqpn; - int ret = 0; - u16 tun_pkey_ix; - u16 cached_pkey; - u8 is_eth = dev->dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH; - - if (dest_qpt > IB_QPT_GSI) - return -EINVAL; - - tun_ctx = dev->sriov.demux[port-1].tun[slave]; - - /* check if proxy qp created */ - if (!tun_ctx || tun_ctx->state != DEMUX_PV_STATE_ACTIVE) - return -EAGAIN; - - /* QP0 forwarding only for Dom0 */ - if (!dest_qpt && (mlx4_master_func_num(dev->dev) != slave)) - return -EINVAL; - - if (!dest_qpt) - tun_qp = &tun_ctx->qp[0]; - else - tun_qp = &tun_ctx->qp[1]; - - /* compute P_Key index to put in tunnel header for slave */ - if (dest_qpt) { - u16 pkey_ix; - ret = ib_get_cached_pkey(&dev->ib_dev, port, wc->pkey_index, &cached_pkey); - if (ret) - return -EINVAL; - - ret = find_slave_port_pkey_ix(dev, slave, port, cached_pkey, &pkey_ix); - if (ret) - return -EINVAL; - tun_pkey_ix = pkey_ix; - } else - tun_pkey_ix = dev->pkeys.virt2phys_pkey[slave][port - 1][0]; - - dqpn = dev->dev->phys_caps.base_proxy_sqpn + 8 * slave + port + (dest_qpt * 2) - 1; - - /* get tunnel tx data buf for slave */ - src_qp = tun_qp->qp; - - /* create ah. Just need an empty one with the port num for the post send. - * The driver will set the force loopback bit in post_send */ - memset(&attr, 0, sizeof attr); - attr.port_num = port; - if (is_eth) { - memcpy(&attr.grh.dgid.raw[0], &grh->dgid.raw[0], 16); - attr.ah_flags = IB_AH_GRH; - } - ah = ib_create_ah(tun_ctx->pd, &attr); - if (IS_ERR(ah)) - return -ENOMEM; - - /* allocate tunnel tx buf after pass failure returns */ - spin_lock(&tun_qp->tx_lock); - if (tun_qp->tx_ix_head - tun_qp->tx_ix_tail >= - (MLX4_NUM_TUNNEL_BUFS - 1)) - ret = -EAGAIN; - else - tun_tx_ix = (++tun_qp->tx_ix_head) & (MLX4_NUM_TUNNEL_BUFS - 1); - spin_unlock(&tun_qp->tx_lock); - if (ret) - goto out; - - tun_mad = (struct mlx4_rcv_tunnel_mad *) (tun_qp->tx_ring[tun_tx_ix].buf.addr); - if (tun_qp->tx_ring[tun_tx_ix].ah) - ib_destroy_ah(tun_qp->tx_ring[tun_tx_ix].ah); - tun_qp->tx_ring[tun_tx_ix].ah = ah; - ib_dma_sync_single_for_cpu(&dev->ib_dev, - tun_qp->tx_ring[tun_tx_ix].buf.map, - sizeof (struct mlx4_rcv_tunnel_mad), - DMA_TO_DEVICE); - - /* copy over to tunnel buffer */ - if (grh) - memcpy(&tun_mad->grh, grh, sizeof *grh); - memcpy(&tun_mad->mad, mad, sizeof *mad); - - /* adjust tunnel data */ - tun_mad->hdr.pkey_index = cpu_to_be16(tun_pkey_ix); - tun_mad->hdr.flags_src_qp = cpu_to_be32(wc->src_qp & 0xFFFFFF); - tun_mad->hdr.g_ml_path = (grh && (wc->wc_flags & IB_WC_GRH)) ? 0x80 : 0; - - if (is_eth) { - u16 vlan = 0; - if (mlx4_get_slave_default_vlan(dev->dev, port, slave, &vlan, - NULL)) { - if (vlan != wc->vlan_id) - /* VST and default vlan is not the packet vlan drop the - * packet*/ - goto out; - else - /* VST , remove hide the vlan from the VF */ - vlan = 0; - } else { - vlan = wc->vlan_id; - } - - tun_mad->hdr.sl_vid = cpu_to_be16(vlan); - memcpy((char *)&tun_mad->hdr.mac_31_0, &(wc->smac[0]), 4); - memcpy((char *)&tun_mad->hdr.slid_mac_47_32, &(wc->smac[4]), 2); - } else { - tun_mad->hdr.sl_vid = cpu_to_be16(((u16)(wc->sl)) << 12); - tun_mad->hdr.slid_mac_47_32 = cpu_to_be16(wc->slid); - } - - ib_dma_sync_single_for_device(&dev->ib_dev, - tun_qp->tx_ring[tun_tx_ix].buf.map, - sizeof (struct mlx4_rcv_tunnel_mad), - DMA_TO_DEVICE); - - list.addr = tun_qp->tx_ring[tun_tx_ix].buf.map; - list.length = sizeof (struct mlx4_rcv_tunnel_mad); - list.lkey = tun_ctx->mr->lkey; - - wr.wr.ud.ah = ah; - wr.wr.ud.port_num = port; - wr.wr.ud.remote_qkey = IB_QP_SET_QKEY; - wr.wr.ud.remote_qpn = dqpn; - wr.next = NULL; - wr.wr_id = ((u64) tun_tx_ix) | MLX4_TUN_SET_WRID_QPN(dest_qpt); - wr.sg_list = &list; - wr.num_sge = 1; - wr.opcode = IB_WR_SEND; - wr.send_flags = IB_SEND_SIGNALED; - - ret = ib_post_send(src_qp, &wr, &bad_wr); -out: - if (ret) - ib_destroy_ah(ah); - return ret; -} - -static int mlx4_ib_demux_mad(struct ib_device *ibdev, u8 port, - struct ib_wc *wc, struct ib_grh *grh, - struct ib_mad *mad) -{ - struct mlx4_ib_dev *dev = to_mdev(ibdev); - int err; - int slave; - u8 *slave_id; - int is_eth = 0; - - if (rdma_port_get_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND) - is_eth = 0; - else - is_eth = 1; - - if (is_eth) { - if (!(wc->wc_flags & IB_WC_GRH)) { - mlx4_ib_warn(ibdev, "RoCE grh not present.\n"); - return -EINVAL; - } - if (mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_CM) { - mlx4_ib_warn(ibdev, "RoCE mgmt class is not CM\n"); - return -EINVAL; - } - if (mlx4_get_slave_from_roce_gid(dev->dev, port, grh->dgid.raw, &slave)) { - mlx4_ib_warn(ibdev, "failed matching grh\n"); - return -ENOENT; - } - if (slave >= dev->dev->caps.sqp_demux) { - mlx4_ib_warn(ibdev, "slave id: %d is bigger than allowed:%d\n", - slave, dev->dev->caps.sqp_demux); - return -ENOENT; - } - - if (mlx4_ib_demux_cm_handler(ibdev, port, &slave, mad, is_eth)) - return 0; - - err = mlx4_ib_send_to_slave(dev, slave, port, wc->qp->qp_type, wc, grh, mad); - if (err) - pr_debug("failed sending to slave %d via tunnel qp (%d)\n", - slave, err); - return 0; - } - - /* Initially assume that this mad is for us */ - slave = mlx4_master_func_num(dev->dev); - - /* See if the slave id is encoded in a response mad */ - if (mad->mad_hdr.method & 0x80) { - slave_id = (u8 *) &mad->mad_hdr.tid; - slave = *slave_id; - if (slave != 255) /*255 indicates the dom0*/ - *slave_id = 0; /* remap tid */ - } - - /* If a grh is present, we demux according to it */ - if (wc->wc_flags & IB_WC_GRH) { - slave = mlx4_ib_find_real_gid(ibdev, port, grh->dgid.global.interface_id); - if (slave < 0) { - mlx4_ib_warn(ibdev, "failed matching grh\n"); - return -ENOENT; - } - } - /* Class-specific handling */ - switch (mad->mad_hdr.mgmt_class) { - case IB_MGMT_CLASS_SUBN_ADM: - if (mlx4_ib_demux_sa_handler(ibdev, port, slave, - (struct ib_sa_mad *) mad)) - return 0; - break; - case IB_MGMT_CLASS_CM: - if (mlx4_ib_demux_cm_handler(ibdev, port, &slave, mad, is_eth)) - return 0; - break; - case IB_MGMT_CLASS_DEVICE_MGMT: - if (mad->mad_hdr.method != IB_MGMT_METHOD_GET_RESP) - return 0; - break; - default: - /* Drop unsupported classes for slaves in tunnel mode */ - if (slave != mlx4_master_func_num(dev->dev)) { - pr_debug("dropping unsupported ingress mad from class:%d " - "for slave:%d\n", mad->mad_hdr.mgmt_class, slave); - return 0; - } - } - /*make sure that no slave==255 was not handled yet.*/ - if (slave >= dev->dev->caps.sqp_demux) { - mlx4_ib_warn(ibdev, "slave id: %d is bigger than allowed:%d\n", - slave, dev->dev->caps.sqp_demux); - return -ENOENT; - } - - err = mlx4_ib_send_to_slave(dev, slave, port, wc->qp->qp_type, wc, grh, mad); - if (err) - pr_debug("failed sending to slave %d via tunnel qp (%d)\n", - slave, err); - return 0; -} - -static int ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, - struct ib_wc *in_wc, struct ib_grh *in_grh, - struct ib_mad *in_mad, struct ib_mad *out_mad) -{ - u16 slid, prev_lid = 0; - int err; - struct ib_port_attr pattr; - - if (in_wc && in_wc->qp->qp_num) { - pr_debug("received MAD: slid:%d sqpn:%d " - "dlid_bits:%d dqpn:%d wc_flags:0x%x, cls %x, mtd %x, atr %x\n", - in_wc->slid, in_wc->src_qp, - in_wc->dlid_path_bits, - in_wc->qp->qp_num, - in_wc->wc_flags, - in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.method, - be16_to_cpu(in_mad->mad_hdr.attr_id)); - if (in_wc->wc_flags & IB_WC_GRH) { - pr_debug("sgid_hi:0x%016llx sgid_lo:0x%016llx\n", - (unsigned long long)be64_to_cpu(in_grh->sgid.global.subnet_prefix), - (unsigned long long)be64_to_cpu(in_grh->sgid.global.interface_id)); - pr_debug("dgid_hi:0x%016llx dgid_lo:0x%016llx\n", - (unsigned long long)be64_to_cpu(in_grh->dgid.global.subnet_prefix), - (unsigned long long)be64_to_cpu(in_grh->dgid.global.interface_id)); - } - } - - slid = in_wc ? in_wc->slid : be16_to_cpu(IB_LID_PERMISSIVE); - - if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP && slid == 0) { - forward_trap(to_mdev(ibdev), port_num, in_mad); - return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; - } - - if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || - in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) { - if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && - in_mad->mad_hdr.method != IB_MGMT_METHOD_SET && - in_mad->mad_hdr.method != IB_MGMT_METHOD_TRAP_REPRESS) - return IB_MAD_RESULT_SUCCESS; - - /* - * Don't process SMInfo queries -- the SMA can't handle them. - */ - if (in_mad->mad_hdr.attr_id == IB_SMP_ATTR_SM_INFO) - return IB_MAD_RESULT_SUCCESS; - } else if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT || - in_mad->mad_hdr.mgmt_class == MLX4_IB_VENDOR_CLASS1 || - in_mad->mad_hdr.mgmt_class == MLX4_IB_VENDOR_CLASS2 || - in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_CONG_MGMT) { - if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && - in_mad->mad_hdr.method != IB_MGMT_METHOD_SET) - return IB_MAD_RESULT_SUCCESS; - } else - return IB_MAD_RESULT_SUCCESS; - - if ((in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || - in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && - in_mad->mad_hdr.method == IB_MGMT_METHOD_SET && - in_mad->mad_hdr.attr_id == IB_SMP_ATTR_PORT_INFO && - !ib_query_port(ibdev, port_num, &pattr)) - prev_lid = pattr.lid; - - err = mlx4_MAD_IFC(to_mdev(ibdev), - (mad_flags & IB_MAD_IGNORE_MKEY ? MLX4_MAD_IFC_IGNORE_MKEY : 0) | - (mad_flags & IB_MAD_IGNORE_BKEY ? MLX4_MAD_IFC_IGNORE_BKEY : 0) | - MLX4_MAD_IFC_NET_VIEW, - port_num, in_wc, in_grh, in_mad, out_mad); - if (err) - return IB_MAD_RESULT_FAILURE; - - if (!out_mad->mad_hdr.status) { - if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV)) - smp_snoop(ibdev, port_num, in_mad, prev_lid); - /* slaves get node desc from FW */ - if (!mlx4_is_slave(to_mdev(ibdev)->dev)) - node_desc_override(ibdev, out_mad); - } - - /* set return bit in status of directed route responses */ - if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) - out_mad->mad_hdr.status |= cpu_to_be16(1 << 15); - - if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP_REPRESS) - /* no response for trap repress */ - return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; - - return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; -} - -static void edit_counter_ext(struct mlx4_if_stat_extended *cnt, void *counters, - __be16 attr_id) -{ - switch (attr_id) { - case IB_PMA_PORT_COUNTERS: - { - struct ib_pma_portcounters *pma_cnt = - (struct ib_pma_portcounters *)counters; - pma_cnt->port_xmit_data = - cpu_to_be32((be64_to_cpu(cnt->counters[0]. - IfTxUnicastOctets) + - be64_to_cpu(cnt->counters[0]. - IfTxMulticastOctets) + - be64_to_cpu(cnt->counters[0]. - IfTxBroadcastOctets) + - be64_to_cpu(cnt->counters[0]. - IfTxDroppedOctets)) >> 2); - pma_cnt->port_rcv_data = - cpu_to_be32((be64_to_cpu(cnt->counters[0]. - IfRxUnicastOctets) + - be64_to_cpu(cnt->counters[0]. - IfRxMulticastOctets) + - be64_to_cpu(cnt->counters[0]. - IfRxBroadcastOctets) + - be64_to_cpu(cnt->counters[0]. - IfRxNoBufferOctets) + - be64_to_cpu(cnt->counters[0]. - IfRxErrorOctets)) >> 2); - pma_cnt->port_xmit_packets = - cpu_to_be32(be64_to_cpu(cnt->counters[0]. - IfTxUnicastFrames) + - be64_to_cpu(cnt->counters[0]. - IfTxMulticastFrames) + - be64_to_cpu(cnt->counters[0]. - IfTxBroadcastFrames) + - be64_to_cpu(cnt->counters[0]. - IfTxDroppedFrames)); - pma_cnt->port_rcv_packets = - cpu_to_be32(be64_to_cpu(cnt->counters[0]. - IfRxUnicastFrames) + - be64_to_cpu(cnt->counters[0]. - IfRxMulticastFrames) + - be64_to_cpu(cnt->counters[0]. - IfRxBroadcastFrames) + - be64_to_cpu(cnt->counters[0]. - IfRxNoBufferFrames) + - be64_to_cpu(cnt->counters[0]. - IfRxErrorFrames)); - pma_cnt->port_rcv_errors = cpu_to_be32(be64_to_cpu(cnt-> - counters[0]. - IfRxErrorFrames)); - break; - } - - case IB_PMA_PORT_COUNTERS_EXT: - { - struct ib_pma_portcounters_ext *pma_cnt_ext = - (struct ib_pma_portcounters_ext *)counters; - - pma_cnt_ext->port_xmit_data = - cpu_to_be64((be64_to_cpu(cnt->counters[0]. - IfTxUnicastOctets) + - be64_to_cpu(cnt->counters[0]. - IfTxMulticastOctets) + - be64_to_cpu(cnt->counters[0]. - IfTxBroadcastOctets) + - be64_to_cpu(cnt->counters[0]. - IfTxDroppedOctets)) >> 2); - pma_cnt_ext->port_rcv_data = - cpu_to_be64((be64_to_cpu(cnt->counters[0]. - IfRxUnicastOctets) + - be64_to_cpu(cnt->counters[0]. - IfRxMulticastOctets) + - be64_to_cpu(cnt->counters[0]. - IfRxBroadcastOctets) + - be64_to_cpu(cnt->counters[0]. - IfRxNoBufferOctets) + - be64_to_cpu(cnt->counters[0]. - IfRxErrorOctets)) >> 2); - pma_cnt_ext->port_xmit_packets = - cpu_to_be64(be64_to_cpu(cnt->counters[0]. - IfTxUnicastFrames) + - be64_to_cpu(cnt->counters[0]. - IfTxMulticastFrames) + - be64_to_cpu(cnt->counters[0]. - IfTxBroadcastFrames) + - be64_to_cpu(cnt->counters[0]. - IfTxDroppedFrames)); - pma_cnt_ext->port_rcv_packets = - cpu_to_be64(be64_to_cpu(cnt->counters[0]. - IfRxUnicastFrames) + - be64_to_cpu(cnt->counters[0]. - IfRxMulticastFrames) + - be64_to_cpu(cnt->counters[0]. - IfRxBroadcastFrames) + - be64_to_cpu(cnt->counters[0]. - IfRxNoBufferFrames) + - be64_to_cpu(cnt->counters[0]. - IfRxErrorFrames)); - pma_cnt_ext->port_unicast_xmit_packets = cnt->counters[0]. - IfTxUnicastFrames; - pma_cnt_ext->port_unicast_rcv_packets = cnt->counters[0]. - IfRxUnicastFrames; - pma_cnt_ext->port_multicast_xmit_packets = - cpu_to_be64(be64_to_cpu(cnt->counters[0]. - IfTxMulticastFrames) + - be64_to_cpu(cnt->counters[0]. - IfTxBroadcastFrames)); - pma_cnt_ext->port_multicast_rcv_packets = - cpu_to_be64(be64_to_cpu(cnt->counters[0]. - IfTxMulticastFrames) + - be64_to_cpu(cnt->counters[0]. - IfTxBroadcastFrames)); - - break; - } - - default: - pr_warn("Unsupported attr_id 0x%x\n", attr_id); - break; - } - -} - -static void edit_counter(struct mlx4_if_stat_basic *cnt, void *counters, - __be16 attr_id) -{ - switch (attr_id) { - case IB_PMA_PORT_COUNTERS: - { - struct ib_pma_portcounters *pma_cnt = - (struct ib_pma_portcounters *) counters; - pma_cnt->port_xmit_data = - cpu_to_be32(be64_to_cpu( - cnt->counters[0].IfTxOctets) >> 2); - pma_cnt->port_rcv_data = - cpu_to_be32(be64_to_cpu( - cnt->counters[0].IfRxOctets) >> 2); - pma_cnt->port_xmit_packets = - cpu_to_be32(be64_to_cpu(cnt->counters[0].IfTxFrames)); - pma_cnt->port_rcv_packets = - cpu_to_be32(be64_to_cpu(cnt->counters[0].IfRxFrames)); - break; - } - case IB_PMA_PORT_COUNTERS_EXT: - { - struct ib_pma_portcounters_ext *pma_cnt_ext = - (struct ib_pma_portcounters_ext *) counters; - - pma_cnt_ext->port_xmit_data = - cpu_to_be64((be64_to_cpu(cnt->counters[0]. - IfTxOctets) >> 2)); - pma_cnt_ext->port_rcv_data = - cpu_to_be64((be64_to_cpu(cnt->counters[0]. - IfRxOctets) >> 2)); - pma_cnt_ext->port_xmit_packets = cnt->counters[0].IfTxFrames; - pma_cnt_ext->port_rcv_packets = cnt->counters[0].IfRxFrames; - break; - } - default: - pr_warn("Unsupported attr_id 0x%x\n", attr_id); - break; - } -} - -int mlx4_ib_query_if_stat(struct mlx4_ib_dev *dev, u32 counter_index, - union mlx4_counter *counter, u8 clear) -{ - struct mlx4_cmd_mailbox *mailbox; - int err; - u32 inmod = counter_index | ((clear & 1) << 31); - - mailbox = mlx4_alloc_cmd_mailbox(dev->dev); - if (IS_ERR(mailbox)) - return IB_MAD_RESULT_FAILURE; - - err = mlx4_cmd_box(dev->dev, 0, mailbox->dma, inmod, 0, - MLX4_CMD_QUERY_IF_STAT, MLX4_CMD_TIME_CLASS_C, - MLX4_CMD_NATIVE); - if (!err) - memcpy(counter, mailbox->buf, MLX4_IF_STAT_SZ(1)); - - mlx4_free_cmd_mailbox(dev->dev, mailbox); - - return err; -} - -static int iboe_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, - struct ib_wc *in_wc, struct ib_grh *in_grh, - struct ib_mad *in_mad, struct ib_mad *out_mad) -{ - struct mlx4_ib_dev *dev = to_mdev(ibdev); - int err; - u32 counter_index = dev->counters[port_num - 1].counter_index & 0xffff; - u8 mode; - char counter_buf[MLX4_IF_STAT_SZ(1)] __aligned(8); - union mlx4_counter *counter = (union mlx4_counter *) - counter_buf; - - if (in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_PERF_MGMT) - return -EINVAL; - - /* in case of default counter IB shares the counter with ETH */ - /* the state could be -EEXIST or -ENOSPC */ - if (dev->counters[port_num - 1].status) { - memset(out_mad->data, 0, sizeof out_mad->data); - err = IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; - } else { - if (mlx4_ib_query_if_stat(dev, counter_index, counter, 0)) - return IB_MAD_RESULT_FAILURE; - - memset(out_mad->data, 0, sizeof(out_mad->data)); - mode = counter->control.cnt_mode & 0xFF; - err = IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; - switch (mode & 0xf) { - case 0: - edit_counter((void *)counter, - (void *)(out_mad->data + 40), - in_mad->mad_hdr.attr_id); - break; - case 1: - edit_counter_ext((void *)counter, - (void *)(out_mad->data + 40), - in_mad->mad_hdr.attr_id); - break; - default: - err = IB_MAD_RESULT_FAILURE; - } - } - - return err; -} - -int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, - struct ib_wc *in_wc, struct ib_grh *in_grh, - struct ib_mad *in_mad, struct ib_mad *out_mad) -{ - switch (rdma_port_get_link_layer(ibdev, port_num)) { - case IB_LINK_LAYER_INFINIBAND: - return ib_process_mad(ibdev, mad_flags, port_num, in_wc, - in_grh, in_mad, out_mad); - case IB_LINK_LAYER_ETHERNET: - return iboe_process_mad(ibdev, mad_flags, port_num, in_wc, - in_grh, in_mad, out_mad); - default: - return -EINVAL; - } -} - -static void send_handler(struct ib_mad_agent *agent, - struct ib_mad_send_wc *mad_send_wc) -{ - if (mad_send_wc->send_buf->context[0]) - ib_destroy_ah(mad_send_wc->send_buf->context[0]); - ib_free_send_mad(mad_send_wc->send_buf); -} - -int mlx4_ib_mad_init(struct mlx4_ib_dev *dev) -{ - struct ib_mad_agent *agent; - int p, q; - int ret; - enum rdma_link_layer ll; - - for (p = 0; p < dev->num_ports; ++p) { - ll = rdma_port_get_link_layer(&dev->ib_dev, p + 1); - for (q = 0; q <= 1; ++q) { - if (ll == IB_LINK_LAYER_INFINIBAND) { - agent = ib_register_mad_agent(&dev->ib_dev, p + 1, - q ? IB_QPT_GSI : IB_QPT_SMI, - NULL, 0, send_handler, - NULL, NULL); - if (IS_ERR(agent)) { - ret = PTR_ERR(agent); - goto err; - } - dev->send_agent[p][q] = agent; - } else - dev->send_agent[p][q] = NULL; - } - } - - return 0; - -err: - for (p = 0; p < dev->num_ports; ++p) - for (q = 0; q <= 1; ++q) - if (dev->send_agent[p][q]) - ib_unregister_mad_agent(dev->send_agent[p][q]); - - return ret; -} - -void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev) -{ - struct ib_mad_agent *agent; - int p, q; - - for (p = 0; p < dev->num_ports; ++p) { - for (q = 0; q <= 1; ++q) { - agent = dev->send_agent[p][q]; - if (agent) { - dev->send_agent[p][q] = NULL; - ib_unregister_mad_agent(agent); - } - } - - if (dev->sm_ah[p]) - ib_destroy_ah(dev->sm_ah[p]); - } -} - -static void handle_lid_change_event(struct mlx4_ib_dev *dev, u8 port_num) -{ - mlx4_ib_dispatch_event(dev, port_num, IB_EVENT_LID_CHANGE); - - if (mlx4_is_master(dev->dev) && !dev->sriov.is_going_down) - mlx4_gen_slaves_port_mgt_ev(dev->dev, port_num, - MLX4_EQ_PORT_INFO_LID_CHANGE_MASK, 0, 0); -} - -static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u8 port_num) -{ - /* re-configure the alias-guid and mcg's */ - if (mlx4_is_master(dev->dev)) { - mlx4_ib_invalidate_all_guid_record(dev, port_num); - - if (!dev->sriov.is_going_down) { - mlx4_ib_mcg_port_cleanup(&dev->sriov.demux[port_num - 1], 0); - mlx4_gen_slaves_port_mgt_ev(dev->dev, port_num, - MLX4_EQ_PORT_INFO_CLIENT_REREG_MASK, 0, 0); - } - } - mlx4_ib_dispatch_event(dev, port_num, IB_EVENT_CLIENT_REREGISTER); -} - -static void propagate_pkey_ev(struct mlx4_ib_dev *dev, int port_num, - struct mlx4_eqe *eqe) -{ - __propagate_pkey_ev(dev, port_num, GET_BLK_PTR_FROM_EQE(eqe), - GET_MASK_FROM_EQE(eqe)); -} - -static void handle_slaves_guid_change(struct mlx4_ib_dev *dev, u8 port_num, - u32 guid_tbl_blk_num, u32 change_bitmap) -{ - struct ib_smp *in_mad = NULL; - struct ib_smp *out_mad = NULL; - u16 i; - - if (!mlx4_is_mfunc(dev->dev) || !mlx4_is_master(dev->dev)) - return; - - in_mad = kmalloc(sizeof *in_mad, GFP_KERNEL); - out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); - if (!in_mad || !out_mad) { - mlx4_ib_warn(&dev->ib_dev, "failed to allocate memory for guid info mads\n"); - goto out; - } - - guid_tbl_blk_num *= 4; - - for (i = 0; i < 4; i++) { - if (change_bitmap && (!((change_bitmap >> (8 * i)) & 0xff))) - continue; - memset(in_mad, 0, sizeof *in_mad); - memset(out_mad, 0, sizeof *out_mad); - - in_mad->base_version = 1; - in_mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; - in_mad->class_version = 1; - in_mad->method = IB_MGMT_METHOD_GET; - in_mad->attr_id = IB_SMP_ATTR_GUID_INFO; - in_mad->attr_mod = cpu_to_be32(guid_tbl_blk_num + i); - - if (mlx4_MAD_IFC(dev, - MLX4_MAD_IFC_IGNORE_KEYS | MLX4_MAD_IFC_NET_VIEW, - port_num, NULL, NULL, in_mad, out_mad)) { - mlx4_ib_warn(&dev->ib_dev, "Failed in get GUID INFO MAD_IFC\n"); - goto out; - } - - mlx4_ib_update_cache_on_guid_change(dev, guid_tbl_blk_num + i, - port_num, - (u8 *)(&((struct ib_smp *)out_mad)->data)); - mlx4_ib_notify_slaves_on_guid_change(dev, guid_tbl_blk_num + i, - port_num, - (u8 *)(&((struct ib_smp *)out_mad)->data)); - } - -out: - kfree(in_mad); - kfree(out_mad); - return; -} - -void handle_port_mgmt_change_event(struct work_struct *work) -{ - struct ib_event_work *ew = container_of(work, struct ib_event_work, work); - struct mlx4_ib_dev *dev = ew->ib_dev; - struct mlx4_eqe *eqe = &(ew->ib_eqe); - u8 port = eqe->event.port_mgmt_change.port; - u32 changed_attr; - u32 tbl_block; - u32 change_bitmap; - - switch (eqe->subtype) { - case MLX4_DEV_PMC_SUBTYPE_PORT_INFO: - changed_attr = be32_to_cpu(eqe->event.port_mgmt_change.params.port_info.changed_attr); - - /* Update the SM ah - This should be done before handling - the other changed attributes so that MADs can be sent to the SM */ - if (changed_attr & MSTR_SM_CHANGE_MASK) { - u16 lid = be16_to_cpu(eqe->event.port_mgmt_change.params.port_info.mstr_sm_lid); - u8 sl = eqe->event.port_mgmt_change.params.port_info.mstr_sm_sl & 0xf; - update_sm_ah(dev, port, lid, sl); - mlx4_ib_dispatch_event(dev, port, IB_EVENT_SM_CHANGE); - if (mlx4_is_master(dev->dev)) - mlx4_gen_slaves_port_mgt_ev(dev->dev, port, - changed_attr & MSTR_SM_CHANGE_MASK, - lid, sl); - } - - /* Check if it is a lid change event */ - if (changed_attr & MLX4_EQ_PORT_INFO_LID_CHANGE_MASK) - handle_lid_change_event(dev, port); - - /* Generate GUID changed event */ - if (changed_attr & MLX4_EQ_PORT_INFO_GID_PFX_CHANGE_MASK) { - mlx4_ib_dispatch_event(dev, port, IB_EVENT_GID_CHANGE); - /*if master, notify all slaves*/ - if (mlx4_is_master(dev->dev)) - mlx4_gen_slaves_port_mgt_ev(dev->dev, port, - MLX4_EQ_PORT_INFO_GID_PFX_CHANGE_MASK, 0, 0); - } - - if (changed_attr & MLX4_EQ_PORT_INFO_CLIENT_REREG_MASK) - handle_client_rereg_event(dev, port); - break; - - case MLX4_DEV_PMC_SUBTYPE_PKEY_TABLE: - mlx4_ib_dispatch_event(dev, port, IB_EVENT_PKEY_CHANGE); - if (mlx4_is_master(dev->dev) && !dev->sriov.is_going_down) - propagate_pkey_ev(dev, port, eqe); - break; - case MLX4_DEV_PMC_SUBTYPE_GUID_INFO: - /* paravirtualized master's guid is guid 0 -- does not change */ - if (!mlx4_is_master(dev->dev)) - mlx4_ib_dispatch_event(dev, port, IB_EVENT_GID_CHANGE); - /*if master, notify relevant slaves*/ - else if (!dev->sriov.is_going_down) { - tbl_block = GET_BLK_PTR_FROM_EQE(eqe); - change_bitmap = GET_MASK_FROM_EQE(eqe); - handle_slaves_guid_change(dev, port, tbl_block, change_bitmap); - } - break; - default: - pr_warn("Unsupported subtype 0x%x for " - "Port Management Change event\n", eqe->subtype); - } - - kfree(ew); -} - -void mlx4_ib_dispatch_event(struct mlx4_ib_dev *dev, u8 port_num, - enum ib_event_type type) -{ - struct ib_event event; - - event.device = &dev->ib_dev; - event.element.port_num = port_num; - event.event = type; - - ib_dispatch_event(&event); -} - -static void mlx4_ib_tunnel_comp_handler(struct ib_cq *cq, void *arg) -{ - unsigned long flags; - struct mlx4_ib_demux_pv_ctx *ctx = cq->cq_context; - struct mlx4_ib_dev *dev = to_mdev(ctx->ib_dev); - spin_lock_irqsave(&dev->sriov.going_down_lock, flags); - if (!dev->sriov.is_going_down && ctx->state == DEMUX_PV_STATE_ACTIVE) - queue_work(ctx->wq, &ctx->work); - spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); -} - -static int mlx4_ib_post_pv_qp_buf(struct mlx4_ib_demux_pv_ctx *ctx, - struct mlx4_ib_demux_pv_qp *tun_qp, - int index) -{ - struct ib_sge sg_list; - struct ib_recv_wr recv_wr, *bad_recv_wr; - int size; - - size = (tun_qp->qp->qp_type == IB_QPT_UD) ? - sizeof (struct mlx4_tunnel_mad) : sizeof (struct mlx4_mad_rcv_buf); - - sg_list.addr = tun_qp->ring[index].map; - sg_list.length = size; - sg_list.lkey = ctx->mr->lkey; - - recv_wr.next = NULL; - recv_wr.sg_list = &sg_list; - recv_wr.num_sge = 1; - recv_wr.wr_id = (u64) index | MLX4_TUN_WRID_RECV | - MLX4_TUN_SET_WRID_QPN(tun_qp->proxy_qpt); - ib_dma_sync_single_for_device(ctx->ib_dev, tun_qp->ring[index].map, - size, DMA_FROM_DEVICE); - return ib_post_recv(tun_qp->qp, &recv_wr, &bad_recv_wr); -} - -static int mlx4_ib_multiplex_sa_handler(struct ib_device *ibdev, int port, - int slave, struct ib_sa_mad *sa_mad) -{ - int ret = 0; - - /* dispatch to different sa handlers */ - switch (be16_to_cpu(sa_mad->mad_hdr.attr_id)) { - case IB_SA_ATTR_MC_MEMBER_REC: - ret = mlx4_ib_mcg_multiplex_handler(ibdev, port, slave, sa_mad); - break; - default: - break; - } - return ret; -} - -static int is_proxy_qp0(struct mlx4_ib_dev *dev, int qpn, int slave) -{ - int proxy_start = dev->dev->phys_caps.base_proxy_sqpn + 8 * slave; - - return (qpn >= proxy_start && qpn <= proxy_start + 1); -} - - -int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port, - enum ib_qp_type dest_qpt, u16 pkey_index, - u32 remote_qpn, u32 qkey, struct ib_ah_attr *attr, - u8 *s_mac, struct ib_mad *mad) -{ - struct ib_sge list; - struct ib_send_wr wr, *bad_wr; - struct mlx4_ib_demux_pv_ctx *sqp_ctx; - struct mlx4_ib_demux_pv_qp *sqp; - struct mlx4_mad_snd_buf *sqp_mad; - struct ib_ah *ah; - struct ib_qp *send_qp = NULL; - unsigned wire_tx_ix = 0; - int ret = 0; - u16 wire_pkey_ix; - int src_qpnum; - u8 sgid_index; - - - sqp_ctx = dev->sriov.sqps[port-1]; - - /* check if proxy qp created */ - if (!sqp_ctx || sqp_ctx->state != DEMUX_PV_STATE_ACTIVE) - return -EAGAIN; - - /* QP0 forwarding only for Dom0 */ - if (dest_qpt == IB_QPT_SMI && (mlx4_master_func_num(dev->dev) != slave)) - return -EINVAL; - - if (dest_qpt == IB_QPT_SMI) { - src_qpnum = 0; - sqp = &sqp_ctx->qp[0]; - wire_pkey_ix = dev->pkeys.virt2phys_pkey[slave][port - 1][0]; - } else { - src_qpnum = 1; - sqp = &sqp_ctx->qp[1]; - wire_pkey_ix = dev->pkeys.virt2phys_pkey[slave][port - 1][pkey_index]; - } - - send_qp = sqp->qp; - - /* create ah */ - sgid_index = attr->grh.sgid_index; - attr->grh.sgid_index = 0; - ah = ib_create_ah(sqp_ctx->pd, attr); - if (IS_ERR(ah)) - return -ENOMEM; - attr->grh.sgid_index = sgid_index; - to_mah(ah)->av.ib.gid_index = sgid_index; - /* get rid of force-loopback bit */ - to_mah(ah)->av.ib.port_pd &= cpu_to_be32(0x7FFFFFFF); - spin_lock(&sqp->tx_lock); - if (sqp->tx_ix_head - sqp->tx_ix_tail >= - (MLX4_NUM_TUNNEL_BUFS - 1)) - ret = -EAGAIN; - else - wire_tx_ix = (++sqp->tx_ix_head) & (MLX4_NUM_TUNNEL_BUFS - 1); - spin_unlock(&sqp->tx_lock); - if (ret) - goto out; - - sqp_mad = (struct mlx4_mad_snd_buf *) (sqp->tx_ring[wire_tx_ix].buf.addr); - if (sqp->tx_ring[wire_tx_ix].ah) - ib_destroy_ah(sqp->tx_ring[wire_tx_ix].ah); - sqp->tx_ring[wire_tx_ix].ah = ah; - ib_dma_sync_single_for_cpu(&dev->ib_dev, - sqp->tx_ring[wire_tx_ix].buf.map, - sizeof (struct mlx4_mad_snd_buf), - DMA_TO_DEVICE); - - memcpy(&sqp_mad->payload, mad, sizeof *mad); - - ib_dma_sync_single_for_device(&dev->ib_dev, - sqp->tx_ring[wire_tx_ix].buf.map, - sizeof (struct mlx4_mad_snd_buf), - DMA_TO_DEVICE); - - list.addr = sqp->tx_ring[wire_tx_ix].buf.map; - list.length = sizeof (struct mlx4_mad_snd_buf); - list.lkey = sqp_ctx->mr->lkey; - - wr.wr.ud.ah = ah; - wr.wr.ud.port_num = port; - wr.wr.ud.pkey_index = wire_pkey_ix; - wr.wr.ud.remote_qkey = qkey; - wr.wr.ud.remote_qpn = remote_qpn; - wr.next = NULL; - wr.wr_id = ((u64) wire_tx_ix) | MLX4_TUN_SET_WRID_QPN(src_qpnum); - wr.sg_list = &list; - wr.num_sge = 1; - wr.opcode = IB_WR_SEND; - wr.send_flags = IB_SEND_SIGNALED; - if (s_mac) - memcpy(to_mah(ah)->av.eth.s_mac, s_mac, 6); - - - ret = ib_post_send(send_qp, &wr, &bad_wr); -out: - if (ret) - ib_destroy_ah(ah); - return ret; -} - -static int get_slave_base_gid_ix(struct mlx4_ib_dev *dev, int slave, int port) -{ - int gids; - int vfs; - - if (rdma_port_get_link_layer(&dev->ib_dev, port) == IB_LINK_LAYER_INFINIBAND) - return slave; - - gids = MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS; - vfs = dev->dev->num_vfs; - - if (slave == 0) - return 0; - if (slave <= gids % vfs) - return MLX4_ROCE_PF_GIDS + ((gids / vfs) + 1) * (slave - 1); - - return MLX4_ROCE_PF_GIDS + (gids % vfs) + ((gids / vfs) * (slave - 1)); -} - -static int get_real_sgid_index(struct mlx4_ib_dev *dev, int slave, int port, - struct ib_ah_attr *ah_attr) -{ - if (rdma_port_get_link_layer(&dev->ib_dev, port) == IB_LINK_LAYER_INFINIBAND) { - ah_attr->grh.sgid_index = slave; - return 0; - } - ah_attr->grh.sgid_index += get_slave_base_gid_ix(dev, slave, port); - return 0; -} - -static void mlx4_ib_multiplex_mad(struct mlx4_ib_demux_pv_ctx *ctx, struct ib_wc *wc) -{ - struct mlx4_ib_dev *dev = to_mdev(ctx->ib_dev); - struct mlx4_ib_demux_pv_qp *tun_qp = &ctx->qp[MLX4_TUN_WRID_QPN(wc->wr_id)]; - int wr_ix = wc->wr_id & (MLX4_NUM_TUNNEL_BUFS - 1); - struct mlx4_tunnel_mad *tunnel = tun_qp->ring[wr_ix].addr; - struct mlx4_ib_ah ah; - struct ib_ah_attr ah_attr; - u8 *slave_id; - int slave; - - /* Get slave that sent this packet */ - if (wc->src_qp < dev->dev->phys_caps.base_proxy_sqpn || - wc->src_qp >= dev->dev->phys_caps.base_proxy_sqpn + 8 * MLX4_MFUNC_MAX || - (wc->src_qp & 0x1) != ctx->port - 1 || - wc->src_qp & 0x4) { - mlx4_ib_warn(ctx->ib_dev, "can't multiplex bad sqp:%d\n", wc->src_qp); - return; - } - slave = ((wc->src_qp & ~0x7) - dev->dev->phys_caps.base_proxy_sqpn) / 8; - if (slave != ctx->slave) { - mlx4_ib_warn(ctx->ib_dev, "can't multiplex bad sqp:%d: " - "belongs to another slave\n", wc->src_qp); - return; - } - if (slave != mlx4_master_func_num(dev->dev) && !(wc->src_qp & 0x2)) { - mlx4_ib_warn(ctx->ib_dev, "can't multiplex bad sqp:%d: " - "non-master trying to send QP0 packets\n", wc->src_qp); - return; - } - - /* Map transaction ID */ - ib_dma_sync_single_for_cpu(ctx->ib_dev, tun_qp->ring[wr_ix].map, - sizeof (struct mlx4_tunnel_mad), - DMA_FROM_DEVICE); - switch (tunnel->mad.mad_hdr.method) { - case IB_MGMT_METHOD_SET: - case IB_MGMT_METHOD_GET: - case IB_MGMT_METHOD_REPORT: - case IB_SA_METHOD_GET_TABLE: - case IB_SA_METHOD_DELETE: - case IB_SA_METHOD_GET_MULTI: - case IB_SA_METHOD_GET_TRACE_TBL: - slave_id = (u8 *) &tunnel->mad.mad_hdr.tid; - if (*slave_id) { - mlx4_ib_warn(ctx->ib_dev, "egress mad has non-null tid msb:%d " - "class:%d slave:%d\n", *slave_id, - tunnel->mad.mad_hdr.mgmt_class, slave); - return; - } else - *slave_id = slave; - default: - /* nothing */; - } - - /* Class-specific handling */ - switch (tunnel->mad.mad_hdr.mgmt_class) { - case IB_MGMT_CLASS_SUBN_ADM: - if (mlx4_ib_multiplex_sa_handler(ctx->ib_dev, ctx->port, slave, - (struct ib_sa_mad *) &tunnel->mad)) - return; - break; - case IB_MGMT_CLASS_CM: - if (mlx4_ib_multiplex_cm_handler(ctx->ib_dev, ctx->port, slave, - (struct ib_mad *) &tunnel->mad)) - return; - break; - case IB_MGMT_CLASS_DEVICE_MGMT: - if (tunnel->mad.mad_hdr.method != IB_MGMT_METHOD_GET && - tunnel->mad.mad_hdr.method != IB_MGMT_METHOD_SET) - return; - break; - default: - /* Drop unsupported classes for slaves in tunnel mode */ - if (slave != mlx4_master_func_num(dev->dev)) { - mlx4_ib_warn(ctx->ib_dev, "dropping unsupported egress mad from class:%d " - "for slave:%d\n", tunnel->mad.mad_hdr.mgmt_class, slave); - return; - } - } - - /* We are using standard ib_core services to send the mad, so generate a - * stadard address handle by decoding the tunnelled mlx4_ah fields */ - memcpy(&ah.av, &tunnel->hdr.av, sizeof (struct mlx4_av)); - ah.ibah.device = ctx->ib_dev; - mlx4_ib_query_ah(&ah.ibah, &ah_attr); - if (ah_attr.ah_flags & IB_AH_GRH) - if (get_real_sgid_index(dev, slave, ctx->port, &ah_attr)) - return; - memcpy(ah_attr.dmac, tunnel->hdr.mac, 6); - ah_attr.vlan_id = tunnel->hdr.vlan; - /* if slave have default vlan use it */ - mlx4_get_slave_default_vlan(dev->dev, ctx->port, slave, - &ah_attr.vlan_id, &ah_attr.sl); - - mlx4_ib_send_to_wire(dev, slave, ctx->port, - is_proxy_qp0(dev, wc->src_qp, slave) ? - IB_QPT_SMI : IB_QPT_GSI, - be16_to_cpu(tunnel->hdr.pkey_index), - be32_to_cpu(tunnel->hdr.remote_qpn), - be32_to_cpu(tunnel->hdr.qkey), - &ah_attr, wc->smac, &tunnel->mad); -} - -static int mlx4_ib_alloc_pv_bufs(struct mlx4_ib_demux_pv_ctx *ctx, - enum ib_qp_type qp_type, int is_tun) -{ - int i; - struct mlx4_ib_demux_pv_qp *tun_qp; - int rx_buf_size, tx_buf_size; - - if (qp_type > IB_QPT_GSI) - return -EINVAL; - - tun_qp = &ctx->qp[qp_type]; - - tun_qp->ring = kzalloc(sizeof (struct mlx4_ib_buf) * MLX4_NUM_TUNNEL_BUFS, - GFP_KERNEL); - if (!tun_qp->ring) - return -ENOMEM; - - tun_qp->tx_ring = kcalloc(MLX4_NUM_TUNNEL_BUFS, - sizeof (struct mlx4_ib_tun_tx_buf), - GFP_KERNEL); - if (!tun_qp->tx_ring) { - kfree(tun_qp->ring); - tun_qp->ring = NULL; - return -ENOMEM; - } - - if (is_tun) { - rx_buf_size = sizeof (struct mlx4_tunnel_mad); - tx_buf_size = sizeof (struct mlx4_rcv_tunnel_mad); - } else { - rx_buf_size = sizeof (struct mlx4_mad_rcv_buf); - tx_buf_size = sizeof (struct mlx4_mad_snd_buf); - } - - for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { - tun_qp->ring[i].addr = kmalloc(rx_buf_size, GFP_KERNEL); - if (!tun_qp->ring[i].addr) - goto err; - tun_qp->ring[i].map = ib_dma_map_single(ctx->ib_dev, - tun_qp->ring[i].addr, - rx_buf_size, - DMA_FROM_DEVICE); - if (unlikely(ib_dma_mapping_error(ctx->ib_dev, - tun_qp->ring[i].map))) { - mlx4_ib_warn(ctx->ib_dev, "ib_dma_map_single failed\n"); - kfree(tun_qp->ring[i].addr); - goto err; - } - } - - for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { - tun_qp->tx_ring[i].buf.addr = - kmalloc(tx_buf_size, GFP_KERNEL); - if (!tun_qp->tx_ring[i].buf.addr) - goto tx_err; - tun_qp->tx_ring[i].buf.map = - ib_dma_map_single(ctx->ib_dev, - tun_qp->tx_ring[i].buf.addr, - tx_buf_size, - DMA_TO_DEVICE); - if (unlikely(ib_dma_mapping_error(ctx->ib_dev, - tun_qp->tx_ring[i].buf.map))) { - mlx4_ib_warn(ctx->ib_dev, "ib_dma_map_single failed\n"); - kfree(tun_qp->tx_ring[i].buf.addr); - goto tx_err; - } - tun_qp->tx_ring[i].ah = NULL; - } - spin_lock_init(&tun_qp->tx_lock); - tun_qp->tx_ix_head = 0; - tun_qp->tx_ix_tail = 0; - tun_qp->proxy_qpt = qp_type; - - return 0; - -tx_err: - while (i > 0) { - --i; - ib_dma_unmap_single(ctx->ib_dev, tun_qp->tx_ring[i].buf.map, - tx_buf_size, DMA_TO_DEVICE); - kfree(tun_qp->tx_ring[i].buf.addr); - } - kfree(tun_qp->tx_ring); - tun_qp->tx_ring = NULL; - i = MLX4_NUM_TUNNEL_BUFS; -err: - while (i > 0) { - --i; - ib_dma_unmap_single(ctx->ib_dev, tun_qp->ring[i].map, - rx_buf_size, DMA_FROM_DEVICE); - kfree(tun_qp->ring[i].addr); - } - kfree(tun_qp->ring); - tun_qp->ring = NULL; - return -ENOMEM; -} - -static void mlx4_ib_free_pv_qp_bufs(struct mlx4_ib_demux_pv_ctx *ctx, - enum ib_qp_type qp_type, int is_tun) -{ - int i; - struct mlx4_ib_demux_pv_qp *tun_qp; - int rx_buf_size, tx_buf_size; - - if (qp_type > IB_QPT_GSI) - return; - - tun_qp = &ctx->qp[qp_type]; - if (is_tun) { - rx_buf_size = sizeof (struct mlx4_tunnel_mad); - tx_buf_size = sizeof (struct mlx4_rcv_tunnel_mad); - } else { - rx_buf_size = sizeof (struct mlx4_mad_rcv_buf); - tx_buf_size = sizeof (struct mlx4_mad_snd_buf); - } - - - for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { - ib_dma_unmap_single(ctx->ib_dev, tun_qp->ring[i].map, - rx_buf_size, DMA_FROM_DEVICE); - kfree(tun_qp->ring[i].addr); - } - - for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { - ib_dma_unmap_single(ctx->ib_dev, tun_qp->tx_ring[i].buf.map, - tx_buf_size, DMA_TO_DEVICE); - kfree(tun_qp->tx_ring[i].buf.addr); - if (tun_qp->tx_ring[i].ah) - ib_destroy_ah(tun_qp->tx_ring[i].ah); - } - kfree(tun_qp->tx_ring); - kfree(tun_qp->ring); -} - -static void mlx4_ib_tunnel_comp_worker(struct work_struct *work) -{ - struct mlx4_ib_demux_pv_ctx *ctx; - struct mlx4_ib_demux_pv_qp *tun_qp; - struct ib_wc wc; - int ret; - ctx = container_of(work, struct mlx4_ib_demux_pv_ctx, work); - ib_req_notify_cq(ctx->cq, IB_CQ_NEXT_COMP); - - while (ib_poll_cq(ctx->cq, 1, &wc) == 1) { - tun_qp = &ctx->qp[MLX4_TUN_WRID_QPN(wc.wr_id)]; - if (wc.status == IB_WC_SUCCESS) { - switch (wc.opcode) { - case IB_WC_RECV: - mlx4_ib_multiplex_mad(ctx, &wc); - ret = mlx4_ib_post_pv_qp_buf(ctx, tun_qp, - wc.wr_id & - (MLX4_NUM_TUNNEL_BUFS - 1)); - if (ret) - pr_err("Failed reposting tunnel " - "buf:%lld\n", (unsigned long long)wc.wr_id); - break; - case IB_WC_SEND: - pr_debug("received tunnel send completion:" - "wrid=0x%llx, status=0x%x\n", - (unsigned long long)wc.wr_id, wc.status); - ib_destroy_ah(tun_qp->tx_ring[wc.wr_id & - (MLX4_NUM_TUNNEL_BUFS - 1)].ah); - tun_qp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah - = NULL; - spin_lock(&tun_qp->tx_lock); - tun_qp->tx_ix_tail++; - spin_unlock(&tun_qp->tx_lock); - - break; - default: - break; - } - } else { - pr_debug("mlx4_ib: completion error in tunnel: %d." - " status = %d, wrid = 0x%llx\n", - ctx->slave, wc.status, (unsigned long long)wc.wr_id); - if (!MLX4_TUN_IS_RECV(wc.wr_id)) { - ib_destroy_ah(tun_qp->tx_ring[wc.wr_id & - (MLX4_NUM_TUNNEL_BUFS - 1)].ah); - tun_qp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah - = NULL; - spin_lock(&tun_qp->tx_lock); - tun_qp->tx_ix_tail++; - spin_unlock(&tun_qp->tx_lock); - } - } - } -} - -static void pv_qp_event_handler(struct ib_event *event, void *qp_context) -{ - struct mlx4_ib_demux_pv_ctx *sqp = qp_context; - - /* It's worse than that! He's dead, Jim! */ - pr_err("Fatal error (%d) on a MAD QP on port %d\n", - event->event, sqp->port); -} - -static int create_pv_sqp(struct mlx4_ib_demux_pv_ctx *ctx, - enum ib_qp_type qp_type, int create_tun) -{ - int i, ret; - struct mlx4_ib_demux_pv_qp *tun_qp; - struct mlx4_ib_qp_tunnel_init_attr qp_init_attr; - struct ib_qp_attr attr; - int qp_attr_mask_INIT; - - if (qp_type > IB_QPT_GSI) - return -EINVAL; - - tun_qp = &ctx->qp[qp_type]; - - memset(&qp_init_attr, 0, sizeof qp_init_attr); - qp_init_attr.init_attr.send_cq = ctx->cq; - qp_init_attr.init_attr.recv_cq = ctx->cq; - qp_init_attr.init_attr.sq_sig_type = IB_SIGNAL_ALL_WR; - qp_init_attr.init_attr.cap.max_send_wr = MLX4_NUM_TUNNEL_BUFS; - qp_init_attr.init_attr.cap.max_recv_wr = MLX4_NUM_TUNNEL_BUFS; - qp_init_attr.init_attr.cap.max_send_sge = 1; - qp_init_attr.init_attr.cap.max_recv_sge = 1; - if (create_tun) { - qp_init_attr.init_attr.qp_type = IB_QPT_UD; - qp_init_attr.init_attr.create_flags = (enum ib_qp_create_flags)MLX4_IB_SRIOV_TUNNEL_QP; - qp_init_attr.port = ctx->port; - qp_init_attr.slave = ctx->slave; - qp_init_attr.proxy_qp_type = qp_type; - qp_attr_mask_INIT = IB_QP_STATE | IB_QP_PKEY_INDEX | - IB_QP_QKEY | IB_QP_PORT; - } else { - qp_init_attr.init_attr.qp_type = qp_type; - qp_init_attr.init_attr.create_flags = (enum ib_qp_create_flags)MLX4_IB_SRIOV_SQP; - qp_attr_mask_INIT = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_QKEY; - } - qp_init_attr.init_attr.port_num = ctx->port; - qp_init_attr.init_attr.qp_context = ctx; - qp_init_attr.init_attr.event_handler = pv_qp_event_handler; - tun_qp->qp = ib_create_qp(ctx->pd, &qp_init_attr.init_attr); - if (IS_ERR(tun_qp->qp)) { - ret = PTR_ERR(tun_qp->qp); - tun_qp->qp = NULL; - pr_err("Couldn't create %s QP (%d)\n", - create_tun ? "tunnel" : "special", ret); - return ret; - } - - memset(&attr, 0, sizeof attr); - attr.qp_state = IB_QPS_INIT; - ret = 0; - if (create_tun) - ret = find_slave_port_pkey_ix(to_mdev(ctx->ib_dev), ctx->slave, - ctx->port, 0xFFFF, &attr.pkey_index); - if (ret || !create_tun) - attr.pkey_index = - to_mdev(ctx->ib_dev)->pkeys.virt2phys_pkey[ctx->slave][ctx->port - 1][0]; - attr.qkey = IB_QP1_QKEY; - attr.port_num = ctx->port; - ret = ib_modify_qp(tun_qp->qp, &attr, qp_attr_mask_INIT); - if (ret) { - pr_err("Couldn't change %s qp state to INIT (%d)\n", - create_tun ? "tunnel" : "special", ret); - goto err_qp; - } - attr.qp_state = IB_QPS_RTR; - ret = ib_modify_qp(tun_qp->qp, &attr, IB_QP_STATE); - if (ret) { - pr_err("Couldn't change %s qp state to RTR (%d)\n", - create_tun ? "tunnel" : "special", ret); - goto err_qp; - } - attr.qp_state = IB_QPS_RTS; - attr.sq_psn = 0; - ret = ib_modify_qp(tun_qp->qp, &attr, IB_QP_STATE | IB_QP_SQ_PSN); - if (ret) { - pr_err("Couldn't change %s qp state to RTS (%d)\n", - create_tun ? "tunnel" : "special", ret); - goto err_qp; - } - - for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { - ret = mlx4_ib_post_pv_qp_buf(ctx, tun_qp, i); - if (ret) { - pr_err(" mlx4_ib_post_pv_buf error" - " (err = %d, i = %d)\n", ret, i); - goto err_qp; - } - } - return 0; - -err_qp: - ib_destroy_qp(tun_qp->qp); - tun_qp->qp = NULL; - return ret; -} - -/* - * IB MAD completion callback for real SQPs - */ -static void mlx4_ib_sqp_comp_worker(struct work_struct *work) -{ - struct mlx4_ib_demux_pv_ctx *ctx; - struct mlx4_ib_demux_pv_qp *sqp; - struct ib_wc wc; - struct ib_grh *grh; - struct ib_mad *mad; - - ctx = container_of(work, struct mlx4_ib_demux_pv_ctx, work); - ib_req_notify_cq(ctx->cq, IB_CQ_NEXT_COMP); - - while (mlx4_ib_poll_cq(ctx->cq, 1, &wc) == 1) { - sqp = &ctx->qp[MLX4_TUN_WRID_QPN(wc.wr_id)]; - if (wc.status == IB_WC_SUCCESS) { - switch (wc.opcode) { - case IB_WC_SEND: - ib_destroy_ah(sqp->tx_ring[wc.wr_id & - (MLX4_NUM_TUNNEL_BUFS - 1)].ah); - sqp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah - = NULL; - spin_lock(&sqp->tx_lock); - sqp->tx_ix_tail++; - spin_unlock(&sqp->tx_lock); - break; - case IB_WC_RECV: - mad = (struct ib_mad *) &(((struct mlx4_mad_rcv_buf *) - (sqp->ring[wc.wr_id & - (MLX4_NUM_TUNNEL_BUFS - 1)].addr))->payload); - grh = &(((struct mlx4_mad_rcv_buf *) - (sqp->ring[wc.wr_id & - (MLX4_NUM_TUNNEL_BUFS - 1)].addr))->grh); - mlx4_ib_demux_mad(ctx->ib_dev, ctx->port, &wc, grh, mad); - if (mlx4_ib_post_pv_qp_buf(ctx, sqp, wc.wr_id & - (MLX4_NUM_TUNNEL_BUFS - 1))) - pr_err("Failed reposting SQP " - "buf:%lld\n", (unsigned long long)wc.wr_id); - break; - default: - BUG_ON(1); - break; - } - } else { - pr_debug("mlx4_ib: completion error in tunnel: %d." - " status = %d, wrid = 0x%llx\n", - ctx->slave, wc.status, (unsigned long long)wc.wr_id); - if (!MLX4_TUN_IS_RECV(wc.wr_id)) { - ib_destroy_ah(sqp->tx_ring[wc.wr_id & - (MLX4_NUM_TUNNEL_BUFS - 1)].ah); - sqp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah - = NULL; - spin_lock(&sqp->tx_lock); - sqp->tx_ix_tail++; - spin_unlock(&sqp->tx_lock); - } - } - } -} - -static int alloc_pv_object(struct mlx4_ib_dev *dev, int slave, int port, - struct mlx4_ib_demux_pv_ctx **ret_ctx) -{ - struct mlx4_ib_demux_pv_ctx *ctx; - - *ret_ctx = NULL; - ctx = kzalloc(sizeof (struct mlx4_ib_demux_pv_ctx), GFP_KERNEL); - if (!ctx) { - pr_err("failed allocating pv resource context " - "for port %d, slave %d\n", port, slave); - return -ENOMEM; - } - - ctx->ib_dev = &dev->ib_dev; - ctx->port = port; - ctx->slave = slave; - *ret_ctx = ctx; - return 0; -} - -static void free_pv_object(struct mlx4_ib_dev *dev, int slave, int port) -{ - if (dev->sriov.demux[port - 1].tun[slave]) { - kfree(dev->sriov.demux[port - 1].tun[slave]); - dev->sriov.demux[port - 1].tun[slave] = NULL; - } -} - -static int create_pv_resources(struct ib_device *ibdev, int slave, int port, - int create_tun, struct mlx4_ib_demux_pv_ctx *ctx) -{ - int ret, cq_size; - - if (ctx->state != DEMUX_PV_STATE_DOWN) - return -EEXIST; - - ctx->state = DEMUX_PV_STATE_STARTING; - /* have QP0 only on port owner, and only if link layer is IB */ - if (ctx->slave == mlx4_master_func_num(to_mdev(ctx->ib_dev)->dev) && - rdma_port_get_link_layer(ibdev, ctx->port) == IB_LINK_LAYER_INFINIBAND) - ctx->has_smi = 1; - - if (ctx->has_smi) { - ret = mlx4_ib_alloc_pv_bufs(ctx, IB_QPT_SMI, create_tun); - if (ret) { - pr_err("Failed allocating qp0 tunnel bufs (%d)\n", ret); - goto err_out; - } - } - - ret = mlx4_ib_alloc_pv_bufs(ctx, IB_QPT_GSI, create_tun); - if (ret) { - pr_err("Failed allocating qp1 tunnel bufs (%d)\n", ret); - goto err_out_qp0; - } - - cq_size = 2 * MLX4_NUM_TUNNEL_BUFS; - if (ctx->has_smi) - cq_size *= 2; - - ctx->cq = ib_create_cq(ctx->ib_dev, mlx4_ib_tunnel_comp_handler, - NULL, ctx, cq_size, 0); - if (IS_ERR(ctx->cq)) { - ret = PTR_ERR(ctx->cq); - pr_err("Couldn't create tunnel CQ (%d)\n", ret); - goto err_buf; - } - - ctx->pd = ib_alloc_pd(ctx->ib_dev); - if (IS_ERR(ctx->pd)) { - ret = PTR_ERR(ctx->pd); - pr_err("Couldn't create tunnel PD (%d)\n", ret); - goto err_cq; - } - - ctx->mr = ib_get_dma_mr(ctx->pd, IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(ctx->mr)) { - ret = PTR_ERR(ctx->mr); - pr_err("Couldn't get tunnel DMA MR (%d)\n", ret); - goto err_pd; - } - - if (ctx->has_smi) { - ret = create_pv_sqp(ctx, IB_QPT_SMI, create_tun); - if (ret) { - pr_err("Couldn't create %s QP0 (%d)\n", - create_tun ? "tunnel for" : "", ret); - goto err_mr; - } - } - - ret = create_pv_sqp(ctx, IB_QPT_GSI, create_tun); - if (ret) { - pr_err("Couldn't create %s QP1 (%d)\n", - create_tun ? "tunnel for" : "", ret); - goto err_qp0; - } - - if (create_tun) - INIT_WORK(&ctx->work, mlx4_ib_tunnel_comp_worker); - else - INIT_WORK(&ctx->work, mlx4_ib_sqp_comp_worker); - - ctx->wq = to_mdev(ibdev)->sriov.demux[port - 1].wq; - - ret = ib_req_notify_cq(ctx->cq, IB_CQ_NEXT_COMP); - if (ret) { - pr_err("Couldn't arm tunnel cq (%d)\n", ret); - goto err_wq; - } - ctx->state = DEMUX_PV_STATE_ACTIVE; - return 0; - -err_wq: - ctx->wq = NULL; - ib_destroy_qp(ctx->qp[1].qp); - ctx->qp[1].qp = NULL; - - -err_qp0: - if (ctx->has_smi) - ib_destroy_qp(ctx->qp[0].qp); - ctx->qp[0].qp = NULL; - -err_mr: - ib_dereg_mr(ctx->mr); - ctx->mr = NULL; - -err_pd: - ib_dealloc_pd(ctx->pd); - ctx->pd = NULL; - -err_cq: - ib_destroy_cq(ctx->cq); - ctx->cq = NULL; - -err_buf: - mlx4_ib_free_pv_qp_bufs(ctx, IB_QPT_GSI, create_tun); - -err_out_qp0: - if (ctx->has_smi) - mlx4_ib_free_pv_qp_bufs(ctx, IB_QPT_SMI, create_tun); -err_out: - ctx->state = DEMUX_PV_STATE_DOWN; - return ret; -} - -static void destroy_pv_resources(struct mlx4_ib_dev *dev, int slave, int port, - struct mlx4_ib_demux_pv_ctx *ctx, int flush) -{ - if (!ctx) - return; - if (ctx->state > DEMUX_PV_STATE_DOWN) { - ctx->state = DEMUX_PV_STATE_DOWNING; - if (flush) - flush_workqueue(ctx->wq); - if (ctx->has_smi) { - ib_destroy_qp(ctx->qp[0].qp); - ctx->qp[0].qp = NULL; - mlx4_ib_free_pv_qp_bufs(ctx, IB_QPT_SMI, 1); - } - ib_destroy_qp(ctx->qp[1].qp); - ctx->qp[1].qp = NULL; - mlx4_ib_free_pv_qp_bufs(ctx, IB_QPT_GSI, 1); - ib_dereg_mr(ctx->mr); - ctx->mr = NULL; - ib_dealloc_pd(ctx->pd); - ctx->pd = NULL; - ib_destroy_cq(ctx->cq); - ctx->cq = NULL; - ctx->state = DEMUX_PV_STATE_DOWN; - } -} - -static int mlx4_ib_tunnels_update(struct mlx4_ib_dev *dev, int slave, - int port, int do_init) -{ - int ret = 0; - - if (!do_init) { - clean_vf_mcast(&dev->sriov.demux[port - 1], slave); - /* for master, destroy real sqp resources */ - if (slave == mlx4_master_func_num(dev->dev)) - destroy_pv_resources(dev, slave, port, - dev->sriov.sqps[port - 1], 1); - /* destroy the tunnel qp resources */ - destroy_pv_resources(dev, slave, port, - dev->sriov.demux[port - 1].tun[slave], 1); - return 0; - } - - /* create the tunnel qp resources */ - ret = create_pv_resources(&dev->ib_dev, slave, port, 1, - dev->sriov.demux[port - 1].tun[slave]); - - /* for master, create the real sqp resources */ - if (!ret && slave == mlx4_master_func_num(dev->dev)) - ret = create_pv_resources(&dev->ib_dev, slave, port, 0, - dev->sriov.sqps[port - 1]); - return ret; -} - -void mlx4_ib_tunnels_update_work(struct work_struct *work) -{ - struct mlx4_ib_demux_work *dmxw; - - dmxw = container_of(work, struct mlx4_ib_demux_work, work); - mlx4_ib_tunnels_update(dmxw->dev, dmxw->slave, (int) dmxw->port, - dmxw->do_init); - kfree(dmxw); - return; -} - -static int mlx4_ib_alloc_demux_ctx(struct mlx4_ib_dev *dev, - struct mlx4_ib_demux_ctx *ctx, - int port) -{ - char name[12]; - int ret = 0; - int i; - - ctx->tun = kcalloc(dev->dev->caps.sqp_demux, - sizeof (struct mlx4_ib_demux_pv_ctx *), GFP_KERNEL); - if (!ctx->tun) - return -ENOMEM; - - ctx->dev = dev; - ctx->port = port; - ctx->ib_dev = &dev->ib_dev; - - for (i = 0; i < dev->dev->caps.sqp_demux; i++) { - ret = alloc_pv_object(dev, i, port, &ctx->tun[i]); - if (ret) { - ret = -ENOMEM; - goto err_mcg; - } - } - - ret = mlx4_ib_mcg_port_init(ctx); - if (ret) { - pr_err("Failed initializing mcg para-virt (%d)\n", ret); - goto err_mcg; - } - - snprintf(name, sizeof name, "mlx4_ibt%d", port); - ctx->wq = create_singlethread_workqueue(name); - if (!ctx->wq) { - pr_err("Failed to create tunnelling WQ for port %d\n", port); - ret = -ENOMEM; - goto err_wq; - } - - snprintf(name, sizeof name, "mlx4_ibud%d", port); - ctx->ud_wq = create_singlethread_workqueue(name); - if (!ctx->ud_wq) { - pr_err("Failed to create up/down WQ for port %d\n", port); - ret = -ENOMEM; - goto err_udwq; - } - - return 0; - -err_udwq: - destroy_workqueue(ctx->wq); - ctx->wq = NULL; - -err_wq: - mlx4_ib_mcg_port_cleanup(ctx, 1); -err_mcg: - for (i = 0; i < dev->dev->caps.sqp_demux; i++) - free_pv_object(dev, i, port); - kfree(ctx->tun); - ctx->tun = NULL; - return ret; -} - -static void mlx4_ib_free_sqp_ctx(struct mlx4_ib_demux_pv_ctx *sqp_ctx) -{ - if (sqp_ctx->state > DEMUX_PV_STATE_DOWN) { - sqp_ctx->state = DEMUX_PV_STATE_DOWNING; - flush_workqueue(sqp_ctx->wq); - if (sqp_ctx->has_smi) { - ib_destroy_qp(sqp_ctx->qp[0].qp); - sqp_ctx->qp[0].qp = NULL; - mlx4_ib_free_pv_qp_bufs(sqp_ctx, IB_QPT_SMI, 0); - } - ib_destroy_qp(sqp_ctx->qp[1].qp); - sqp_ctx->qp[1].qp = NULL; - mlx4_ib_free_pv_qp_bufs(sqp_ctx, IB_QPT_GSI, 0); - ib_dereg_mr(sqp_ctx->mr); - sqp_ctx->mr = NULL; - ib_dealloc_pd(sqp_ctx->pd); - sqp_ctx->pd = NULL; - ib_destroy_cq(sqp_ctx->cq); - sqp_ctx->cq = NULL; - sqp_ctx->state = DEMUX_PV_STATE_DOWN; - } -} - -static void mlx4_ib_free_demux_ctx(struct mlx4_ib_demux_ctx *ctx) -{ - int i; - if (ctx) { - struct mlx4_ib_dev *dev = to_mdev(ctx->ib_dev); - mlx4_ib_mcg_port_cleanup(ctx, 1); - for (i = 0; i < dev->dev->caps.sqp_demux; i++) { - if (!ctx->tun[i]) - continue; - if (ctx->tun[i]->state > DEMUX_PV_STATE_DOWN) - ctx->tun[i]->state = DEMUX_PV_STATE_DOWNING; - } - flush_workqueue(ctx->wq); - for (i = 0; i < dev->dev->caps.sqp_demux; i++) { - destroy_pv_resources(dev, i, ctx->port, ctx->tun[i], 0); - free_pv_object(dev, i, ctx->port); - } - kfree(ctx->tun); - destroy_workqueue(ctx->ud_wq); - destroy_workqueue(ctx->wq); - } -} - -static void mlx4_ib_master_tunnels(struct mlx4_ib_dev *dev, int do_init) -{ - int i; - - if (!mlx4_is_master(dev->dev)) - return; - /* initialize or tear down tunnel QPs for the master */ - for (i = 0; i < dev->dev->caps.num_ports; i++) - mlx4_ib_tunnels_update(dev, mlx4_master_func_num(dev->dev), i + 1, do_init); - return; -} - -int mlx4_ib_init_sriov(struct mlx4_ib_dev *dev) -{ - int i = 0; - int err; - - if (!mlx4_is_mfunc(dev->dev)) - return 0; - - dev->sriov.is_going_down = 0; - spin_lock_init(&dev->sriov.going_down_lock); - mlx4_ib_cm_paravirt_init(dev); - - mlx4_ib_warn(&dev->ib_dev, "multi-function enabled\n"); - - if (mlx4_is_slave(dev->dev)) { - mlx4_ib_warn(&dev->ib_dev, "operating in qp1 tunnel mode\n"); - return 0; - } - - for (i = 0; i < dev->dev->caps.sqp_demux; i++) { - if (i == mlx4_master_func_num(dev->dev)) - mlx4_put_slave_node_guid(dev->dev, i, dev->ib_dev.node_guid); - else - mlx4_put_slave_node_guid(dev->dev, i, mlx4_ib_gen_node_guid()); - } - - err = mlx4_ib_init_alias_guid_service(dev); - if (err) { - mlx4_ib_warn(&dev->ib_dev, "Failed init alias guid process.\n"); - goto paravirt_err; - } - err = mlx4_ib_device_register_sysfs(dev); - if (err) { - mlx4_ib_warn(&dev->ib_dev, "Failed to register sysfs\n"); - goto sysfs_err; - } - - mlx4_ib_warn(&dev->ib_dev, "initializing demux service for %d qp1 clients\n", - dev->dev->caps.sqp_demux); - for (i = 0; i < dev->num_ports; i++) { - union ib_gid gid; - err = __mlx4_ib_query_gid(&dev->ib_dev, i + 1, 0, &gid, 1); - if (err) - goto demux_err; - dev->sriov.demux[i].guid_cache[0] = gid.global.interface_id; - err = alloc_pv_object(dev, mlx4_master_func_num(dev->dev), i + 1, - &dev->sriov.sqps[i]); - if (err) - goto demux_err; - err = mlx4_ib_alloc_demux_ctx(dev, &dev->sriov.demux[i], i + 1); - if (err) - goto demux_err; - } - mlx4_ib_master_tunnels(dev, 1); - return 0; - -demux_err: - while (i > 0) { - free_pv_object(dev, mlx4_master_func_num(dev->dev), i + 1); - mlx4_ib_free_demux_ctx(&dev->sriov.demux[i]); - --i; - } - mlx4_ib_device_unregister_sysfs(dev); - -sysfs_err: - mlx4_ib_destroy_alias_guid_service(dev); - -paravirt_err: - mlx4_ib_cm_paravirt_clean(dev, -1); - - return err; -} - -void mlx4_ib_close_sriov(struct mlx4_ib_dev *dev) -{ - int i; - unsigned long flags; - - if (!mlx4_is_mfunc(dev->dev)) - return; - - spin_lock_irqsave(&dev->sriov.going_down_lock, flags); - dev->sriov.is_going_down = 1; - spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); - if (mlx4_is_master(dev->dev)) { - for (i = 0; i < dev->num_ports; i++) { - flush_workqueue(dev->sriov.demux[i].ud_wq); - mlx4_ib_free_sqp_ctx(dev->sriov.sqps[i]); - kfree(dev->sriov.sqps[i]); - dev->sriov.sqps[i] = NULL; - mlx4_ib_free_demux_ctx(&dev->sriov.demux[i]); - } - - mlx4_ib_cm_paravirt_clean(dev, -1); - mlx4_ib_destroy_alias_guid_service(dev); - mlx4_ib_device_unregister_sysfs(dev); - } -} Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mad.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mcg.c =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mcg.c (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mcg.c (nonexistent) @@ -1,1260 +0,0 @@ -/* - * Copyright (c) 2012 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include - -#include -#include -#include - -#include "mlx4_ib.h" - -#define MAX_VFS 80 -#define MAX_PEND_REQS_PER_FUNC 4 -#define MAD_TIMEOUT_MS 2000 - -#define mcg_warn(fmt, arg...) pr_warn("MCG WARNING: " fmt, ##arg) -#define mcg_error(fmt, arg...) pr_err(fmt, ##arg) -#define mcg_warn_group(group, format, arg...) \ - pr_warn("%s-%d: %16s (port %d): WARNING: " format, __func__, __LINE__,\ - (group)->name, group->demux->port, ## arg) - -#define mcg_error_group(group, format, arg...) \ - pr_err(" %16s: " format, (group)->name, ## arg) - - -static union ib_gid mgid0; - -static struct workqueue_struct *clean_wq; - -enum mcast_state { - MCAST_NOT_MEMBER = 0, - MCAST_MEMBER, -}; - -enum mcast_group_state { - MCAST_IDLE, - MCAST_JOIN_SENT, - MCAST_LEAVE_SENT, - MCAST_RESP_READY -}; - -struct mcast_member { - enum mcast_state state; - uint8_t join_state; - int num_pend_reqs; - struct list_head pending; -}; - -struct ib_sa_mcmember_data { - union ib_gid mgid; - union ib_gid port_gid; - __be32 qkey; - __be16 mlid; - u8 mtusel_mtu; - u8 tclass; - __be16 pkey; - u8 ratesel_rate; - u8 lifetmsel_lifetm; - __be32 sl_flowlabel_hoplimit; - u8 scope_join_state; - u8 proxy_join; - u8 reserved[2]; -}; - -struct mcast_group { - struct ib_sa_mcmember_data rec; - struct rb_node node; - struct list_head mgid0_list; - struct mlx4_ib_demux_ctx *demux; - struct mcast_member func[MAX_VFS]; - struct mutex lock; - struct work_struct work; - struct list_head pending_list; - int members[3]; - enum mcast_group_state state; - enum mcast_group_state prev_state; - struct ib_sa_mad response_sa_mad; - __be64 last_req_tid; - - char name[33]; /* MGID string */ - struct device_attribute dentry; - - /* refcount is the reference count for the following: - 1. Each queued request - 2. Each invocation of the worker thread - 3. Membership of the port at the SA - */ - atomic_t refcount; - - /* delayed work to clean pending SM request */ - struct delayed_work timeout_work; - struct list_head cleanup_list; -}; - -struct mcast_req { - int func; - struct ib_sa_mad sa_mad; - struct list_head group_list; - struct list_head func_list; - struct mcast_group *group; - int clean; -}; - - -#define safe_atomic_dec(ref) \ - do {\ - if (atomic_dec_and_test(ref)) \ - mcg_warn_group(group, "did not expect to reach zero\n"); \ - } while (0) - -static const char *get_state_string(enum mcast_group_state state) -{ - switch (state) { - case MCAST_IDLE: - return "MCAST_IDLE"; - case MCAST_JOIN_SENT: - return "MCAST_JOIN_SENT"; - case MCAST_LEAVE_SENT: - return "MCAST_LEAVE_SENT"; - case MCAST_RESP_READY: - return "MCAST_RESP_READY"; - } - return "Invalid State"; -} - -static struct mcast_group *mcast_find(struct mlx4_ib_demux_ctx *ctx, - union ib_gid *mgid) -{ - struct rb_node *node = ctx->mcg_table.rb_node; - struct mcast_group *group; - int ret; - - while (node) { - group = rb_entry(node, struct mcast_group, node); - ret = memcmp(mgid->raw, group->rec.mgid.raw, sizeof *mgid); - if (!ret) - return group; - - if (ret < 0) - node = node->rb_left; - else - node = node->rb_right; - } - return NULL; -} - -static struct mcast_group *mcast_insert(struct mlx4_ib_demux_ctx *ctx, - struct mcast_group *group) -{ - struct rb_node **link = &ctx->mcg_table.rb_node; - struct rb_node *parent = NULL; - struct mcast_group *cur_group; - int ret; - - while (*link) { - parent = *link; - cur_group = rb_entry(parent, struct mcast_group, node); - - ret = memcmp(group->rec.mgid.raw, cur_group->rec.mgid.raw, - sizeof group->rec.mgid); - if (ret < 0) - link = &(*link)->rb_left; - else if (ret > 0) - link = &(*link)->rb_right; - else - return cur_group; - } - rb_link_node(&group->node, parent, link); - rb_insert_color(&group->node, &ctx->mcg_table); - return NULL; -} - -static int send_mad_to_wire(struct mlx4_ib_demux_ctx *ctx, struct ib_mad *mad) -{ - struct mlx4_ib_dev *dev = ctx->dev; - struct ib_ah_attr ah_attr; - - spin_lock(&dev->sm_lock); - if (!dev->sm_ah[ctx->port - 1]) { - /* port is not yet Active, sm_ah not ready */ - spin_unlock(&dev->sm_lock); - return -EAGAIN; - } - mlx4_ib_query_ah(dev->sm_ah[ctx->port - 1], &ah_attr); - spin_unlock(&dev->sm_lock); - return mlx4_ib_send_to_wire(dev, mlx4_master_func_num(dev->dev), ctx->port, - IB_QPT_GSI, 0, 1, IB_QP1_QKEY, &ah_attr, 0, mad); -} - -static int send_mad_to_slave(int slave, struct mlx4_ib_demux_ctx *ctx, - struct ib_mad *mad) -{ - struct mlx4_ib_dev *dev = ctx->dev; - struct ib_mad_agent *agent = dev->send_agent[ctx->port - 1][1]; - struct ib_wc wc; - struct ib_ah_attr ah_attr; - - /* Our agent might not yet be registered when mads start to arrive */ - if (!agent) - return -EAGAIN; - - ib_query_ah(dev->sm_ah[ctx->port - 1], &ah_attr); - - if (ib_find_cached_pkey(&dev->ib_dev, ctx->port, IB_DEFAULT_PKEY_FULL, &wc.pkey_index)) - return -EINVAL; - wc.sl = 0; - wc.dlid_path_bits = 0; - wc.port_num = ctx->port; - wc.slid = ah_attr.dlid; /* opensm lid */ - wc.src_qp = 1; - return mlx4_ib_send_to_slave(dev, slave, ctx->port, IB_QPT_GSI, &wc, NULL, mad); -} - -static int send_join_to_wire(struct mcast_group *group, struct ib_sa_mad *sa_mad) -{ - struct ib_sa_mad mad; - struct ib_sa_mcmember_data *sa_mad_data = (struct ib_sa_mcmember_data *)&mad.data; - int ret; - - /* we rely on a mad request as arrived from a VF */ - memcpy(&mad, sa_mad, sizeof mad); - - /* fix port GID to be the real one (slave 0) */ - sa_mad_data->port_gid.global.interface_id = group->demux->guid_cache[0]; - - /* assign our own TID */ - mad.mad_hdr.tid = mlx4_ib_get_new_demux_tid(group->demux); - group->last_req_tid = mad.mad_hdr.tid; /* keep it for later validation */ - - ret = send_mad_to_wire(group->demux, (struct ib_mad *)&mad); - /* set timeout handler */ - if (!ret) { - /* calls mlx4_ib_mcg_timeout_handler */ - queue_delayed_work(group->demux->mcg_wq, &group->timeout_work, - msecs_to_jiffies(MAD_TIMEOUT_MS)); - } - - return ret; -} - -static int send_leave_to_wire(struct mcast_group *group, u8 join_state) -{ - struct ib_sa_mad mad; - struct ib_sa_mcmember_data *sa_data = (struct ib_sa_mcmember_data *)&mad.data; - int ret; - - memset(&mad, 0, sizeof mad); - mad.mad_hdr.base_version = 1; - mad.mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM; - mad.mad_hdr.class_version = 2; - mad.mad_hdr.method = IB_SA_METHOD_DELETE; - mad.mad_hdr.status = cpu_to_be16(0); - mad.mad_hdr.class_specific = cpu_to_be16(0); - mad.mad_hdr.tid = mlx4_ib_get_new_demux_tid(group->demux); - group->last_req_tid = mad.mad_hdr.tid; /* keep it for later validation */ - mad.mad_hdr.attr_id = cpu_to_be16(IB_SA_ATTR_MC_MEMBER_REC); - mad.mad_hdr.attr_mod = cpu_to_be32(0); - mad.sa_hdr.sm_key = 0x0; - mad.sa_hdr.attr_offset = cpu_to_be16(7); - mad.sa_hdr.comp_mask = IB_SA_MCMEMBER_REC_MGID | - IB_SA_MCMEMBER_REC_PORT_GID | IB_SA_MCMEMBER_REC_JOIN_STATE; - - *sa_data = group->rec; - sa_data->scope_join_state = join_state; - - ret = send_mad_to_wire(group->demux, (struct ib_mad *)&mad); - if (ret) - group->state = MCAST_IDLE; - - /* set timeout handler */ - if (!ret) { - /* calls mlx4_ib_mcg_timeout_handler */ - queue_delayed_work(group->demux->mcg_wq, &group->timeout_work, - msecs_to_jiffies(MAD_TIMEOUT_MS)); - } - - return ret; -} - -static int send_reply_to_slave(int slave, struct mcast_group *group, - struct ib_sa_mad *req_sa_mad, u16 status) -{ - struct ib_sa_mad mad; - struct ib_sa_mcmember_data *sa_data = (struct ib_sa_mcmember_data *)&mad.data; - struct ib_sa_mcmember_data *req_sa_data = (struct ib_sa_mcmember_data *)&req_sa_mad->data; - int ret; - - memset(&mad, 0, sizeof mad); - mad.mad_hdr.base_version = 1; - mad.mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM; - mad.mad_hdr.class_version = 2; - mad.mad_hdr.method = IB_MGMT_METHOD_GET_RESP; - mad.mad_hdr.status = cpu_to_be16(status); - mad.mad_hdr.class_specific = cpu_to_be16(0); - mad.mad_hdr.tid = req_sa_mad->mad_hdr.tid; - *(u8 *)&mad.mad_hdr.tid = 0; /* resetting tid to 0 */ - mad.mad_hdr.attr_id = cpu_to_be16(IB_SA_ATTR_MC_MEMBER_REC); - mad.mad_hdr.attr_mod = cpu_to_be32(0); - mad.sa_hdr.sm_key = req_sa_mad->sa_hdr.sm_key; - mad.sa_hdr.attr_offset = cpu_to_be16(7); - mad.sa_hdr.comp_mask = 0; /* ignored on responses, see IBTA spec */ - - *sa_data = group->rec; - - /* reconstruct VF's requested join_state and port_gid */ - sa_data->scope_join_state &= 0xf0; - sa_data->scope_join_state |= (group->func[slave].join_state & 0x0f); - memcpy(&sa_data->port_gid, &req_sa_data->port_gid, sizeof req_sa_data->port_gid); - - ret = send_mad_to_slave(slave, group->demux, (struct ib_mad *)&mad); - return ret; -} - -static int check_selector(ib_sa_comp_mask comp_mask, - ib_sa_comp_mask selector_mask, - ib_sa_comp_mask value_mask, - u8 src_value, u8 dst_value) -{ - int err; - u8 selector = dst_value >> 6; - dst_value &= 0x3f; - src_value &= 0x3f; - - if (!(comp_mask & selector_mask) || !(comp_mask & value_mask)) - return 0; - - switch (selector) { - case IB_SA_GT: - err = (src_value <= dst_value); - break; - case IB_SA_LT: - err = (src_value >= dst_value); - break; - case IB_SA_EQ: - err = (src_value != dst_value); - break; - default: - err = 0; - break; - } - - return err; -} - -static u16 cmp_rec(struct ib_sa_mcmember_data *src, - struct ib_sa_mcmember_data *dst, ib_sa_comp_mask comp_mask) -{ - /* src is group record, dst is request record */ - /* MGID must already match */ - /* Port_GID we always replace to our Port_GID, so it is a match */ - -#define MAD_STATUS_REQ_INVALID 0x0200 - if (comp_mask & IB_SA_MCMEMBER_REC_QKEY && src->qkey != dst->qkey) - return MAD_STATUS_REQ_INVALID; - if (comp_mask & IB_SA_MCMEMBER_REC_MLID && src->mlid != dst->mlid) - return MAD_STATUS_REQ_INVALID; - if (check_selector(comp_mask, IB_SA_MCMEMBER_REC_MTU_SELECTOR, - IB_SA_MCMEMBER_REC_MTU, - src->mtusel_mtu, dst->mtusel_mtu)) - return MAD_STATUS_REQ_INVALID; - if (comp_mask & IB_SA_MCMEMBER_REC_TRAFFIC_CLASS && - src->tclass != dst->tclass) - return MAD_STATUS_REQ_INVALID; - if (comp_mask & IB_SA_MCMEMBER_REC_PKEY && src->pkey != dst->pkey) - return MAD_STATUS_REQ_INVALID; - if (check_selector(comp_mask, IB_SA_MCMEMBER_REC_RATE_SELECTOR, - IB_SA_MCMEMBER_REC_RATE, - src->ratesel_rate, dst->ratesel_rate)) - return MAD_STATUS_REQ_INVALID; - if (check_selector(comp_mask, - IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME_SELECTOR, - IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME, - src->lifetmsel_lifetm, dst->lifetmsel_lifetm)) - return MAD_STATUS_REQ_INVALID; - if (comp_mask & IB_SA_MCMEMBER_REC_SL && - (be32_to_cpu(src->sl_flowlabel_hoplimit) & 0xf0000000) != - (be32_to_cpu(dst->sl_flowlabel_hoplimit) & 0xf0000000)) - return MAD_STATUS_REQ_INVALID; - if (comp_mask & IB_SA_MCMEMBER_REC_FLOW_LABEL && - (be32_to_cpu(src->sl_flowlabel_hoplimit) & 0x0fffff00) != - (be32_to_cpu(dst->sl_flowlabel_hoplimit) & 0x0fffff00)) - return MAD_STATUS_REQ_INVALID; - if (comp_mask & IB_SA_MCMEMBER_REC_HOP_LIMIT && - (be32_to_cpu(src->sl_flowlabel_hoplimit) & 0x000000ff) != - (be32_to_cpu(dst->sl_flowlabel_hoplimit) & 0x000000ff)) - return MAD_STATUS_REQ_INVALID; - if (comp_mask & IB_SA_MCMEMBER_REC_SCOPE && - (src->scope_join_state & 0xf0) != - (dst->scope_join_state & 0xf0)) - return MAD_STATUS_REQ_INVALID; - - /* join_state checked separately, proxy_join ignored */ - - return 0; -} - -/* release group, return 1 if this was last release and group is destroyed - * timout work is canceled sync */ -static int release_group(struct mcast_group *group, int from_timeout_handler) -{ - struct mlx4_ib_demux_ctx *ctx = group->demux; - int nzgroup; - - mutex_lock(&ctx->mcg_table_lock); - mutex_lock(&group->lock); - if (atomic_dec_and_test(&group->refcount)) { - if (!from_timeout_handler) { - if (group->state != MCAST_IDLE && - !cancel_delayed_work(&group->timeout_work)) { - atomic_inc(&group->refcount); - mutex_unlock(&group->lock); - mutex_unlock(&ctx->mcg_table_lock); - return 0; - } - } - - nzgroup = memcmp(&group->rec.mgid, &mgid0, sizeof mgid0); - if (nzgroup) - del_sysfs_port_mcg_attr(ctx->dev, ctx->port, &group->dentry.attr); - if (!list_empty(&group->pending_list)) - mcg_warn_group(group, "releasing a group with non empty pending list\n"); - if (nzgroup) - rb_erase(&group->node, &ctx->mcg_table); - list_del_init(&group->mgid0_list); - mutex_unlock(&group->lock); - mutex_unlock(&ctx->mcg_table_lock); - kfree(group); - return 1; - } else { - mutex_unlock(&group->lock); - mutex_unlock(&ctx->mcg_table_lock); - } - return 0; -} - -static void adjust_membership(struct mcast_group *group, u8 join_state, int inc) -{ - int i; - - for (i = 0; i < 3; i++, join_state >>= 1) - if (join_state & 0x1) - group->members[i] += inc; -} - -static u8 get_leave_state(struct mcast_group *group) -{ - u8 leave_state = 0; - int i; - - for (i = 0; i < 3; i++) - if (!group->members[i]) - leave_state |= (1 << i); - - return leave_state & (group->rec.scope_join_state & 7); -} - -static int join_group(struct mcast_group *group, int slave, u8 join_mask) -{ - int ret = 0; - u8 join_state; - - /* remove bits that slave is already member of, and adjust */ - join_state = join_mask & (~group->func[slave].join_state); - adjust_membership(group, join_state, 1); - group->func[slave].join_state |= join_state; - if (group->func[slave].state != MCAST_MEMBER && join_state) { - group->func[slave].state = MCAST_MEMBER; - ret = 1; - } - return ret; -} - -static int leave_group(struct mcast_group *group, int slave, u8 leave_state) -{ - int ret = 0; - - adjust_membership(group, leave_state, -1); - group->func[slave].join_state &= ~leave_state; - if (!group->func[slave].join_state) { - group->func[slave].state = MCAST_NOT_MEMBER; - ret = 1; - } - return ret; -} - -static int check_leave(struct mcast_group *group, int slave, u8 leave_mask) -{ - if (group->func[slave].state != MCAST_MEMBER) - return MAD_STATUS_REQ_INVALID; - - /* make sure we're not deleting unset bits */ - if (~group->func[slave].join_state & leave_mask) - return MAD_STATUS_REQ_INVALID; - - if (!leave_mask) - return MAD_STATUS_REQ_INVALID; - - return 0; -} - -static void mlx4_ib_mcg_timeout_handler(struct work_struct *work) -{ - struct delayed_work *delay = to_delayed_work(work); - struct mcast_group *group; - struct mcast_req *req = NULL; - - group = container_of(delay, typeof(*group), timeout_work); - - mutex_lock(&group->lock); - if (group->state == MCAST_JOIN_SENT) { - if (!list_empty(&group->pending_list)) { - req = list_first_entry(&group->pending_list, struct mcast_req, group_list); - list_del(&req->group_list); - list_del(&req->func_list); - --group->func[req->func].num_pend_reqs; - mutex_unlock(&group->lock); - kfree(req); - if (memcmp(&group->rec.mgid, &mgid0, sizeof mgid0)) { - if (release_group(group, 1)) - return; - } else { - kfree(group); - return; - } - mutex_lock(&group->lock); - } else - mcg_warn_group(group, "DRIVER BUG\n"); - } else if (group->state == MCAST_LEAVE_SENT) { - if (group->rec.scope_join_state & 7) - group->rec.scope_join_state &= 0xf8; - group->state = MCAST_IDLE; - mutex_unlock(&group->lock); - if (release_group(group, 1)) - return; - mutex_lock(&group->lock); - } else - mcg_warn_group(group, "invalid state %s\n", get_state_string(group->state)); - group->state = MCAST_IDLE; - atomic_inc(&group->refcount); - if (!queue_work(group->demux->mcg_wq, &group->work)) - safe_atomic_dec(&group->refcount); - - mutex_unlock(&group->lock); -} - -static int handle_leave_req(struct mcast_group *group, u8 leave_mask, - struct mcast_req *req) -{ - u16 status; - - if (req->clean) - leave_mask = group->func[req->func].join_state; - - status = check_leave(group, req->func, leave_mask); - if (!status) - leave_group(group, req->func, leave_mask); - - if (!req->clean) - send_reply_to_slave(req->func, group, &req->sa_mad, status); - --group->func[req->func].num_pend_reqs; - list_del(&req->group_list); - list_del(&req->func_list); - kfree(req); - return 1; -} - -static int handle_join_req(struct mcast_group *group, u8 join_mask, - struct mcast_req *req) -{ - u8 group_join_state = group->rec.scope_join_state & 7; - int ref = 0; - u16 status; - struct ib_sa_mcmember_data *sa_data = (struct ib_sa_mcmember_data *)req->sa_mad.data; - - if (join_mask == (group_join_state & join_mask)) { - /* port's membership need not change */ - status = cmp_rec(&group->rec, sa_data, req->sa_mad.sa_hdr.comp_mask); - if (!status) - join_group(group, req->func, join_mask); - - --group->func[req->func].num_pend_reqs; - send_reply_to_slave(req->func, group, &req->sa_mad, status); - list_del(&req->group_list); - list_del(&req->func_list); - kfree(req); - ++ref; - } else { - /* port's membership needs to be updated */ - group->prev_state = group->state; - if (send_join_to_wire(group, &req->sa_mad)) { - --group->func[req->func].num_pend_reqs; - list_del(&req->group_list); - list_del(&req->func_list); - kfree(req); - ref = 1; - group->state = group->prev_state; - } else - group->state = MCAST_JOIN_SENT; - } - - return ref; -} - -static void mlx4_ib_mcg_work_handler(struct work_struct *work) -{ - struct mcast_group *group; - struct mcast_req *req = NULL; - struct ib_sa_mcmember_data *sa_data; - u8 req_join_state; - int rc = 1; /* release_count - this is for the scheduled work */ - u16 status; - u8 method; - - group = container_of(work, typeof(*group), work); - - mutex_lock(&group->lock); - - /* First, let's see if a response from SM is waiting regarding this group. - * If so, we need to update the group's REC. If this is a bad response, we - * may need to send a bad response to a VF waiting for it. If VF is waiting - * and this is a good response, the VF will be answered later in this func. */ - if (group->state == MCAST_RESP_READY) { - /* cancels mlx4_ib_mcg_timeout_handler */ - cancel_delayed_work(&group->timeout_work); - status = be16_to_cpu(group->response_sa_mad.mad_hdr.status); - method = group->response_sa_mad.mad_hdr.method; - if (group->last_req_tid != group->response_sa_mad.mad_hdr.tid) { - mcg_warn_group(group, "Got MAD response to existing MGID but wrong TID, dropping. Resp TID=%llx, group TID=%llx\n", - (long long)be64_to_cpu( - group->response_sa_mad.mad_hdr.tid), - (long long)be64_to_cpu(group->last_req_tid)); - group->state = group->prev_state; - goto process_requests; - } - if (status) { - if (!list_empty(&group->pending_list)) - req = list_first_entry(&group->pending_list, - struct mcast_req, group_list); - if (method == IB_MGMT_METHOD_GET_RESP) { - if (req) { - send_reply_to_slave(req->func, group, &req->sa_mad, status); - --group->func[req->func].num_pend_reqs; - list_del(&req->group_list); - list_del(&req->func_list); - kfree(req); - ++rc; - } else - mcg_warn_group(group, "no request for failed join\n"); - } else if (method == IB_SA_METHOD_DELETE_RESP && group->demux->flushing) - ++rc; - } else { - u8 resp_join_state; - u8 cur_join_state; - - resp_join_state = ((struct ib_sa_mcmember_data *) - group->response_sa_mad.data)->scope_join_state & 7; - cur_join_state = group->rec.scope_join_state & 7; - - if (method == IB_MGMT_METHOD_GET_RESP) { - /* successful join */ - if (!cur_join_state && resp_join_state) - --rc; - } else if (!resp_join_state) - ++rc; - memcpy(&group->rec, group->response_sa_mad.data, sizeof group->rec); - } - group->state = MCAST_IDLE; - } - -process_requests: - /* We should now go over pending join/leave requests, as long as we are idle. */ - while (!list_empty(&group->pending_list) && group->state == MCAST_IDLE) { - req = list_first_entry(&group->pending_list, struct mcast_req, - group_list); - sa_data = (struct ib_sa_mcmember_data *)req->sa_mad.data; - req_join_state = sa_data->scope_join_state & 0x7; - - /* For a leave request, we will immediately answer the VF, and - * update our internal counters. The actual leave will be sent - * to SM later, if at all needed. We dequeue the request now. */ - if (req->sa_mad.mad_hdr.method == IB_SA_METHOD_DELETE) - rc += handle_leave_req(group, req_join_state, req); - else - rc += handle_join_req(group, req_join_state, req); - } - - /* Handle leaves */ - if (group->state == MCAST_IDLE) { - req_join_state = get_leave_state(group); - if (req_join_state) { - group->rec.scope_join_state &= ~req_join_state; - group->prev_state = group->state; - if (send_leave_to_wire(group, req_join_state)) { - group->state = group->prev_state; - ++rc; - } else - group->state = MCAST_LEAVE_SENT; - } - } - - if (!list_empty(&group->pending_list) && group->state == MCAST_IDLE) - goto process_requests; - mutex_unlock(&group->lock); - - while (rc--) - release_group(group, 0); -} - -static struct mcast_group *search_relocate_mgid0_group(struct mlx4_ib_demux_ctx *ctx, - __be64 tid, - union ib_gid *new_mgid) -{ - struct mcast_group *group = NULL, *cur_group; - struct mcast_req *req; - struct list_head *pos; - struct list_head *n; - - mutex_lock(&ctx->mcg_table_lock); - list_for_each_safe(pos, n, &ctx->mcg_mgid0_list) { - group = list_entry(pos, struct mcast_group, mgid0_list); - mutex_lock(&group->lock); - if (group->last_req_tid == tid) { - if (memcmp(new_mgid, &mgid0, sizeof mgid0)) { - group->rec.mgid = *new_mgid; - sprintf(group->name, "%016llx%016llx", - (long long)be64_to_cpu(group->rec.mgid.global.subnet_prefix), - (long long)be64_to_cpu(group->rec.mgid.global.interface_id)); - list_del_init(&group->mgid0_list); - cur_group = mcast_insert(ctx, group); - if (cur_group) { - /* A race between our code and SM. Silently cleaning the new one */ - req = list_first_entry(&group->pending_list, - struct mcast_req, group_list); - --group->func[req->func].num_pend_reqs; - list_del(&req->group_list); - list_del(&req->func_list); - kfree(req); - mutex_unlock(&group->lock); - mutex_unlock(&ctx->mcg_table_lock); - release_group(group, 0); - return NULL; - } - - atomic_inc(&group->refcount); - add_sysfs_port_mcg_attr(ctx->dev, ctx->port, &group->dentry.attr); - mutex_unlock(&group->lock); - mutex_unlock(&ctx->mcg_table_lock); - return group; - } else { - struct mcast_req *tmp1, *tmp2; - - list_del(&group->mgid0_list); - if (!list_empty(&group->pending_list) && group->state != MCAST_IDLE) - cancel_delayed_work_sync(&group->timeout_work); - - list_for_each_entry_safe(tmp1, tmp2, &group->pending_list, group_list) { - list_del(&tmp1->group_list); - kfree(tmp1); - } - mutex_unlock(&group->lock); - mutex_unlock(&ctx->mcg_table_lock); - kfree(group); - return NULL; - } - } - mutex_unlock(&group->lock); - } - mutex_unlock(&ctx->mcg_table_lock); - - return NULL; -} - -static ssize_t sysfs_show_group(struct device *dev, - struct device_attribute *attr, char *buf); - -static struct mcast_group *acquire_group(struct mlx4_ib_demux_ctx *ctx, - union ib_gid *mgid, int create, - gfp_t gfp_mask) -{ - struct mcast_group *group, *cur_group; - int is_mgid0; - int i; - - is_mgid0 = !memcmp(&mgid0, mgid, sizeof mgid0); - if (!is_mgid0) { - group = mcast_find(ctx, mgid); - if (group) - goto found; - } - - if (!create) - return ERR_PTR(-ENOENT); - - group = kzalloc(sizeof *group, gfp_mask); - if (!group) - return ERR_PTR(-ENOMEM); - - group->demux = ctx; - group->rec.mgid = *mgid; - INIT_LIST_HEAD(&group->pending_list); - INIT_LIST_HEAD(&group->mgid0_list); - for (i = 0; i < MAX_VFS; ++i) - INIT_LIST_HEAD(&group->func[i].pending); - INIT_WORK(&group->work, mlx4_ib_mcg_work_handler); - INIT_DELAYED_WORK(&group->timeout_work, mlx4_ib_mcg_timeout_handler); - mutex_init(&group->lock); - sprintf(group->name, "%016llx%016llx", - (long long)be64_to_cpu( - group->rec.mgid.global.subnet_prefix), - (long long)be64_to_cpu( - group->rec.mgid.global.interface_id)); - sysfs_attr_init(&group->dentry.attr); - group->dentry.show = sysfs_show_group; - group->dentry.store = NULL; - group->dentry.attr.name = group->name; - group->dentry.attr.mode = 0400; - group->state = MCAST_IDLE; - - if (is_mgid0) { - list_add(&group->mgid0_list, &ctx->mcg_mgid0_list); - goto found; - } - - cur_group = mcast_insert(ctx, group); - if (cur_group) { - mcg_warn("group just showed up %s - confused\n", cur_group->name); - kfree(group); - return ERR_PTR(-EINVAL); - } - - add_sysfs_port_mcg_attr(ctx->dev, ctx->port, &group->dentry.attr); - -found: - atomic_inc(&group->refcount); - return group; -} - -static void queue_req(struct mcast_req *req) -{ - struct mcast_group *group = req->group; - - atomic_inc(&group->refcount); /* for the request */ - atomic_inc(&group->refcount); /* for scheduling the work */ - list_add_tail(&req->group_list, &group->pending_list); - list_add_tail(&req->func_list, &group->func[req->func].pending); - /* calls mlx4_ib_mcg_work_handler */ - if (!queue_work(group->demux->mcg_wq, &group->work)) - safe_atomic_dec(&group->refcount); -} - -int mlx4_ib_mcg_demux_handler(struct ib_device *ibdev, int port, int slave, - struct ib_sa_mad *mad) -{ - struct mlx4_ib_dev *dev = to_mdev(ibdev); - struct ib_sa_mcmember_data *rec = (struct ib_sa_mcmember_data *)mad->data; - struct mlx4_ib_demux_ctx *ctx = &dev->sriov.demux[port - 1]; - struct mcast_group *group; - - switch (mad->mad_hdr.method) { - case IB_MGMT_METHOD_GET_RESP: - case IB_SA_METHOD_DELETE_RESP: - mutex_lock(&ctx->mcg_table_lock); - group = acquire_group(ctx, &rec->mgid, 0, GFP_KERNEL); - mutex_unlock(&ctx->mcg_table_lock); - if (IS_ERR(group)) { - if (mad->mad_hdr.method == IB_MGMT_METHOD_GET_RESP) { - __be64 tid = mad->mad_hdr.tid; - *(u8 *)(&tid) = (u8)slave; /* in group we kept the modified TID */ - group = search_relocate_mgid0_group(ctx, tid, &rec->mgid); - } else - group = NULL; - } - - if (!group) - return 1; - - mutex_lock(&group->lock); - group->response_sa_mad = *mad; - group->prev_state = group->state; - group->state = MCAST_RESP_READY; - /* calls mlx4_ib_mcg_work_handler */ - atomic_inc(&group->refcount); - if (!queue_work(ctx->mcg_wq, &group->work)) - safe_atomic_dec(&group->refcount); - mutex_unlock(&group->lock); - release_group(group, 0); - return 1; /* consumed */ - case IB_MGMT_METHOD_SET: - case IB_SA_METHOD_GET_TABLE: - case IB_SA_METHOD_GET_TABLE_RESP: - case IB_SA_METHOD_DELETE: - return 0; /* not consumed, pass-through to guest over tunnel */ - default: - mcg_warn("In demux, port %d: unexpected MCMember method: 0x%x, dropping\n", - port, mad->mad_hdr.method); - return 1; /* consumed */ - } -} - -int mlx4_ib_mcg_multiplex_handler(struct ib_device *ibdev, int port, - int slave, struct ib_sa_mad *sa_mad) -{ - struct mlx4_ib_dev *dev = to_mdev(ibdev); - struct ib_sa_mcmember_data *rec = (struct ib_sa_mcmember_data *)sa_mad->data; - struct mlx4_ib_demux_ctx *ctx = &dev->sriov.demux[port - 1]; - struct mcast_group *group; - struct mcast_req *req; - int may_create = 0; - - if (ctx->flushing) - return -EAGAIN; - - switch (sa_mad->mad_hdr.method) { - case IB_MGMT_METHOD_SET: - may_create = 1; - case IB_SA_METHOD_DELETE: - req = kzalloc(sizeof *req, GFP_KERNEL); - if (!req) - return -ENOMEM; - - req->func = slave; - req->sa_mad = *sa_mad; - - mutex_lock(&ctx->mcg_table_lock); - group = acquire_group(ctx, &rec->mgid, may_create, GFP_KERNEL); - mutex_unlock(&ctx->mcg_table_lock); - if (IS_ERR(group)) { - kfree(req); - return PTR_ERR(group); - } - mutex_lock(&group->lock); - if (group->func[slave].num_pend_reqs > MAX_PEND_REQS_PER_FUNC) { - mutex_unlock(&group->lock); - mcg_warn_group(group, "Port %d, Func %d has too many pending requests (%d), dropping\n", - port, slave, MAX_PEND_REQS_PER_FUNC); - release_group(group, 0); - kfree(req); - return -ENOMEM; - } - ++group->func[slave].num_pend_reqs; - req->group = group; - queue_req(req); - mutex_unlock(&group->lock); - release_group(group, 0); - return 1; /* consumed */ - case IB_SA_METHOD_GET_TABLE: - case IB_MGMT_METHOD_GET_RESP: - case IB_SA_METHOD_GET_TABLE_RESP: - case IB_SA_METHOD_DELETE_RESP: - return 0; /* not consumed, pass-through */ - default: - mcg_warn("In multiplex, port %d, func %d: unexpected MCMember method: 0x%x, dropping\n", - port, slave, sa_mad->mad_hdr.method); - return 1; /* consumed */ - } -} - -static ssize_t sysfs_show_group(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct mcast_group *group = - container_of(attr, struct mcast_group, dentry); - struct mcast_req *req = NULL; - char pending_str[40]; - char state_str[40]; - ssize_t len = 0; - int f; - - if (group->state == MCAST_IDLE) - sprintf(state_str, "%s", get_state_string(group->state)); - else - sprintf(state_str, "%s(TID=0x%llx)", - get_state_string(group->state), - (long long)be64_to_cpu(group->last_req_tid)); - if (list_empty(&group->pending_list)) { - sprintf(pending_str, "No"); - } else { - req = list_first_entry(&group->pending_list, struct mcast_req, group_list); - sprintf(pending_str, "Yes(TID=0x%llx)", - (long long)be64_to_cpu( - req->sa_mad.mad_hdr.tid)); - } - len += sprintf(buf + len, "%1d [%02d,%02d,%02d] %4d %4s %5s ", - group->rec.scope_join_state & 0xf, - group->members[2], group->members[1], group->members[0], - atomic_read(&group->refcount), - pending_str, - state_str); - for (f = 0; f < MAX_VFS; ++f) - if (group->func[f].state == MCAST_MEMBER) - len += sprintf(buf + len, "%d[%1x] ", - f, group->func[f].join_state); - - len += sprintf(buf + len, "\t\t(%4hx %4x %2x %2x %2x %2x %2x " - "%4x %4x %2x %2x)\n", - be16_to_cpu(group->rec.pkey), - be32_to_cpu(group->rec.qkey), - (group->rec.mtusel_mtu & 0xc0) >> 6, - group->rec.mtusel_mtu & 0x3f, - group->rec.tclass, - (group->rec.ratesel_rate & 0xc0) >> 6, - group->rec.ratesel_rate & 0x3f, - (be32_to_cpu(group->rec.sl_flowlabel_hoplimit) & 0xf0000000) >> 28, - (be32_to_cpu(group->rec.sl_flowlabel_hoplimit) & 0x0fffff00) >> 8, - be32_to_cpu(group->rec.sl_flowlabel_hoplimit) & 0x000000ff, - group->rec.proxy_join); - - return len; -} - -int mlx4_ib_mcg_port_init(struct mlx4_ib_demux_ctx *ctx) -{ - char name[20]; - - atomic_set(&ctx->tid, 0); - sprintf(name, "mlx4_ib_mcg%d", ctx->port); - ctx->mcg_wq = create_singlethread_workqueue(name); - if (!ctx->mcg_wq) - return -ENOMEM; - - mutex_init(&ctx->mcg_table_lock); - ctx->mcg_table = RB_ROOT; - INIT_LIST_HEAD(&ctx->mcg_mgid0_list); - ctx->flushing = 0; - - return 0; -} - -static void force_clean_group(struct mcast_group *group) -{ - struct mcast_req *req, *tmp - ; - list_for_each_entry_safe(req, tmp, &group->pending_list, group_list) { - list_del(&req->group_list); - kfree(req); - } - del_sysfs_port_mcg_attr(group->demux->dev, group->demux->port, &group->dentry.attr); - rb_erase(&group->node, &group->demux->mcg_table); - kfree(group); -} - -static void _mlx4_ib_mcg_port_cleanup(struct mlx4_ib_demux_ctx *ctx, int destroy_wq) -{ - int i; - struct rb_node *p; - struct mcast_group *group; - unsigned long end; - int count; - - for (i = 0; i < MAX_VFS; ++i) - clean_vf_mcast(ctx, i); - - end = jiffies + msecs_to_jiffies(MAD_TIMEOUT_MS + 3000); - do { - count = 0; - mutex_lock(&ctx->mcg_table_lock); - for (p = rb_first(&ctx->mcg_table); p; p = rb_next(p)) - ++count; - mutex_unlock(&ctx->mcg_table_lock); - if (!count) - break; - - msleep(1); - } while (time_after(end, jiffies)); - - flush_workqueue(ctx->mcg_wq); - if (destroy_wq) - destroy_workqueue(ctx->mcg_wq); - - mutex_lock(&ctx->mcg_table_lock); - while ((p = rb_first(&ctx->mcg_table)) != NULL) { - group = rb_entry(p, struct mcast_group, node); - if (atomic_read(&group->refcount)) - mcg_warn_group(group, "group refcount %d!!! (pointer %p)\n", atomic_read(&group->refcount), group); - - force_clean_group(group); - } - mutex_unlock(&ctx->mcg_table_lock); -} - -struct clean_work { - struct work_struct work; - struct mlx4_ib_demux_ctx *ctx; - int destroy_wq; -}; - -static void mcg_clean_task(struct work_struct *work) -{ - struct clean_work *cw = container_of(work, struct clean_work, work); - - _mlx4_ib_mcg_port_cleanup(cw->ctx, cw->destroy_wq); - cw->ctx->flushing = 0; - kfree(cw); -} - -void mlx4_ib_mcg_port_cleanup(struct mlx4_ib_demux_ctx *ctx, int destroy_wq) -{ - struct clean_work *work; - - if (ctx->flushing) - return; - - ctx->flushing = 1; - - if (destroy_wq) { - _mlx4_ib_mcg_port_cleanup(ctx, destroy_wq); - ctx->flushing = 0; - return; - } - - work = kmalloc(sizeof *work, GFP_KERNEL); - if (!work) { - ctx->flushing = 0; - mcg_warn("failed allocating work for cleanup\n"); - return; - } - - work->ctx = ctx; - work->destroy_wq = destroy_wq; - INIT_WORK(&work->work, mcg_clean_task); - queue_work(clean_wq, &work->work); -} - -static void build_leave_mad(struct mcast_req *req) -{ - struct ib_sa_mad *mad = &req->sa_mad; - - mad->mad_hdr.method = IB_SA_METHOD_DELETE; -} - - -static void clear_pending_reqs(struct mcast_group *group, int vf) -{ - struct mcast_req *req, *tmp, *group_first = NULL; - int clear; - int pend = 0; - - if (!list_empty(&group->pending_list)) - group_first = list_first_entry(&group->pending_list, struct mcast_req, group_list); - - list_for_each_entry_safe(req, tmp, &group->func[vf].pending, func_list) { - clear = 1; - if (group_first == req && - (group->state == MCAST_JOIN_SENT || - group->state == MCAST_LEAVE_SENT)) { - clear = cancel_delayed_work(&group->timeout_work); - pend = !clear; - group->state = MCAST_IDLE; - } - if (clear) { - --group->func[vf].num_pend_reqs; - list_del(&req->group_list); - list_del(&req->func_list); - kfree(req); - atomic_dec(&group->refcount); - } - } - - if (!pend && (!list_empty(&group->func[vf].pending) || group->func[vf].num_pend_reqs)) { - mcg_warn_group(group, "DRIVER BUG: list_empty %d, num_pend_reqs %d\n", - list_empty(&group->func[vf].pending), group->func[vf].num_pend_reqs); - } -} - -static int push_deleteing_req(struct mcast_group *group, int slave) -{ - struct mcast_req *req; - struct mcast_req *pend_req; - - if (!group->func[slave].join_state) - return 0; - - req = kzalloc(sizeof *req, GFP_KERNEL); - if (!req) { - mcg_warn_group(group, "failed allocation - may leave stall groups\n"); - return -ENOMEM; - } - - if (!list_empty(&group->func[slave].pending)) { - pend_req = list_entry(group->func[slave].pending.prev, struct mcast_req, group_list); - if (pend_req->clean) { - kfree(req); - return 0; - } - } - - req->clean = 1; - req->func = slave; - req->group = group; - ++group->func[slave].num_pend_reqs; - build_leave_mad(req); - queue_req(req); - return 0; -} - -void clean_vf_mcast(struct mlx4_ib_demux_ctx *ctx, int slave) -{ - struct mcast_group *group; - struct rb_node *p; - - mutex_lock(&ctx->mcg_table_lock); - for (p = rb_first(&ctx->mcg_table); p; p = rb_next(p)) { - group = rb_entry(p, struct mcast_group, node); - mutex_lock(&group->lock); - if (atomic_read(&group->refcount)) { - /* clear pending requests of this VF */ - clear_pending_reqs(group, slave); - push_deleteing_req(group, slave); - } - mutex_unlock(&group->lock); - } - mutex_unlock(&ctx->mcg_table_lock); -} - - -int mlx4_ib_mcg_init(void) -{ - clean_wq = create_singlethread_workqueue("mlx4_ib_mcg"); - if (!clean_wq) - return -ENOMEM; - - return 0; -} - -void mlx4_ib_mcg_destroy(void) -{ - destroy_workqueue(clean_wq); -} Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mcg.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/doorbell.c =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/doorbell.c (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/doorbell.c (nonexistent) @@ -1,96 +0,0 @@ -/* - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include - -#include "mlx4_ib.h" - -struct mlx4_ib_user_db_page { - struct list_head list; - struct ib_umem *umem; - unsigned long user_virt; - int refcnt; -}; - -int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context, unsigned long virt, - struct mlx4_db *db) -{ - struct mlx4_ib_user_db_page *page; - int err = 0; - - mutex_lock(&context->db_page_mutex); - - list_for_each_entry(page, &context->db_page_list, list) - if (page->user_virt == (virt & PAGE_MASK)) - goto found; - - page = kmalloc(sizeof *page, GFP_KERNEL); - if (!page) { - err = -ENOMEM; - goto out; - } - - page->user_virt = (virt & PAGE_MASK); - page->refcnt = 0; - page->umem = ib_umem_get(&context->ibucontext, virt & PAGE_MASK, - PAGE_SIZE, 0, 0); - if (IS_ERR(page->umem)) { - err = PTR_ERR(page->umem); - kfree(page); - goto out; - } - - list_add(&page->list, &context->db_page_list); - -found: - db->dma = sg_dma_address(page->umem->sg_head.sgl) + (virt & ~PAGE_MASK); - db->u.user_page = page; - ++page->refcnt; - -out: - mutex_unlock(&context->db_page_mutex); - - return err; -} - -void mlx4_ib_db_unmap_user(struct mlx4_ib_ucontext *context, struct mlx4_db *db) -{ - mutex_lock(&context->db_page_mutex); - - if (!--db->u.user_page->refcnt) { - list_del(&db->u.user_page->list); - ib_umem_release(db->u.user_page->umem); - kfree(db->u.user_page); - } - - mutex_unlock(&context->db_page_mutex); -} Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/doorbell.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mr.c =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mr.c (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mr.c (nonexistent) @@ -1,885 +0,0 @@ -/* - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include - -#include "mlx4_ib.h" - -static u32 convert_access(int acc) -{ - return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX4_PERM_ATOMIC : 0) | - (acc & IB_ACCESS_REMOTE_WRITE ? MLX4_PERM_REMOTE_WRITE : 0) | - (acc & IB_ACCESS_REMOTE_READ ? MLX4_PERM_REMOTE_READ : 0) | - (acc & IB_ACCESS_LOCAL_WRITE ? MLX4_PERM_LOCAL_WRITE : 0) | - (acc & IB_ACCESS_MW_BIND ? MLX4_PERM_BIND_MW : 0) | - MLX4_PERM_LOCAL_READ; -} -/* No suuport for Shared MR feature */ -#if 0 -static ssize_t shared_mr_proc_read(struct file *file, - char __user *buffer, - size_t len, - loff_t *offset) -{ - - return -ENOSYS; - -} - -static ssize_t shared_mr_proc_write(struct file *file, - const char __user *buffer, - size_t len, - loff_t *offset) -{ - - return -ENOSYS; -} - -static int shared_mr_mmap(struct file *filep, struct vm_area_struct *vma) -{ - - struct proc_dir_entry *pde = PDE(filep->f_path.dentry->d_inode); - struct mlx4_shared_mr_info *smr_info = - (struct mlx4_shared_mr_info *)pde->data; - - /* Prevent any mapping not on start of area */ - if (vma->vm_pgoff != 0) - return -EINVAL; - - return ib_umem_map_to_vma(smr_info->umem, - vma); - -} - -static const struct file_operations shared_mr_proc_ops = { - .owner = THIS_MODULE, - .read = shared_mr_proc_read, - .write = shared_mr_proc_write, - .mmap = shared_mr_mmap -}; - -static mode_t convert_shared_access(int acc) -{ - - return (acc & IB_ACCESS_SHARED_MR_USER_READ ? S_IRUSR : 0) | - (acc & IB_ACCESS_SHARED_MR_USER_WRITE ? S_IWUSR : 0) | - (acc & IB_ACCESS_SHARED_MR_GROUP_READ ? S_IRGRP : 0) | - (acc & IB_ACCESS_SHARED_MR_GROUP_WRITE ? S_IWGRP : 0) | - (acc & IB_ACCESS_SHARED_MR_OTHER_READ ? S_IROTH : 0) | - (acc & IB_ACCESS_SHARED_MR_OTHER_WRITE ? S_IWOTH : 0); - -} -#endif -struct ib_mr *mlx4_ib_get_dma_mr(struct ib_pd *pd, int acc) -{ - struct mlx4_ib_mr *mr; - int err; - - mr = kzalloc(sizeof *mr, GFP_KERNEL); - if (!mr) - return ERR_PTR(-ENOMEM); - - err = mlx4_mr_alloc(to_mdev(pd->device)->dev, to_mpd(pd)->pdn, 0, - ~0ull, convert_access(acc), 0, 0, &mr->mmr); - if (err) - goto err_free; - - err = mlx4_mr_enable(to_mdev(pd->device)->dev, &mr->mmr); - if (err) - goto err_mr; - - mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; - mr->umem = NULL; - - return &mr->ibmr; - -err_mr: - (void) mlx4_mr_free(to_mdev(pd->device)->dev, &mr->mmr); - -err_free: - kfree(mr); - - return ERR_PTR(err); -} - -static int mlx4_ib_umem_write_mtt_block(struct mlx4_ib_dev *dev, - struct mlx4_mtt *mtt, - u64 mtt_size, - u64 mtt_shift, - u64 len, - u64 cur_start_addr, - u64 *pages, - int *start_index, - int *npages) -{ - int k; - int err = 0; - u64 mtt_entries; - u64 cur_end_addr = cur_start_addr + len; - u64 cur_end_addr_aligned = 0; - - len += (cur_start_addr & (mtt_size-1ULL)); - cur_end_addr_aligned = round_up(cur_end_addr, mtt_size); - len += (cur_end_addr_aligned - cur_end_addr); - if (len & (mtt_size-1ULL)) { - WARN(1 , - "write_block: len %llx is not aligned to mtt_size %llx\n", - (unsigned long long)len, (unsigned long long)mtt_size); - return -EINVAL; - } - - - mtt_entries = (len >> mtt_shift); - - /* Align the MTT start address to - the mtt_size. - Required to handle cases when the MR - starts in the middle of an MTT record. - Was not required in old code since - the physical addresses provided by - the dma subsystem were page aligned, - which was also the MTT size. - */ - cur_start_addr = round_down(cur_start_addr, mtt_size); - /* A new block is started ...*/ - for (k = 0; k < mtt_entries; ++k) { - pages[*npages] = cur_start_addr + (mtt_size * k); - (*npages)++; - /* - * Be friendly to mlx4_write_mtt() and - * pass it chunks of appropriate size. - */ - if (*npages == PAGE_SIZE / sizeof(u64)) { - err = mlx4_write_mtt(dev->dev, - mtt, *start_index, - *npages, pages); - if (err) - return err; - - (*start_index) += *npages; - *npages = 0; - } - } - - return 0; -} - -int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt, - struct ib_umem *umem) -{ - u64 *pages; - u64 len = 0; - int err = 0; - u64 mtt_size; - u64 cur_start_addr = 0; - u64 mtt_shift; - int start_index = 0; - int npages = 0; - struct scatterlist *sg; - int i; - - pages = (u64 *) __get_free_page(GFP_KERNEL); - if (!pages) - return -ENOMEM; - - mtt_shift = mtt->page_shift; - mtt_size = 1ULL << mtt_shift; - - for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { - if (cur_start_addr + len == - sg_dma_address(sg)) { - /* still the same block */ - len += sg_dma_len(sg); - continue; - } - /* A new block is started ...*/ - /* If len is malaligned, write an extra mtt entry to - cover the misaligned area (round up the division) - */ - err = mlx4_ib_umem_write_mtt_block(dev, - mtt, mtt_size, mtt_shift, - len, cur_start_addr, - pages, - &start_index, - &npages); - if (err) - goto out; - - cur_start_addr = - sg_dma_address(sg); - len = sg_dma_len(sg); - } - - /* Handle the last block */ - if (len > 0) { - /* If len is malaligned, write an extra mtt entry to cover - the misaligned area (round up the division) - */ - err = mlx4_ib_umem_write_mtt_block(dev, - mtt, mtt_size, mtt_shift, - len, cur_start_addr, - pages, - &start_index, - &npages); - if (err) - goto out; - } - - - if (npages) - err = mlx4_write_mtt(dev->dev, mtt, start_index, npages, pages); - -out: - free_page((unsigned long) pages); - return err; -} - -static inline u64 alignment_of(u64 ptr) -{ - return ilog2(ptr & (~(ptr-1))); -} - -static int mlx4_ib_umem_calc_block_mtt(u64 next_block_start, - u64 current_block_end, - u64 block_shift) -{ - /* Check whether the alignment of the new block - is aligned as well as the previous block. - Block address must start with zeros till size of entity_size. - */ - if ((next_block_start & ((1ULL << block_shift) - 1ULL)) != 0) - /* It is not as well aligned as the - previous block-reduce the mtt size - accordingly. - Here we take the last right bit - which is 1. - */ - block_shift = alignment_of(next_block_start); - - /* Check whether the alignment of the - end of previous block - is it aligned - as well as the start of the block - */ - if (((current_block_end) & ((1ULL << block_shift) - 1ULL)) != 0) - /* It is not as well aligned as - the start of the block - reduce the - mtt size accordingly. - */ - block_shift = alignment_of(current_block_end); - - return block_shift; -} - -/* Calculate optimal mtt size based on contiguous pages. -* Function will return also the number of pages that are not aligned to the - calculated mtt_size to be added to total number - of pages. For that we should check the first chunk length & last chunk - length and if not aligned to mtt_size we should increment - the non_aligned_pages number. - All chunks in the middle already handled as part of mtt shift calculation - for both their start & end addresses. -*/ -int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, - u64 start_va, - int *num_of_mtts) -{ - u64 block_shift = MLX4_MAX_MTT_SHIFT; - u64 current_block_len = 0; - u64 current_block_start = 0; - u64 misalignment_bits; - u64 first_block_start = 0; - u64 last_block_end = 0; - u64 total_len = 0; - u64 last_block_aligned_end = 0; - u64 min_shift = ilog2(umem->page_size); - struct scatterlist *sg; - int i; - u64 next_block_start; - u64 current_block_end; - - for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { - /* Initialization - save the first chunk start as - the current_block_start - block means contiguous pages. - */ - if (current_block_len == 0 && current_block_start == 0) { - first_block_start = current_block_start = - sg_dma_address(sg); - /* Find the bits that are different between - the physical address and the virtual - address for the start of the MR. - */ - /* umem_get aligned the start_va to a page - boundary. Therefore, we need to align the - start va to the same boundary */ - /* misalignment_bits is needed to handle the - case of a single memory region. In this - case, the rest of the logic will not reduce - the block size. If we use a block size - which is bigger than the alignment of the - misalignment bits, we might use the virtual - page number instead of the physical page - number, resulting in access to the wrong - data. */ - misalignment_bits = - (start_va & (~(((u64)(umem->page_size))-1ULL))) - ^ current_block_start; - block_shift = min(alignment_of(misalignment_bits) - , block_shift); - } - - /* Go over the scatter entries and check - if they continue the previous scatter entry. - */ - next_block_start = - sg_dma_address(sg); - current_block_end = current_block_start - + current_block_len; - /* If we have a split (non-contig.) between two block*/ - if (current_block_end != next_block_start) { - block_shift = mlx4_ib_umem_calc_block_mtt( - next_block_start, - current_block_end, - block_shift); - - /* If we reached the minimum shift for 4k - page we stop the loop. - */ - if (block_shift <= min_shift) - goto end; - - /* If not saved yet we are in first block - - we save the length of first block to - calculate the non_aligned_pages number at - * the end. - */ - total_len += current_block_len; - - /* Start a new block */ - current_block_start = next_block_start; - current_block_len = - sg_dma_len(sg); - continue; - } - /* The scatter entry is another part of - the current block, increase the block size - * An entry in the scatter can be larger than - 4k (page) as of dma mapping - which merge some blocks together. - */ - current_block_len += - sg_dma_len(sg); - } - - /* Account for the last block in the total len */ - total_len += current_block_len; - /* Add to the first block the misalignment that it suffers from.*/ - total_len += (first_block_start & ((1ULL<> block_shift; -end: - if (block_shift < min_shift) { - /* If shift is less than the min we set a WARN and - return the min shift. - */ - WARN(1, - "mlx4_ib_umem_calc_optimal_mtt_size - unexpected shift %lld\n", - (unsigned long long)block_shift); - - block_shift = min_shift; - } - return block_shift; - -} - -/* No suuport for Shared MR */ -#if 0 -static int prepare_shared_mr(struct mlx4_ib_mr *mr, int access_flags, int mr_id) -{ - - struct proc_dir_entry *mr_proc_entry; - mode_t mode = S_IFREG; - char name_buff[16]; - - mode |= convert_shared_access(access_flags); - sprintf(name_buff, "%X", mr_id); - mr->smr_info = kmalloc(sizeof(struct mlx4_shared_mr_info), GFP_KERNEL); - mr->smr_info->mr_id = mr_id; - mr->smr_info->umem = mr->umem; - - mr_proc_entry = proc_create_data(name_buff, mode, - mlx4_mrs_dir_entry, - &shared_mr_proc_ops, - mr->smr_info); - - if (!mr_proc_entry) { - pr_err("prepare_shared_mr failed via proc\n"); - kfree(mr->smr_info); - return -ENODEV; - } - - current_uid_gid(&(mr_proc_entry->uid), &(mr_proc_entry->gid)); - mr_proc_entry->size = mr->umem->length; - return 0; - -} -static int is_shared_mr(int access_flags) -{ - /* We should check whether IB_ACCESS_SHARED_MR_USER_READ or - other shared bits were turned on. - */ - return !!(access_flags & (IB_ACCESS_SHARED_MR_USER_READ | - IB_ACCESS_SHARED_MR_USER_WRITE | - IB_ACCESS_SHARED_MR_GROUP_READ | - IB_ACCESS_SHARED_MR_GROUP_WRITE | - IB_ACCESS_SHARED_MR_OTHER_READ | - IB_ACCESS_SHARED_MR_OTHER_WRITE)); - -} - -static void free_smr_info(struct mlx4_ib_mr *mr) -{ - /* When master/parent shared mr is dereged there is - no ability to share this mr any more - its mr_id will be - returned to the kernel as part of ib_uverbs_dereg_mr - and may be allocated again as part of other reg_mr. - */ - char name_buff[16]; - - sprintf(name_buff, "%X", mr->smr_info->mr_id); - /* Remove proc entry is checking internally that no operation - was strated on that proc fs file and if in the middle - current process will wait till end of operation. - That's why no sync mechanism is needed when we release - below the shared umem. - */ - remove_proc_entry(name_buff, mlx4_mrs_dir_entry); - kfree(mr->smr_info); - mr->smr_info = NULL; -} -#endif - -static void mlx4_invalidate_umem(void *invalidation_cookie, - struct ib_umem *umem, - unsigned long addr, size_t size) -{ - struct mlx4_ib_mr *mr = (struct mlx4_ib_mr *)invalidation_cookie; - - /* This function is called under client peer lock so its resources are race protected */ - if (atomic_inc_return(&mr->invalidated) > 1) { - umem->invalidation_ctx->inflight_invalidation = 1; - goto end; - } - - umem->invalidation_ctx->peer_callback = 1; - mlx4_mr_free(to_mdev(mr->ibmr.device)->dev, &mr->mmr); - ib_umem_release(umem); - complete(&mr->invalidation_comp); - -end: - return; - -} - -struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, - u64 virt_addr, int access_flags, - struct ib_udata *udata, - int mr_id) -{ - struct mlx4_ib_dev *dev = to_mdev(pd->device); - struct mlx4_ib_mr *mr; - int shift; - int err; - int n; - struct ib_peer_memory_client *ib_peer_mem; - - mr = kzalloc(sizeof *mr, GFP_KERNEL); - if (!mr) - return ERR_PTR(-ENOMEM); - - mr->umem = ib_umem_get_ex(pd->uobject->context, start, length, - access_flags, 0, 1); - if (IS_ERR(mr->umem)) { - err = PTR_ERR(mr->umem); - goto err_free; - } - - ib_peer_mem = mr->umem->ib_peer_mem; - n = ib_umem_page_count(mr->umem); - shift = mlx4_ib_umem_calc_optimal_mtt_size(mr->umem, start, - &n); - err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, virt_addr, length, - convert_access(access_flags), n, shift, &mr->mmr); - if (err) - goto err_umem; - - err = mlx4_ib_umem_write_mtt(dev, &mr->mmr.mtt, mr->umem); - if (err) - goto err_mr; - - err = mlx4_mr_enable(dev->dev, &mr->mmr); - if (err) - goto err_mr; - - mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; -/* No suuport for Shared MR */ -#if 0 - /* Check whether MR should be shared */ - if (is_shared_mr(access_flags)) { - /* start address and length must be aligned to page size in order - to map a full page and preventing leakage of data */ - if (mr->umem->offset || (length & ~PAGE_MASK)) { - err = -EINVAL; - goto err_mr; - } - - err = prepare_shared_mr(mr, access_flags, mr_id); - if (err) - goto err_mr; - } -#endif - if (ib_peer_mem) { - if (access_flags & IB_ACCESS_MW_BIND) { - /* Prevent binding MW on peer clients. - * mlx4_invalidate_umem must be void, - * therefore, mlx4_mr_free should not fail - * when using peer clients. */ - err = -ENOSYS; - pr_err("MW is not supported with peer memory client"); - goto err_smr; - } - init_completion(&mr->invalidation_comp); - ib_umem_activate_invalidation_notifier(mr->umem, - mlx4_invalidate_umem, mr); - } - - atomic_set(&mr->invalidated, 0); - return &mr->ibmr; - -err_smr: -/* No suuport for Shared MR */ -#if 0 - if (mr->smr_info) - free_smr_info(mr); -#endif -err_mr: - (void) mlx4_mr_free(to_mdev(pd->device)->dev, &mr->mmr); - -err_umem: - ib_umem_release(mr->umem); - -err_free: - kfree(mr); - - return ERR_PTR(err); -} - -int mlx4_ib_dereg_mr(struct ib_mr *ibmr) -{ - struct mlx4_ib_mr *mr = to_mmr(ibmr); - struct ib_umem *umem = mr->umem; - int ret; - -/* No suuport for Shared MR */ -#if 0 - if (mr->smr_info) - free_smr_info(mr); -#endif - - if (atomic_inc_return(&mr->invalidated) > 1) { - wait_for_completion(&mr->invalidation_comp); - goto end; - } - - ret = mlx4_mr_free(to_mdev(ibmr->device)->dev, &mr->mmr); - if (ret) { - /* Error is not expected here, except when memory windows - * are bound to MR which is not supported with - * peer memory clients */ - atomic_set(&mr->invalidated, 0); - return ret; - } - - if (!umem) - goto end; - - ib_umem_release(mr->umem); -end: - - kfree(mr); - - return 0; -} - -struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type) -{ - struct mlx4_ib_dev *dev = to_mdev(pd->device); - struct mlx4_ib_mw *mw; - int err; - - mw = kmalloc(sizeof(*mw), GFP_KERNEL); - if (!mw) - return ERR_PTR(-ENOMEM); - - err = mlx4_mw_alloc(dev->dev, to_mpd(pd)->pdn, (enum mlx4_mw_type)type, &mw->mmw); - if (err) - goto err_free; - - err = mlx4_mw_enable(dev->dev, &mw->mmw); - if (err) - goto err_mw; - - mw->ibmw.rkey = mw->mmw.key; - - return &mw->ibmw; - -err_mw: - mlx4_mw_free(dev->dev, &mw->mmw); - -err_free: - kfree(mw); - - return ERR_PTR(err); -} - -int mlx4_ib_bind_mw(struct ib_qp *qp, struct ib_mw *mw, - struct ib_mw_bind *mw_bind) -{ - struct ib_send_wr wr; - struct ib_send_wr *bad_wr; - int ret; - - memset(&wr, 0, sizeof(wr)); - wr.opcode = IB_WR_BIND_MW; - wr.wr_id = mw_bind->wr_id; - wr.send_flags = mw_bind->send_flags; - wr.wr.bind_mw.mw = mw; - wr.wr.bind_mw.bind_info = mw_bind->bind_info; - wr.wr.bind_mw.rkey = ib_inc_rkey(mw->rkey); - - ret = mlx4_ib_post_send(qp, &wr, &bad_wr); - if (!ret) - mw->rkey = wr.wr.bind_mw.rkey; - - return ret; -} - -int mlx4_ib_dealloc_mw(struct ib_mw *ibmw) -{ - struct mlx4_ib_mw *mw = to_mmw(ibmw); - - mlx4_mw_free(to_mdev(ibmw->device)->dev, &mw->mmw); - kfree(mw); - - return 0; -} - -struct ib_mr *mlx4_ib_alloc_fast_reg_mr(struct ib_pd *pd, - int max_page_list_len) -{ - struct mlx4_ib_dev *dev = to_mdev(pd->device); - struct mlx4_ib_mr *mr; - int err; - - mr = kzalloc(sizeof *mr, GFP_KERNEL); - if (!mr) - return ERR_PTR(-ENOMEM); - - err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, 0, 0, 0, - max_page_list_len, 0, &mr->mmr); - if (err) - goto err_free; - - err = mlx4_mr_enable(dev->dev, &mr->mmr); - if (err) - goto err_mr; - - mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; - mr->umem = NULL; - - return &mr->ibmr; - -err_mr: - (void) mlx4_mr_free(dev->dev, &mr->mmr); - -err_free: - kfree(mr); - return ERR_PTR(err); -} - -struct ib_fast_reg_page_list *mlx4_ib_alloc_fast_reg_page_list(struct ib_device *ibdev, - int page_list_len) -{ - struct mlx4_ib_dev *dev = to_mdev(ibdev); - struct mlx4_ib_fast_reg_page_list *mfrpl; - int size = page_list_len * sizeof (u64); - - if (page_list_len > MLX4_MAX_FAST_REG_PAGES) - return ERR_PTR(-EINVAL); - - mfrpl = kmalloc(sizeof *mfrpl, GFP_KERNEL); - if (!mfrpl) - return ERR_PTR(-ENOMEM); - - mfrpl->ibfrpl.page_list = kmalloc(size, GFP_KERNEL); - if (!mfrpl->ibfrpl.page_list) - goto err_free; - - mfrpl->mapped_page_list = dma_alloc_coherent(&dev->dev->pdev->dev, - size, &mfrpl->map, - GFP_KERNEL); - if (!mfrpl->mapped_page_list) - goto err_free; - - WARN_ON(mfrpl->map & 0x3f); - - return &mfrpl->ibfrpl; - -err_free: - kfree(mfrpl->ibfrpl.page_list); - kfree(mfrpl); - return ERR_PTR(-ENOMEM); -} - -void mlx4_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list) -{ - struct mlx4_ib_dev *dev = to_mdev(page_list->device); - struct mlx4_ib_fast_reg_page_list *mfrpl = to_mfrpl(page_list); - int size = page_list->max_page_list_len * sizeof (u64); - - dma_free_coherent(&dev->dev->pdev->dev, size, mfrpl->mapped_page_list, - mfrpl->map); - kfree(mfrpl->ibfrpl.page_list); - kfree(mfrpl); -} - -struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int acc, - struct ib_fmr_attr *fmr_attr) -{ - struct mlx4_ib_dev *dev = to_mdev(pd->device); - struct mlx4_ib_fmr *fmr; - int err = -ENOMEM; - - fmr = kmalloc(sizeof *fmr, GFP_KERNEL); - if (!fmr) - return ERR_PTR(-ENOMEM); - - err = mlx4_fmr_alloc(dev->dev, to_mpd(pd)->pdn, convert_access(acc), - fmr_attr->max_pages, fmr_attr->max_maps, - fmr_attr->page_shift, &fmr->mfmr); - if (err) - goto err_free; - - err = mlx4_fmr_enable(to_mdev(pd->device)->dev, &fmr->mfmr); - if (err) - goto err_mr; - - fmr->ibfmr.rkey = fmr->ibfmr.lkey = fmr->mfmr.mr.key; - - return &fmr->ibfmr; - -err_mr: - (void) mlx4_mr_free(to_mdev(pd->device)->dev, &fmr->mfmr.mr); - -err_free: - kfree(fmr); - - return ERR_PTR(err); -} - -int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, - int npages, u64 iova) -{ - struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr); - struct mlx4_ib_dev *dev = to_mdev(ifmr->ibfmr.device); - - return mlx4_map_phys_fmr(dev->dev, &ifmr->mfmr, page_list, npages, iova, - &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey); -} - -int mlx4_ib_unmap_fmr(struct list_head *fmr_list) -{ - struct ib_fmr *ibfmr; - int err; - struct mlx4_dev *mdev = NULL; - - list_for_each_entry(ibfmr, fmr_list, list) { - if (mdev && to_mdev(ibfmr->device)->dev != mdev) - return -EINVAL; - mdev = to_mdev(ibfmr->device)->dev; - } - - if (!mdev) - return 0; - - list_for_each_entry(ibfmr, fmr_list, list) { - struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr); - - mlx4_fmr_unmap(mdev, &ifmr->mfmr, &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey); - } - - /* - * Make sure all MPT status updates are visible before issuing - * SYNC_TPT firmware command. - */ - wmb(); - - err = mlx4_SYNC_TPT(mdev); - if (err) - pr_warn("SYNC_TPT error %d when " - "unmapping FMRs\n", err); - - return 0; -} - -int mlx4_ib_fmr_dealloc(struct ib_fmr *ibfmr) -{ - struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr); - struct mlx4_ib_dev *dev = to_mdev(ibfmr->device); - int err; - - err = mlx4_fmr_free(dev->dev, &ifmr->mfmr); - - if (!err) - kfree(ifmr); - - return err; -} Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mr.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/qp.c =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/qp.c (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/qp.c (nonexistent) @@ -1,3687 +0,0 @@ -/* - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include - -#include "mlx4_ib.h" -#include "user.h" - -#define asm __asm - -enum { - MLX4_IB_ACK_REQ_FREQ = 8, -}; - -enum { - MLX4_IB_DEFAULT_SCHED_QUEUE = 0x83, - MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f, - MLX4_IB_LINK_TYPE_IB = 0, - MLX4_IB_LINK_TYPE_ETH = 1 -}; - -enum { - /* - * Largest possible UD header: send with GRH and immediate - * data plus 18 bytes for an Ethernet header with VLAN/802.1Q - * tag. (LRH would only use 8 bytes, so Ethernet is the - * biggest case) - */ - MLX4_IB_UD_HEADER_SIZE = 82, - MLX4_IB_LSO_HEADER_SPARE = 128, -}; - -enum { - MLX4_IB_IBOE_ETHERTYPE = 0x8915 -}; - -struct mlx4_ib_sqp { - struct mlx4_ib_qp qp; - int pkey_index; - u32 qkey; - u32 send_psn; - struct ib_ud_header ud_header; - u8 header_buf[MLX4_IB_UD_HEADER_SIZE]; -}; - -enum { - MLX4_IB_MIN_SQ_STRIDE = 6, - MLX4_IB_CACHE_LINE_SIZE = 64, -}; - -enum { - MLX4_RAW_QP_MTU = 7, - MLX4_RAW_QP_MSGMAX = 31, -}; - -static const __be32 mlx4_ib_opcode[] = { - [IB_WR_SEND] = cpu_to_be32(MLX4_OPCODE_SEND), - [IB_WR_LSO] = cpu_to_be32(MLX4_OPCODE_LSO), - [IB_WR_SEND_WITH_IMM] = cpu_to_be32(MLX4_OPCODE_SEND_IMM), - [IB_WR_RDMA_WRITE] = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE), - [IB_WR_RDMA_WRITE_WITH_IMM] = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM), - [IB_WR_RDMA_READ] = cpu_to_be32(MLX4_OPCODE_RDMA_READ), - [IB_WR_ATOMIC_CMP_AND_SWP] = cpu_to_be32(MLX4_OPCODE_ATOMIC_CS), - [IB_WR_ATOMIC_FETCH_AND_ADD] = cpu_to_be32(MLX4_OPCODE_ATOMIC_FA), - [IB_WR_SEND_WITH_INV] = cpu_to_be32(MLX4_OPCODE_SEND_INVAL), - [IB_WR_LOCAL_INV] = cpu_to_be32(MLX4_OPCODE_LOCAL_INVAL), - [IB_WR_FAST_REG_MR] = cpu_to_be32(MLX4_OPCODE_FMR), - [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_CS), - [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_FA), - [IB_WR_BIND_MW] = cpu_to_be32( - MLX4_OPCODE_BIND_MW), -}; - -#ifndef wc_wmb - #if defined(__i386__) - #define wc_wmb() asm volatile("lock; addl $0,0(%%esp) " ::: "memory") - #elif defined(__x86_64__) - #define wc_wmb() asm volatile("sfence" ::: "memory") - #elif defined(__ia64__) - #define wc_wmb() asm volatile("fwb" ::: "memory") - #else - #define wc_wmb() wmb() - #endif -#endif - -static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp) -{ - return container_of(mqp, struct mlx4_ib_sqp, qp); -} - -static int is_tunnel_qp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) -{ - if (!mlx4_is_master(dev->dev)) - return 0; - - return qp->mqp.qpn >= dev->dev->phys_caps.base_tunnel_sqpn && - qp->mqp.qpn < dev->dev->phys_caps.base_tunnel_sqpn + - 8 * MLX4_MFUNC_MAX; -} - -static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) -{ - int proxy_sqp = 0; - int real_sqp = 0; - int i; - /* PPF or Native -- real SQP */ - real_sqp = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) && - qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn && - qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 3); - if (real_sqp) - return 1; - /* VF or PF -- proxy SQP */ - if (mlx4_is_mfunc(dev->dev)) { - for (i = 0; i < dev->dev->caps.num_ports; i++) { - if (qp->mqp.qpn == dev->dev->caps.qp0_proxy[i] || - qp->mqp.qpn == dev->dev->caps.qp1_proxy[i]) { - proxy_sqp = 1; - break; - } - } - } - return proxy_sqp; -} - -/* used for INIT/CLOSE port logic */ -static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) -{ - int proxy_qp0 = 0; - int real_qp0 = 0; - int i; - /* PPF or Native -- real QP0 */ - real_qp0 = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) && - qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn && - qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 1); - if (real_qp0) - return 1; - /* VF or PF -- proxy QP0 */ - if (mlx4_is_mfunc(dev->dev)) { - for (i = 0; i < dev->dev->caps.num_ports; i++) { - if (qp->mqp.qpn == dev->dev->caps.qp0_proxy[i]) { - proxy_qp0 = 1; - break; - } - } - } - return proxy_qp0; -} - -static void *get_wqe(struct mlx4_ib_qp *qp, int offset) -{ - return mlx4_buf_offset(&qp->buf, offset); -} - -static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n) -{ - return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift)); -} - -static void *get_send_wqe(struct mlx4_ib_qp *qp, int n) -{ - return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift)); -} - -/* - * Stamp a SQ WQE so that it is invalid if prefetched by marking the - * first four bytes of every 64 byte chunk with - * 0x7FFFFFF | (invalid_ownership_value << 31). - * - * When the max work request size is less than or equal to the WQE - * basic block size, as an optimization, we can stamp all WQEs with - * 0xffffffff, and skip the very first chunk of each WQE. - */ -static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n, int size) -{ - __be32 *wqe; - int i; - int s; - int ind; - void *buf; - __be32 stamp; - struct mlx4_wqe_ctrl_seg *ctrl; - - if (qp->sq_max_wqes_per_wr > 1) { - s = roundup(size, 1U << qp->sq.wqe_shift); - for (i = 0; i < s; i += 64) { - ind = (i >> qp->sq.wqe_shift) + n; - stamp = ind & qp->sq.wqe_cnt ? cpu_to_be32(0x7fffffff) : - cpu_to_be32(0xffffffff); - buf = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1)); - wqe = buf + (i & ((1 << qp->sq.wqe_shift) - 1)); - *wqe = stamp; - } - } else { - ctrl = buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1)); - s = (ctrl->fence_size & 0x3f) << 4; - for (i = 64; i < s; i += 64) { - wqe = buf + i; - *wqe = cpu_to_be32(0xffffffff); - } - } -} - -static void post_nop_wqe(struct mlx4_ib_qp *qp, int n, int size) -{ - struct mlx4_wqe_ctrl_seg *ctrl; - struct mlx4_wqe_inline_seg *inl; - void *wqe; - int s; - - ctrl = wqe = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1)); - s = sizeof(struct mlx4_wqe_ctrl_seg); - - if (qp->ibqp.qp_type == IB_QPT_UD) { - struct mlx4_wqe_datagram_seg *dgram = wqe + sizeof *ctrl; - struct mlx4_av *av = (struct mlx4_av *)dgram->av; - memset(dgram, 0, sizeof *dgram); - av->port_pd = cpu_to_be32((qp->port << 24) | to_mpd(qp->ibqp.pd)->pdn); - s += sizeof(struct mlx4_wqe_datagram_seg); - } - - /* Pad the remainder of the WQE with an inline data segment. */ - if (size > s) { - inl = wqe + s; - inl->byte_count = cpu_to_be32(1 << 31 | (size - s - sizeof *inl)); - } - ctrl->srcrb_flags = 0; - ctrl->fence_size = size / 16; - /* - * Make sure descriptor is fully written before setting ownership bit - * (because HW can start executing as soon as we do). - */ - wmb(); - - ctrl->owner_opcode = cpu_to_be32(MLX4_OPCODE_NOP | MLX4_WQE_CTRL_NEC) | - (n & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0); - - stamp_send_wqe(qp, n + qp->sq_spare_wqes, size); -} - -/* Post NOP WQE to prevent wrap-around in the middle of WR */ -static inline unsigned pad_wraparound(struct mlx4_ib_qp *qp, int ind) -{ - unsigned s = qp->sq.wqe_cnt - (ind & (qp->sq.wqe_cnt - 1)); - if (unlikely(s < qp->sq_max_wqes_per_wr)) { - post_nop_wqe(qp, ind, s << qp->sq.wqe_shift); - ind += s; - } - return ind; -} - -static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type) -{ - struct ib_event event; - struct ib_qp *ibqp = &to_mibqp(qp)->ibqp; - - if (type == MLX4_EVENT_TYPE_PATH_MIG) - to_mibqp(qp)->port = to_mibqp(qp)->alt_port; - - if (ibqp->event_handler) { - event.device = ibqp->device; - event.element.qp = ibqp; - switch (type) { - case MLX4_EVENT_TYPE_PATH_MIG: - event.event = IB_EVENT_PATH_MIG; - break; - case MLX4_EVENT_TYPE_COMM_EST: - event.event = IB_EVENT_COMM_EST; - break; - case MLX4_EVENT_TYPE_SQ_DRAINED: - event.event = IB_EVENT_SQ_DRAINED; - break; - case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE: - event.event = IB_EVENT_QP_LAST_WQE_REACHED; - break; - case MLX4_EVENT_TYPE_WQ_CATAS_ERROR: - event.event = IB_EVENT_QP_FATAL; - break; - case MLX4_EVENT_TYPE_PATH_MIG_FAILED: - event.event = IB_EVENT_PATH_MIG_ERR; - break; - case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR: - event.event = IB_EVENT_QP_REQ_ERR; - break; - case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR: - event.event = IB_EVENT_QP_ACCESS_ERR; - break; - default: - pr_warn("Unexpected event type %d " - "on QP %06x\n", type, qp->qpn); - return; - } - - ibqp->event_handler(&event, ibqp->qp_context); - } -} - -static int send_wqe_overhead(enum mlx4_ib_qp_type type, u32 flags) -{ - /* - * UD WQEs must have a datagram segment. - * RC and UC WQEs might have a remote address segment. - * MLX WQEs need two extra inline data segments (for the UD - * header and space for the ICRC). - */ - switch (type) { - case MLX4_IB_QPT_UD: - return sizeof (struct mlx4_wqe_ctrl_seg) + - sizeof (struct mlx4_wqe_datagram_seg) + - ((flags & MLX4_IB_QP_LSO) ? MLX4_IB_LSO_HEADER_SPARE : 0); - case MLX4_IB_QPT_PROXY_SMI_OWNER: - case MLX4_IB_QPT_PROXY_SMI: - case MLX4_IB_QPT_PROXY_GSI: - return sizeof (struct mlx4_wqe_ctrl_seg) + - sizeof (struct mlx4_wqe_datagram_seg) + 64; - case MLX4_IB_QPT_TUN_SMI_OWNER: - case MLX4_IB_QPT_TUN_GSI: - return sizeof (struct mlx4_wqe_ctrl_seg) + - sizeof (struct mlx4_wqe_datagram_seg); - - case MLX4_IB_QPT_UC: - return sizeof (struct mlx4_wqe_ctrl_seg) + - sizeof (struct mlx4_wqe_raddr_seg); - case MLX4_IB_QPT_RC: - return sizeof (struct mlx4_wqe_ctrl_seg) + - sizeof (struct mlx4_wqe_masked_atomic_seg) + - sizeof (struct mlx4_wqe_raddr_seg); - case MLX4_IB_QPT_SMI: - case MLX4_IB_QPT_GSI: - return sizeof (struct mlx4_wqe_ctrl_seg) + - ALIGN(MLX4_IB_UD_HEADER_SIZE + - DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE, - MLX4_INLINE_ALIGN) * - sizeof (struct mlx4_wqe_inline_seg), - sizeof (struct mlx4_wqe_data_seg)) + - ALIGN(4 + - sizeof (struct mlx4_wqe_inline_seg), - sizeof (struct mlx4_wqe_data_seg)); - default: - return sizeof (struct mlx4_wqe_ctrl_seg); - } -} - -static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap, - int is_user, int has_rq, struct mlx4_ib_qp *qp) -{ - /* Sanity check RQ size before proceeding */ - if (cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE || - cap->max_recv_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg)) - return -EINVAL; - - if (!has_rq) { - if (cap->max_recv_wr) - return -EINVAL; - - qp->rq.wqe_cnt = qp->rq.max_gs = 0; - } else { - /* HW requires >= 1 RQ entry with >= 1 gather entry */ - if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge)) - return -EINVAL; - - qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, cap->max_recv_wr)); - qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge)); - qp->rq.wqe_shift = ilog2(qp->rq.max_gs * sizeof (struct mlx4_wqe_data_seg)); - } - - /* leave userspace return values as they were, so as not to break ABI */ - if (is_user) { - cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt; - cap->max_recv_sge = qp->rq.max_gs; - } else { - cap->max_recv_wr = qp->rq.max_post = - min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE, qp->rq.wqe_cnt); - cap->max_recv_sge = min(qp->rq.max_gs, - min(dev->dev->caps.max_sq_sg, - dev->dev->caps.max_rq_sg)); - } - - return 0; -} - -static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap, - enum mlx4_ib_qp_type type, struct mlx4_ib_qp *qp) -{ - int s; - - /* Sanity check SQ size before proceeding */ - if (cap->max_send_wr > (dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE) || - cap->max_send_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) || - cap->max_inline_data + send_wqe_overhead(type, qp->flags) + - sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz) - return -EINVAL; - - /* - * For MLX transport we need 2 extra S/G entries: - * one for the header and one for the checksum at the end - */ - if ((type == MLX4_IB_QPT_SMI || type == MLX4_IB_QPT_GSI || - type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) && - cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg) - return -EINVAL; - - s = max(cap->max_send_sge * sizeof (struct mlx4_wqe_data_seg), - cap->max_inline_data + sizeof (struct mlx4_wqe_inline_seg)) + - send_wqe_overhead(type, qp->flags); - - if (s > dev->dev->caps.max_sq_desc_sz) - return -EINVAL; - - /* - * Hermon supports shrinking WQEs, such that a single work - * request can include multiple units of 1 << wqe_shift. This - * way, work requests can differ in size, and do not have to - * be a power of 2 in size, saving memory and speeding up send - * WR posting. Unfortunately, if we do this then the - * wqe_index field in CQEs can't be used to look up the WR ID - * anymore, so we do this only if selective signaling is off. - * - * Further, on 32-bit platforms, we can't use vmap() to make - * the QP buffer virtually contiguous. Thus we have to use - * constant-sized WRs to make sure a WR is always fully within - * a single page-sized chunk. - * - * Finally, we use NOP work requests to pad the end of the - * work queue, to avoid wrap-around in the middle of WR. We - * set NEC bit to avoid getting completions with error for - * these NOP WRs, but since NEC is only supported starting - * with firmware 2.2.232, we use constant-sized WRs for older - * firmware. - * - * And, since MLX QPs only support SEND, we use constant-sized - * WRs in this case. - * - * We look for the smallest value of wqe_shift such that the - * resulting number of wqes does not exceed device - * capabilities. - * - * We set WQE size to at least 64 bytes, this way stamping - * invalidates each WQE. - */ - if (dev->dev->caps.fw_ver >= MLX4_FW_VER_WQE_CTRL_NEC && - qp->sq_signal_bits && BITS_PER_LONG == 64 && - type != MLX4_IB_QPT_SMI && type != MLX4_IB_QPT_GSI && - !(type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_PROXY_SMI | - MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) - qp->sq.wqe_shift = ilog2(64); - else - qp->sq.wqe_shift = ilog2(roundup_pow_of_two(s)); - - for (;;) { - qp->sq_max_wqes_per_wr = DIV_ROUND_UP(s, 1U << qp->sq.wqe_shift); - - /* - * We need to leave 2 KB + 1 WR of headroom in the SQ to - * allow HW to prefetch. - */ - qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + qp->sq_max_wqes_per_wr; - qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr * - qp->sq_max_wqes_per_wr + - qp->sq_spare_wqes); - - if (qp->sq.wqe_cnt <= dev->dev->caps.max_wqes) - break; - - if (qp->sq_max_wqes_per_wr <= 1) - return -EINVAL; - - ++qp->sq.wqe_shift; - } - - qp->sq.max_gs = (min(dev->dev->caps.max_sq_desc_sz, - (qp->sq_max_wqes_per_wr << qp->sq.wqe_shift)) - - send_wqe_overhead(type, qp->flags)) / - sizeof (struct mlx4_wqe_data_seg); - - qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + - (qp->sq.wqe_cnt << qp->sq.wqe_shift); - if (qp->rq.wqe_shift > qp->sq.wqe_shift) { - qp->rq.offset = 0; - qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; - } else { - qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift; - qp->sq.offset = 0; - } - - cap->max_send_wr = qp->sq.max_post = - (qp->sq.wqe_cnt - qp->sq_spare_wqes) / qp->sq_max_wqes_per_wr; - cap->max_send_sge = min(qp->sq.max_gs, - min(dev->dev->caps.max_sq_sg, - dev->dev->caps.max_rq_sg)); - qp->max_inline_data = cap->max_inline_data; - - return 0; -} - -static int set_user_sq_size(struct mlx4_ib_dev *dev, - struct mlx4_ib_qp *qp, - struct mlx4_ib_create_qp *ucmd) -{ - /* Sanity check SQ size before proceeding */ - if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes || - ucmd->log_sq_stride > - ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) || - ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE) - return -EINVAL; - - qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count; - qp->sq.wqe_shift = ucmd->log_sq_stride; - - qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + - (qp->sq.wqe_cnt << qp->sq.wqe_shift); - - return 0; -} - -static int alloc_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp) -{ - int i; - - qp->sqp_proxy_rcv = - kmalloc(sizeof (struct mlx4_ib_buf) * qp->rq.wqe_cnt, - GFP_KERNEL); - if (!qp->sqp_proxy_rcv) - return -ENOMEM; - for (i = 0; i < qp->rq.wqe_cnt; i++) { - qp->sqp_proxy_rcv[i].addr = - kmalloc(sizeof (struct mlx4_ib_proxy_sqp_hdr), - GFP_KERNEL); - if (!qp->sqp_proxy_rcv[i].addr) - goto err; - qp->sqp_proxy_rcv[i].map = - ib_dma_map_single(dev, qp->sqp_proxy_rcv[i].addr, - sizeof (struct mlx4_ib_proxy_sqp_hdr), - DMA_FROM_DEVICE); - if (unlikely(ib_dma_mapping_error(dev, - qp->sqp_proxy_rcv[i].map))) { - pr_warn("ib_dma_map_single failed\n"); - kfree(qp->sqp_proxy_rcv[i].addr); - goto err; - } - } - return 0; - -err: - while (i > 0) { - --i; - ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map, - sizeof (struct mlx4_ib_proxy_sqp_hdr), - DMA_FROM_DEVICE); - kfree(qp->sqp_proxy_rcv[i].addr); - } - kfree(qp->sqp_proxy_rcv); - qp->sqp_proxy_rcv = NULL; - return -ENOMEM; -} - -static void free_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp) -{ - int i; - - for (i = 0; i < qp->rq.wqe_cnt; i++) { - ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map, - sizeof (struct mlx4_ib_proxy_sqp_hdr), - DMA_FROM_DEVICE); - kfree(qp->sqp_proxy_rcv[i].addr); - } - kfree(qp->sqp_proxy_rcv); -} - -static int init_qpg_parent(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *pqp, - struct ib_qp_init_attr *attr, int *qpn) -{ - struct mlx4_ib_qpg_data *qpg_data; - int tss_num, rss_num; - int tss_align_num, rss_align_num; - int tss_base, rss_base = 0; - int err; - - /* Parent is part of the TSS range (in SW TSS ARP is sent via parent) */ - tss_num = 1 + attr->parent_attrib.tss_child_count; - tss_align_num = roundup_pow_of_two(tss_num); - rss_num = attr->parent_attrib.rss_child_count; - rss_align_num = roundup_pow_of_two(rss_num); - - if (rss_num > 1) { - /* RSS is requested */ - if (!(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS)) - return -ENOSYS; - if (rss_align_num > dev->dev->caps.max_rss_tbl_sz) - return -EINVAL; - /* We must work with power of two */ - attr->parent_attrib.rss_child_count = rss_align_num; - } - - qpg_data = kzalloc(sizeof *qpg_data, GFP_KERNEL); - if (!qpg_data) - return -ENOMEM; - - if(pqp->flags & MLX4_IB_QP_NETIF) - err = mlx4_ib_steer_qp_alloc(dev, tss_align_num, &tss_base); - else - err = mlx4_qp_reserve_range(dev->dev, tss_align_num, - tss_align_num, &tss_base, MLX4_RESERVE_BF_QP); - if (err) - goto err1; - - if (tss_num > 1) { - u32 alloc = BITS_TO_LONGS(tss_align_num) * sizeof(long); - qpg_data->tss_bitmap = kzalloc(alloc, GFP_KERNEL); - if (qpg_data->tss_bitmap == NULL) { - err = -ENOMEM; - goto err2; - } - bitmap_fill(qpg_data->tss_bitmap, tss_num); - /* Note parent takes first index */ - clear_bit(0, qpg_data->tss_bitmap); - } - - if (rss_num > 1) { - u32 alloc = BITS_TO_LONGS(rss_align_num) * sizeof(long); - err = mlx4_qp_reserve_range(dev->dev, rss_align_num, - 1, &rss_base, 0); - if (err) - goto err3; - qpg_data->rss_bitmap = kzalloc(alloc, GFP_KERNEL); - if (qpg_data->rss_bitmap == NULL) { - err = -ENOMEM; - goto err4; - } - bitmap_fill(qpg_data->rss_bitmap, rss_align_num); - } - - qpg_data->tss_child_count = attr->parent_attrib.tss_child_count; - qpg_data->rss_child_count = attr->parent_attrib.rss_child_count; - qpg_data->qpg_parent = pqp; - qpg_data->qpg_tss_mask_sz = ilog2(tss_align_num); - qpg_data->tss_qpn_base = tss_base; - qpg_data->rss_qpn_base = rss_base; - - pqp->qpg_data = qpg_data; - *qpn = tss_base; - - return 0; - -err4: - mlx4_qp_release_range(dev->dev, rss_base, rss_align_num); - -err3: - if (tss_num > 1) - kfree(qpg_data->tss_bitmap); - -err2: - if(pqp->flags & MLX4_IB_QP_NETIF) - mlx4_ib_steer_qp_free(dev, tss_base, tss_align_num); - else - mlx4_qp_release_range(dev->dev, tss_base, tss_align_num); - -err1: - kfree(qpg_data); - return err; -} - -static void free_qpg_parent(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *pqp) -{ - struct mlx4_ib_qpg_data *qpg_data = pqp->qpg_data; - int align_num; - - if (qpg_data->tss_child_count > 1) - kfree(qpg_data->tss_bitmap); - - align_num = roundup_pow_of_two(1 + qpg_data->tss_child_count); - if(pqp->flags & MLX4_IB_QP_NETIF) - mlx4_ib_steer_qp_free(dev, qpg_data->tss_qpn_base, align_num); - else - mlx4_qp_release_range(dev->dev, qpg_data->tss_qpn_base, align_num); - - if (qpg_data->rss_child_count > 1) { - kfree(qpg_data->rss_bitmap); - align_num = roundup_pow_of_two(qpg_data->rss_child_count); - mlx4_qp_release_range(dev->dev, qpg_data->rss_qpn_base, - align_num); - } - - kfree(qpg_data); -} - -static int alloc_qpg_qpn(struct ib_qp_init_attr *init_attr, - struct mlx4_ib_qp *pqp, int *qpn) -{ - struct mlx4_ib_qp *mqp = to_mqp(init_attr->qpg_parent); - struct mlx4_ib_qpg_data *qpg_data = mqp->qpg_data; - u32 idx, old; - - switch (init_attr->qpg_type) { - case IB_QPG_CHILD_TX: - if (qpg_data->tss_child_count == 0) - return -EINVAL; - do { - /* Parent took index 0 */ - idx = find_first_bit(qpg_data->tss_bitmap, - qpg_data->tss_child_count + 1); - if (idx >= qpg_data->tss_child_count + 1) - return -ENOMEM; - old = test_and_clear_bit(idx, qpg_data->tss_bitmap); - } while (old == 0); - idx += qpg_data->tss_qpn_base; - break; - case IB_QPG_CHILD_RX: - if (qpg_data->rss_child_count == 0) - return -EINVAL; - do { - idx = find_first_bit(qpg_data->rss_bitmap, - qpg_data->rss_child_count); - if (idx >= qpg_data->rss_child_count) - return -ENOMEM; - old = test_and_clear_bit(idx, qpg_data->rss_bitmap); - } while (old == 0); - idx += qpg_data->rss_qpn_base; - break; - default: - return -EINVAL; - } - - pqp->qpg_data = qpg_data; - *qpn = idx; - - return 0; -} - -static void free_qpg_qpn(struct mlx4_ib_qp *mqp, int qpn) -{ - struct mlx4_ib_qpg_data *qpg_data = mqp->qpg_data; - - switch (mqp->qpg_type) { - case IB_QPG_CHILD_TX: - /* Do range check */ - qpn -= qpg_data->tss_qpn_base; - set_bit(qpn, qpg_data->tss_bitmap); - break; - case IB_QPG_CHILD_RX: - qpn -= qpg_data->rss_qpn_base; - set_bit(qpn, qpg_data->rss_bitmap); - break; - default: - /* error */ - pr_warn("wrong qpg type (%d)\n", mqp->qpg_type); - break; - } -} - -static int alloc_qpn_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, - struct ib_qp_init_attr *attr, int *qpn) -{ - int err = 0; - - switch (attr->qpg_type) { - case IB_QPG_NONE: - /* Raw packet QPNs may not have bits 6,7 set in their qp_num; - * otherwise, the WQE BlueFlame setup flow wrongly causes - * VLAN insertion. */ - if (attr->qp_type == IB_QPT_RAW_PACKET) { - err = mlx4_qp_reserve_range(dev->dev, 1, 1, qpn, - MLX4_RESERVE_BF_QP); - } else { - if(qp->flags & MLX4_IB_QP_NETIF) - err = mlx4_ib_steer_qp_alloc(dev, 1, qpn); - else - err = mlx4_qp_reserve_range(dev->dev, 1, 1, qpn, 0); - } - break; - case IB_QPG_PARENT: - err = init_qpg_parent(dev, qp, attr, qpn); - break; - case IB_QPG_CHILD_TX: - case IB_QPG_CHILD_RX: - err = alloc_qpg_qpn(attr, qp, qpn); - break; - default: - qp->qpg_type = IB_QPG_NONE; - err = -EINVAL; - break; - } - if (err) - return err; - qp->qpg_type = attr->qpg_type; - return 0; -} - -static void free_qpn_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, - enum ib_qpg_type qpg_type, int qpn) -{ - switch (qpg_type) { - case IB_QPG_NONE: - if (qp->flags & MLX4_IB_QP_NETIF) - mlx4_ib_steer_qp_free(dev, qpn, 1); - else - mlx4_qp_release_range(dev->dev, qpn, 1); - break; - case IB_QPG_PARENT: - free_qpg_parent(dev, qp); - break; - case IB_QPG_CHILD_TX: - case IB_QPG_CHILD_RX: - free_qpg_qpn(qp, qpn); - break; - default: - break; - } -} - -/* Revert allocation on create_qp_common */ -static void unalloc_qpn_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, - struct ib_qp_init_attr *attr, int qpn) -{ - free_qpn_common(dev, qp, attr->qpg_type, qpn); -} - -static void release_qpn_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) -{ - free_qpn_common(dev, qp, qp->qpg_type, qp->mqp.qpn); -} - -static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata, int sqpn, struct mlx4_ib_qp **caller_qp) -{ - int qpn; - int err; - struct mlx4_ib_sqp *sqp; - struct mlx4_ib_qp *qp; - enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr->qp_type; - - /* When tunneling special qps, we use a plain UD qp */ - if (sqpn) { - if (mlx4_is_mfunc(dev->dev) && - (!mlx4_is_master(dev->dev) || - !(init_attr->create_flags & MLX4_IB_SRIOV_SQP))) { - if (init_attr->qp_type == IB_QPT_GSI) - qp_type = MLX4_IB_QPT_PROXY_GSI; - else if (mlx4_is_master(dev->dev)) - qp_type = MLX4_IB_QPT_PROXY_SMI_OWNER; - else - qp_type = MLX4_IB_QPT_PROXY_SMI; - } - qpn = sqpn; - /* add extra sg entry for tunneling */ - init_attr->cap.max_recv_sge++; - } else if (init_attr->create_flags & MLX4_IB_SRIOV_TUNNEL_QP) { - struct mlx4_ib_qp_tunnel_init_attr *tnl_init = - container_of(init_attr, - struct mlx4_ib_qp_tunnel_init_attr, init_attr); - if ((tnl_init->proxy_qp_type != IB_QPT_SMI && - tnl_init->proxy_qp_type != IB_QPT_GSI) || - !mlx4_is_master(dev->dev)) - return -EINVAL; - if (tnl_init->proxy_qp_type == IB_QPT_GSI) - qp_type = MLX4_IB_QPT_TUN_GSI; - else if (tnl_init->slave == mlx4_master_func_num(dev->dev)) - qp_type = MLX4_IB_QPT_TUN_SMI_OWNER; - else - qp_type = MLX4_IB_QPT_TUN_SMI; - /* we are definitely in the PPF here, since we are creating - * tunnel QPs. base_tunnel_sqpn is therefore valid. */ - qpn = dev->dev->phys_caps.base_tunnel_sqpn + 8 * tnl_init->slave - + tnl_init->proxy_qp_type * 2 + tnl_init->port - 1; - sqpn = qpn; - } - - if (!*caller_qp) { - if (qp_type == MLX4_IB_QPT_SMI || qp_type == MLX4_IB_QPT_GSI || - (qp_type & (MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_SMI_OWNER | - MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) { - sqp = kzalloc(sizeof (struct mlx4_ib_sqp), GFP_KERNEL); - if (!sqp) - return -ENOMEM; - qp = &sqp->qp; - qp->pri.vid = qp->alt.vid = 0xFFFF; - } else { - qp = kzalloc(sizeof (struct mlx4_ib_qp), GFP_KERNEL); - if (!qp) - return -ENOMEM; - qp->pri.vid = qp->alt.vid = 0xFFFF; - } - } else - qp = *caller_qp; - - qp->mlx4_ib_qp_type = qp_type; - - if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) - qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK; - - if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) - qp->flags |= MLX4_IB_QP_LSO; - - if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) { - if (dev->dev->caps.steering_mode == - MLX4_STEERING_MODE_DEVICE_MANAGED && - !mlx4_is_mfunc(dev->dev)) - qp->flags |= MLX4_IB_QP_NETIF; - else { - err = -EINVAL; - goto err; - } - } - - mutex_init(&qp->mutex); - spin_lock_init(&qp->sq.lock); - spin_lock_init(&qp->rq.lock); - INIT_LIST_HEAD(&qp->gid_list); - INIT_LIST_HEAD(&qp->steering_rules); - INIT_LIST_HEAD(&qp->rules_list); - - qp->state = IB_QPS_RESET; - if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) - qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); - - err = set_rq_size(dev, &init_attr->cap, !!pd->uobject, mlx4_ib_qp_has_rq(init_attr), qp); - if (err) - goto err; - - if (pd->uobject) { - struct mlx4_ib_create_qp ucmd; - int shift; - int n; - - if (!udata || ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) { - err = -EFAULT; - goto err; - } - - if (init_attr->create_flags & IB_QP_CREATE_CROSS_CHANNEL) - qp->flags |= MLX4_IB_QP_CAP_CROSS_CHANNEL; - - if (init_attr->create_flags & IB_QP_CREATE_MANAGED_SEND) - qp->flags |= MLX4_IB_QP_CAP_MANAGED_SEND; - - if (init_attr->create_flags & IB_QP_CREATE_MANAGED_RECV) - qp->flags |= MLX4_IB_QP_CAP_MANAGED_RECV; - - qp->sq_no_prefetch = ucmd.sq_no_prefetch; - - err = set_user_sq_size(dev, qp, &ucmd); - if (err) - goto err; - - qp->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, - qp->buf_size, 0, 0); - if (IS_ERR(qp->umem)) { - err = PTR_ERR(qp->umem); - goto err; - } - - n = ib_umem_page_count(qp->umem); - shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n); - err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt); - - if (err) - goto err_buf; - - err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem); - if (err) - goto err_mtt; - - if (mlx4_ib_qp_has_rq(init_attr)) { - err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context), - ucmd.db_addr, &qp->db); - if (err) - goto err_mtt; - } - } else { - qp->sq_no_prefetch = 0; - - err = set_kernel_sq_size(dev, &init_attr->cap, qp_type, qp); - if (err) - goto err; - - if (mlx4_ib_qp_has_rq(init_attr)) { - err = mlx4_db_alloc(dev->dev, &qp->db, 0); - if (err) - goto err; - - *qp->db.db = 0; - } - - if (qp->max_inline_data) { - err = mlx4_bf_alloc(dev->dev, &qp->bf, 0); - if (err) { - pr_debug("failed to allocate blue flame" - " register (%d)", err); - qp->bf.uar = &dev->priv_uar; - } - } else - qp->bf.uar = &dev->priv_uar; - - if (mlx4_buf_alloc(dev->dev, qp->buf_size, PAGE_SIZE * 2, &qp->buf)) { - err = -ENOMEM; - goto err_db; - } - - err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift, - &qp->mtt); - if (err) - goto err_buf; - - err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf); - if (err) - goto err_mtt; - - qp->sq.wrid = kmalloc(qp->sq.wqe_cnt * sizeof (u64), GFP_KERNEL); - qp->rq.wrid = kmalloc(qp->rq.wqe_cnt * sizeof (u64), GFP_KERNEL); - - if (!qp->sq.wrid || !qp->rq.wrid) { - err = -ENOMEM; - goto err_wrid; - } - } - - if (sqpn) { - if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER | - MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) { - if (alloc_proxy_bufs(pd->device, qp)) { - err = -ENOMEM; - goto err_wrid; - } - } - } else { - err = alloc_qpn_common(dev, qp, init_attr, &qpn); - if (err) - goto err_proxy; - } - - err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp); - if (err) - goto err_qpn; - - if (init_attr->qp_type == IB_QPT_XRC_TGT) - qp->mqp.qpn |= (1 << 23); - - /* - * Hardware wants QPN written in big-endian order (after - * shifting) for send doorbell. Precompute this value to save - * a little bit when posting sends. - */ - qp->doorbell_qpn = swab32(qp->mqp.qpn << 8); - - qp->mqp.event = mlx4_ib_qp_event; - if (!*caller_qp) - *caller_qp = qp; - return 0; - -err_qpn: - unalloc_qpn_common(dev, qp, init_attr, qpn); - -err_proxy: - if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI) - free_proxy_bufs(pd->device, qp); -err_wrid: - if (pd->uobject) { - if (mlx4_ib_qp_has_rq(init_attr)) - mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &qp->db); - } else { - kfree(qp->sq.wrid); - kfree(qp->rq.wrid); - } - -err_mtt: - mlx4_mtt_cleanup(dev->dev, &qp->mtt); - -err_buf: - if (pd->uobject) - ib_umem_release(qp->umem); - else - mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf); - -err_db: - if (!pd->uobject && mlx4_ib_qp_has_rq(init_attr)) - mlx4_db_free(dev->dev, &qp->db); - - if (qp->max_inline_data) - mlx4_bf_free(dev->dev, &qp->bf); - -err: - if (!*caller_qp) - kfree(qp); - return err; -} - -static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state) -{ - switch (state) { - case IB_QPS_RESET: return MLX4_QP_STATE_RST; - case IB_QPS_INIT: return MLX4_QP_STATE_INIT; - case IB_QPS_RTR: return MLX4_QP_STATE_RTR; - case IB_QPS_RTS: return MLX4_QP_STATE_RTS; - case IB_QPS_SQD: return MLX4_QP_STATE_SQD; - case IB_QPS_SQE: return MLX4_QP_STATE_SQER; - case IB_QPS_ERR: return MLX4_QP_STATE_ERR; - default: return -1; - } -} - -static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq) - __acquires(&send_cq->lock) __acquires(&recv_cq->lock) -{ - if (send_cq == recv_cq) { - spin_lock_irq(&send_cq->lock); - __acquire(&recv_cq->lock); - } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) { - spin_lock_irq(&send_cq->lock); - spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); - } else { - spin_lock_irq(&recv_cq->lock); - spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); - } -} - -static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq) - __releases(&send_cq->lock) __releases(&recv_cq->lock) -{ - if (send_cq == recv_cq) { - __release(&recv_cq->lock); - spin_unlock_irq(&send_cq->lock); - } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) { - spin_unlock(&recv_cq->lock); - spin_unlock_irq(&send_cq->lock); - } else { - spin_unlock(&send_cq->lock); - spin_unlock_irq(&recv_cq->lock); - } -} - -static void del_gid_entries(struct mlx4_ib_qp *qp) -{ - struct mlx4_ib_gid_entry *ge, *tmp; - - list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) { - list_del(&ge->list); - kfree(ge); - } -} - -static struct mlx4_ib_pd *get_pd(struct mlx4_ib_qp *qp) -{ - if (qp->ibqp.qp_type == IB_QPT_XRC_TGT) - return to_mpd(to_mxrcd(qp->ibqp.xrcd)->pd); - else - return to_mpd(qp->ibqp.pd); -} - -static void get_cqs(struct mlx4_ib_qp *qp, - struct mlx4_ib_cq **send_cq, struct mlx4_ib_cq **recv_cq) -{ - switch (qp->ibqp.qp_type) { - case IB_QPT_XRC_TGT: - *send_cq = to_mcq(to_mxrcd(qp->ibqp.xrcd)->cq); - *recv_cq = *send_cq; - break; - case IB_QPT_XRC_INI: - *send_cq = to_mcq(qp->ibqp.send_cq); - *recv_cq = *send_cq; - break; - default: - *send_cq = to_mcq(qp->ibqp.send_cq); - *recv_cq = to_mcq(qp->ibqp.recv_cq); - break; - } -} - -static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, - int is_user) -{ - struct mlx4_ib_cq *send_cq, *recv_cq; - - if (qp->state != IB_QPS_RESET) { - if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state), - MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp)) - pr_warn("modify QP %06x to RESET failed.\n", - qp->mqp.qpn); - if (qp->pri.smac) { - mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac); - qp->pri.smac = 0; - } - if (qp->alt.smac) { - mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac); - qp->alt.smac = 0; - } - if (qp->pri.vid < 0x1000) { - mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid); - qp->pri.vid = 0xFFFF; - qp->pri.candidate_vid = 0xFFFF; - qp->pri.update_vid = 0; - } - if (qp->alt.vid < 0x1000) { - mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid); - qp->alt.vid = 0xFFFF; - qp->alt.candidate_vid = 0xFFFF; - qp->alt.update_vid = 0; - } - } - - get_cqs(qp, &send_cq, &recv_cq); - - mlx4_ib_lock_cqs(send_cq, recv_cq); - - if (!is_user) { - __mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn, - qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL); - if (send_cq != recv_cq) - __mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL); - } - - mlx4_qp_remove(dev->dev, &qp->mqp); - - mlx4_ib_unlock_cqs(send_cq, recv_cq); - - mlx4_qp_free(dev->dev, &qp->mqp); - - if (!is_sqp(dev, qp) && !is_tunnel_qp(dev, qp)) - release_qpn_common(dev, qp); - - mlx4_mtt_cleanup(dev->dev, &qp->mtt); - - if (is_user) { - if (qp->rq.wqe_cnt) - mlx4_ib_db_unmap_user(to_mucontext(qp->ibqp.uobject->context), - &qp->db); - ib_umem_release(qp->umem); - } else { - kfree(qp->sq.wrid); - kfree(qp->rq.wrid); - if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER | - MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) - free_proxy_bufs(&dev->ib_dev, qp); - mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf); - if (qp->max_inline_data) - mlx4_bf_free(dev->dev, &qp->bf); - - if (qp->rq.wqe_cnt) - mlx4_db_free(dev->dev, &qp->db); - } - - del_gid_entries(qp); -} - -static u32 get_sqp_num(struct mlx4_ib_dev *dev, struct ib_qp_init_attr *attr) -{ - /* Native or PPF */ - if (!mlx4_is_mfunc(dev->dev) || - (mlx4_is_master(dev->dev) && - attr->create_flags & MLX4_IB_SRIOV_SQP)) { - return dev->dev->phys_caps.base_sqpn + - (attr->qp_type == IB_QPT_SMI ? 0 : 2) + - attr->port_num - 1; - } - /* PF or VF -- creating proxies */ - if (attr->qp_type == IB_QPT_SMI) - return dev->dev->caps.qp0_proxy[attr->port_num - 1]; - else - return dev->dev->caps.qp1_proxy[attr->port_num - 1]; -} - -static int check_qpg_attr(struct mlx4_ib_dev *dev, - struct ib_qp_init_attr *attr) -{ - if (attr->qpg_type == IB_QPG_NONE) - return 0; - - if (attr->qp_type != IB_QPT_UD && - attr->qp_type != IB_QPT_RAW_PACKET) - return -EINVAL; - - if (attr->qpg_type == IB_QPG_PARENT) { - if (attr->parent_attrib.tss_child_count == 1) - return -EINVAL; /* Doesn't make sense */ - if (attr->parent_attrib.rss_child_count == 1) - return -EINVAL; /* Doesn't make sense */ - if ((attr->parent_attrib.tss_child_count == 0) && - (attr->parent_attrib.rss_child_count == 0)) - /* Should be called with IP_QPG_NONE */ - return -EINVAL; - if (attr->parent_attrib.rss_child_count > 1) { - int rss_align_num; - if (!(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS)) - return -ENOSYS; - rss_align_num = roundup_pow_of_two( - attr->parent_attrib.rss_child_count); - if (rss_align_num > dev->dev->caps.max_rss_tbl_sz) - return -EINVAL; - } - } else { - struct mlx4_ib_qpg_data *qpg_data; - if (attr->qpg_parent == NULL) - return -EINVAL; - if (IS_ERR(attr->qpg_parent)) - return -EINVAL; - qpg_data = to_mqp(attr->qpg_parent)->qpg_data; - if (qpg_data == NULL) - return -EINVAL; - if (attr->qpg_type == IB_QPG_CHILD_TX && - !qpg_data->tss_child_count) - return -EINVAL; - if (attr->qpg_type == IB_QPG_CHILD_RX && - !qpg_data->rss_child_count) - return -EINVAL; - } - return 0; -} - -#define RESERVED_FLAGS_MASK ((((unsigned int)IB_QP_CREATE_RESERVED_END - 1) | IB_QP_CREATE_RESERVED_END) \ - & ~(IB_QP_CREATE_RESERVED_START - 1)) - -static enum mlx4_ib_qp_flags to_mlx4_ib_qp_flags(enum ib_qp_create_flags ib_qp_flags) -{ - enum mlx4_ib_qp_flags mlx4_ib_qp_flags = 0; - - if (ib_qp_flags & IB_QP_CREATE_IPOIB_UD_LSO) - mlx4_ib_qp_flags |= MLX4_IB_QP_LSO; - - if (ib_qp_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) - mlx4_ib_qp_flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK; - - if (ib_qp_flags & IB_QP_CREATE_NETIF_QP) - mlx4_ib_qp_flags |= MLX4_IB_QP_NETIF; - - if (ib_qp_flags & IB_QP_CREATE_CROSS_CHANNEL) - mlx4_ib_qp_flags |= MLX4_IB_QP_CAP_CROSS_CHANNEL; - - if (ib_qp_flags & IB_QP_CREATE_MANAGED_SEND) - mlx4_ib_qp_flags |= MLX4_IB_QP_CAP_MANAGED_SEND; - - if (ib_qp_flags & IB_QP_CREATE_MANAGED_RECV) - mlx4_ib_qp_flags |= MLX4_IB_QP_CAP_MANAGED_RECV; - - /* reserved flags */ - mlx4_ib_qp_flags |= (ib_qp_flags & RESERVED_FLAGS_MASK); - - return mlx4_ib_qp_flags; -} - -struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata) -{ - struct mlx4_ib_qp *qp = NULL; - int err; - u16 xrcdn = 0; - enum mlx4_ib_qp_flags mlx4_qp_flags = to_mlx4_ib_qp_flags(init_attr->create_flags); - struct ib_device *device; - - /* see ib_core::ib_create_qp same handling */ - device = pd ? pd->device : init_attr->xrcd->device; - /* - * We only support LSO, vendor flag1, and multicast loopback blocking, - * and only for kernel UD QPs. - */ - if (mlx4_qp_flags & ~(MLX4_IB_QP_LSO | - MLX4_IB_QP_CAP_CROSS_CHANNEL | - MLX4_IB_QP_CAP_MANAGED_SEND | - MLX4_IB_QP_CAP_MANAGED_RECV | - MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK | - MLX4_IB_SRIOV_TUNNEL_QP | MLX4_IB_SRIOV_SQP | - MLX4_IB_QP_NETIF)) - return ERR_PTR(-EINVAL); - - if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) { - if (init_attr->qp_type != IB_QPT_UD) - return ERR_PTR(-EINVAL); - } - - if ((mlx4_qp_flags & - (MLX4_IB_QP_CAP_CROSS_CHANNEL | - MLX4_IB_QP_CAP_MANAGED_SEND | - MLX4_IB_QP_CAP_MANAGED_RECV)) && - !(to_mdev(device)->dev->caps.flags & - MLX4_DEV_CAP_FLAG_CROSS_CHANNEL)) { - pr_debug("%s Does not support cross-channel operations\n", - to_mdev(device)->ib_dev.name); - return ERR_PTR(-EINVAL); - } - - if ((init_attr->create_flags & - ~(IB_QP_CREATE_CROSS_CHANNEL | - IB_QP_CREATE_MANAGED_SEND | - IB_QP_CREATE_MANAGED_RECV)) && - (((mlx4_qp_flags & ~MLX4_IB_SRIOV_SQP) && - init_attr->qp_type != IB_QPT_UD) || - ((mlx4_qp_flags & MLX4_IB_SRIOV_SQP) && - init_attr->qp_type > IB_QPT_GSI))) - return ERR_PTR(-EINVAL); - - err = check_qpg_attr(to_mdev(device), init_attr); - if (err) - return ERR_PTR(err); - - switch (init_attr->qp_type) { - case IB_QPT_XRC_TGT: - pd = to_mxrcd(init_attr->xrcd)->pd; - xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn; - init_attr->send_cq = to_mxrcd(init_attr->xrcd)->cq; - /* fall through */ - case IB_QPT_XRC_INI: - if (!(to_mdev(device)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)) - return ERR_PTR(-ENOSYS); - init_attr->recv_cq = init_attr->send_cq; - /* fall through */ - case IB_QPT_RC: - case IB_QPT_UC: - case IB_QPT_RAW_PACKET: - qp = kzalloc(sizeof *qp, GFP_KERNEL); - if (!qp) - return ERR_PTR(-ENOMEM); - qp->pri.vid = qp->alt.vid = 0xFFFF; - /* fall through */ - case IB_QPT_UD: - { - err = create_qp_common(to_mdev(device), pd, init_attr, udata, 0, &qp); - if (err) { - kfree(qp); - return ERR_PTR(err); - } - - qp->ibqp.qp_num = qp->mqp.qpn; - qp->xrcdn = xrcdn; - - break; - } - case IB_QPT_SMI: - case IB_QPT_GSI: - { - /* Userspace is not allowed to create special QPs: */ - if (udata) - return ERR_PTR(-EINVAL); - - err = create_qp_common(to_mdev(device), pd, init_attr, udata, - get_sqp_num(to_mdev(device), init_attr), - &qp); - if (err) - return ERR_PTR(err); - - qp->port = init_attr->port_num; - qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1; - - break; - } - default: - /* Don't support raw QPs */ - return ERR_PTR(-EINVAL); - } - - return &qp->ibqp; -} - -int mlx4_ib_destroy_qp(struct ib_qp *qp) -{ - struct mlx4_ib_dev *dev = to_mdev(qp->device); - struct mlx4_ib_qp *mqp = to_mqp(qp); - struct mlx4_ib_pd *pd; - - if (is_qp0(dev, mqp)) - mlx4_CLOSE_PORT(dev->dev, mqp->port); - - pd = get_pd(mqp); - destroy_qp_common(dev, mqp, !!pd->ibpd.uobject); - - if (is_sqp(dev, mqp)) - kfree(to_msqp(mqp)); - else - kfree(mqp); - - return 0; -} - -static int to_mlx4_st(struct mlx4_ib_dev *dev, enum mlx4_ib_qp_type type) -{ - switch (type) { - case MLX4_IB_QPT_RC: return MLX4_QP_ST_RC; - case MLX4_IB_QPT_UC: return MLX4_QP_ST_UC; - case MLX4_IB_QPT_UD: return MLX4_QP_ST_UD; - case MLX4_IB_QPT_XRC_INI: - case MLX4_IB_QPT_XRC_TGT: return MLX4_QP_ST_XRC; - case MLX4_IB_QPT_SMI: - case MLX4_IB_QPT_GSI: - case MLX4_IB_QPT_RAW_PACKET: return MLX4_QP_ST_MLX; - - case MLX4_IB_QPT_PROXY_SMI_OWNER: - case MLX4_IB_QPT_TUN_SMI_OWNER: return (mlx4_is_mfunc(dev->dev) ? - MLX4_QP_ST_MLX : -1); - case MLX4_IB_QPT_PROXY_SMI: - case MLX4_IB_QPT_TUN_SMI: - case MLX4_IB_QPT_PROXY_GSI: - case MLX4_IB_QPT_TUN_GSI: return (mlx4_is_mfunc(dev->dev) ? - MLX4_QP_ST_UD : -1); - default: return -1; - } -} - -static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr, - int attr_mask) -{ - u8 dest_rd_atomic; - u32 access_flags; - u32 hw_access_flags = 0; - - if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) - dest_rd_atomic = attr->max_dest_rd_atomic; - else - dest_rd_atomic = qp->resp_depth; - - if (attr_mask & IB_QP_ACCESS_FLAGS) - access_flags = attr->qp_access_flags; - else - access_flags = qp->atomic_rd_en; - - if (!dest_rd_atomic) - access_flags &= IB_ACCESS_REMOTE_WRITE; - - if (access_flags & IB_ACCESS_REMOTE_READ) - hw_access_flags |= MLX4_QP_BIT_RRE; - if (access_flags & IB_ACCESS_REMOTE_ATOMIC) - hw_access_flags |= MLX4_QP_BIT_RAE; - if (access_flags & IB_ACCESS_REMOTE_WRITE) - hw_access_flags |= MLX4_QP_BIT_RWE; - - return cpu_to_be32(hw_access_flags); -} - -static void store_sqp_attrs(struct mlx4_ib_sqp *sqp, const struct ib_qp_attr *attr, - int attr_mask) -{ - if (attr_mask & IB_QP_PKEY_INDEX) - sqp->pkey_index = attr->pkey_index; - if (attr_mask & IB_QP_QKEY) - sqp->qkey = attr->qkey; - if (attr_mask & IB_QP_SQ_PSN) - sqp->send_psn = attr->sq_psn; -} - -static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port) -{ - path->sched_queue = (path->sched_queue & 0xbf) | ((port - 1) << 6); -} - -static int ib_rate_to_mlx4(struct mlx4_ib_dev *dev, u8 rate) -{ - if (rate == IB_RATE_PORT_CURRENT) { - return 0; - } else if (rate < IB_RATE_2_5_GBPS || rate > IB_RATE_300_GBPS) { - return -EINVAL; - } else { - while (rate != IB_RATE_2_5_GBPS && - !(1 << (rate + MLX4_STAT_RATE_OFFSET) & - dev->dev->caps.stat_rate_support)) - --rate; - } - - return rate + MLX4_STAT_RATE_OFFSET; -} - -static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah, - u8 *smac, u16 vlan_id, struct mlx4_ib_qp *qp, - struct mlx4_qp_path *path, u8 port, int is_primary) -{ - int is_eth = rdma_port_get_link_layer(&dev->ib_dev, port) == - IB_LINK_LAYER_ETHERNET; - u16 vlan_tag; - int vidx; - int smac_index; - int err; - u64 u64_mac; - struct mlx4_roce_smac_vlan_info *smac_info; - - path->grh_mylmc = ah->src_path_bits & 0x7f; - path->rlid = cpu_to_be16(ah->dlid); - - err = ib_rate_to_mlx4(dev, ah->static_rate); - if (err < 0) - return err; - path->static_rate = err; - - if (ah->ah_flags & IB_AH_GRH) { - if (ah->grh.sgid_index >= dev->dev->caps.gid_table_len[port]) { - pr_err("sgid_index (%u) too large. max is %d\n", - ah->grh.sgid_index, dev->dev->caps.gid_table_len[port] - 1); - return -1; - } - - path->grh_mylmc |= 1 << 7; - path->mgid_index = ah->grh.sgid_index; - path->hop_limit = ah->grh.hop_limit; - path->tclass_flowlabel = - cpu_to_be32((ah->grh.traffic_class << 20) | - (ah->grh.flow_label)); - memcpy(path->rgid, ah->grh.dgid.raw, 16); - } - - if (is_eth) { - if (!(ah->ah_flags & IB_AH_GRH)) - return -1; - - path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | - ((port - 1) << 6) | ((ah->sl & 7) << 3); - - if (is_primary) - smac_info = &qp->pri; - else - smac_info = &qp->alt; - - vlan_tag = vlan_id; - if (vlan_tag < 0x1000) { - if (smac_info->vid < 0x1000) { - /* both valid vlan ids */ - if (smac_info->vid != vlan_tag) { - /* different VIDs. unreg old and reg new */ - err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx); - if (err) - return err; - smac_info->candidate_vid = vlan_tag; - smac_info->candidate_vlan_index = vidx; - smac_info->candidate_vlan_port = port; - smac_info->update_vid = 1; - path->vlan_index = vidx; - path->fl = 1 << 6; - } else { - path->vlan_index = smac_info->vlan_index; - path->fl = 1 << 6; - } - } else { - /* no current vlan tag in qp */ - err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx); - if (err) - return err; - smac_info->candidate_vid = vlan_tag; - smac_info->candidate_vlan_index = vidx; - smac_info->candidate_vlan_port = port; - smac_info->update_vid = 1; - path->vlan_index = vidx; - path->fl = 1 << 6; - } - } else { - /* have current vlan tag. unregister it at modify-qp success */ - if (smac_info->vid < 0x1000) { - smac_info->candidate_vid = 0xFFFF; - smac_info->update_vid = 1; - } - } - - - /* get smac_index for RoCE use. - * If no smac was yet assigned, register one. - * If one was already assigned, but the new mac differs, - * unregister the old one and register the new one. - */ - u64_mac = mlx4_mac_to_u64(smac); - - if (!smac_info->smac || smac_info->smac != u64_mac) { - /* register candidate now, unreg if needed, after success */ - smac_index = mlx4_register_mac(dev->dev, port, u64_mac); - if (smac_index >= 0) { - smac_info->candidate_smac_index = smac_index; - smac_info->candidate_smac = u64_mac; - smac_info->candidate_smac_port = port; - } else - return -EINVAL; - } else - smac_index = smac_info->smac_index; - - memcpy(path->dmac, ah->dmac, 6); - path->ackto = MLX4_IB_LINK_TYPE_ETH; - /* put MAC table smac index for IBoE */ - path->grh_mylmc = (u8) (smac_index) | 0x80 ; - - } else - path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | - ((port - 1) << 6) | ((ah->sl & 0xf) << 2); - - return 0; -} - -static void update_mcg_macs(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) -{ - struct mlx4_ib_gid_entry *ge, *tmp; - - list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) { - if (!ge->added && mlx4_ib_add_mc(dev, qp, &ge->gid)) { - ge->added = 1; - ge->port = qp->port; - } - } -} - -static int handle_eth_ud_smac_index(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, const u8 *smac, - struct mlx4_qp_context *context) -{ - struct net_device *ndev; - u64 u64_mac; - int smac_index; - - - ndev = dev->iboe.netdevs[qp->port - 1]; - if (ndev) { - smac = IF_LLADDR(ndev); - u64_mac = mlx4_mac_to_u64(smac); - } else { - u64_mac = dev->dev->caps.def_mac[qp->port]; - } - - context->pri_path.sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | ((qp->port - 1) << 6); - if (!qp->pri.smac) { - smac_index = mlx4_register_mac(dev->dev, qp->port, u64_mac); - if (smac_index >= 0) { - qp->pri.candidate_smac_index = smac_index; - qp->pri.candidate_smac = u64_mac; - qp->pri.candidate_smac_port = qp->port; - context->pri_path.grh_mylmc = 0x80 | (u8) smac_index; - } else - return -ENOENT; - } - return 0; -} -static int __mlx4_ib_modify_qp(struct ib_qp *ibqp, - const struct ib_qp_attr *attr, int attr_mask, - enum ib_qp_state cur_state, enum ib_qp_state new_state) -{ - struct mlx4_ib_dev *dev = to_mdev(ibqp->device); - struct mlx4_ib_qp *qp = to_mqp(ibqp); - struct mlx4_ib_pd *pd; - struct mlx4_ib_cq *send_cq, *recv_cq; - struct mlx4_qp_context *context; - enum mlx4_qp_optpar optpar = 0; - int sqd_event; - int steer_qp = 0; - int err = -EINVAL; - int is_eth = -1; - - context = kzalloc(sizeof *context, GFP_KERNEL); - if (!context) - return -ENOMEM; - - context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) | - (to_mlx4_st(dev, qp->mlx4_ib_qp_type) << 16)); - - if (!(attr_mask & IB_QP_PATH_MIG_STATE)) - context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11); - else { - optpar |= MLX4_QP_OPTPAR_PM_STATE; - switch (attr->path_mig_state) { - case IB_MIG_MIGRATED: - context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11); - break; - case IB_MIG_REARM: - context->flags |= cpu_to_be32(MLX4_QP_PM_REARM << 11); - break; - case IB_MIG_ARMED: - context->flags |= cpu_to_be32(MLX4_QP_PM_ARMED << 11); - break; - } - } - - if (qp->max_inlr_data) - context->param3 |= cpu_to_be32(1 << 25); - - if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI) - context->mtu_msgmax = (IB_MTU_4096 << 5) | 11; - else if (ibqp->qp_type == IB_QPT_RAW_PACKET) - context->mtu_msgmax = (MLX4_RAW_QP_MTU << 5) | MLX4_RAW_QP_MSGMAX; - else if (ibqp->qp_type == IB_QPT_UD) { - if (qp->flags & MLX4_IB_QP_LSO) - context->mtu_msgmax = (IB_MTU_4096 << 5) | - ilog2(dev->dev->caps.max_gso_sz); - else - context->mtu_msgmax = (IB_MTU_4096 << 5) | 12; - } else if (attr_mask & IB_QP_PATH_MTU) { - if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) { - pr_err("path MTU (%u) is invalid\n", - attr->path_mtu); - goto out; - } - context->mtu_msgmax = (attr->path_mtu << 5) | - ilog2(dev->dev->caps.max_msg_sz); - } - - if (qp->rq.wqe_cnt) - context->rq_size_stride = ilog2(qp->rq.wqe_cnt) << 3; - context->rq_size_stride |= qp->rq.wqe_shift - 4; - - if (qp->sq.wqe_cnt) - context->sq_size_stride = ilog2(qp->sq.wqe_cnt) << 3; - context->sq_size_stride |= qp->sq.wqe_shift - 4; - - if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { - context->sq_size_stride |= !!qp->sq_no_prefetch << 7; - context->xrcd = cpu_to_be32((u32) qp->xrcdn); - context->param3 |= cpu_to_be32(1 << 30); - } - - if (qp->ibqp.uobject) - context->usr_page = cpu_to_be32(to_mucontext(ibqp->uobject->context)->uar.index); - else - context->usr_page = cpu_to_be32(qp->bf.uar->index); - - if (attr_mask & IB_QP_DEST_QPN) - context->remote_qpn = cpu_to_be32(attr->dest_qp_num); - - if (attr_mask & IB_QP_PORT) { - if (cur_state == IB_QPS_SQD && new_state == IB_QPS_SQD && - !(attr_mask & IB_QP_AV)) { - mlx4_set_sched(&context->pri_path, attr->port_num); - optpar |= MLX4_QP_OPTPAR_SCHED_QUEUE; - } - } - - if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { - if (dev->counters[qp->port - 1].counter_index != -1) { - context->pri_path.counter_index = - dev->counters[qp->port - 1].counter_index; - optpar |= MLX4_QP_OPTPAR_COUNTER_INDEX; - } else { - context->pri_path.counter_index = 0xff; - } - - if (qp->flags & MLX4_IB_QP_NETIF && - (qp->qpg_type == IB_QPG_NONE || qp->qpg_type == IB_QPG_PARENT)) { - mlx4_ib_steer_qp_reg(dev, qp, 1); - steer_qp = 1; - } - } - - if (attr_mask & IB_QP_PKEY_INDEX) { - if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) - context->pri_path.disable_pkey_check = 0x40; - context->pri_path.pkey_index = attr->pkey_index; - optpar |= MLX4_QP_OPTPAR_PKEY_INDEX; - } - - if ((attr_mask & IB_QP_AV) && (ibqp->qp_type != IB_QPT_RAW_PACKET)) { - if (mlx4_set_path(dev, &attr->ah_attr, (u8 *)attr->smac, - attr_mask & IB_QP_VID ? - attr->vlan_id : 0xffff , - qp, &context->pri_path, - attr_mask & IB_QP_PORT ? - attr->port_num : qp->port, 1)) - goto out; - - optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH | - MLX4_QP_OPTPAR_SCHED_QUEUE); - } - - if (attr_mask & IB_QP_TIMEOUT) { - context->pri_path.ackto |= attr->timeout << 3; - optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT; - } - - if (attr_mask & IB_QP_ALT_PATH) { - if (attr->alt_port_num == 0 || - attr->alt_port_num > dev->dev->caps.num_ports) - goto out; - - if (attr->alt_pkey_index >= - dev->dev->caps.pkey_table_len[attr->alt_port_num]) - goto out; - - if (mlx4_set_path(dev, &attr->alt_ah_attr, (u8 *)attr->smac, - attr_mask & IB_QP_ALT_VID ? - attr->alt_vlan_id : 0xffff, - qp, &context->alt_path, - attr->alt_port_num, 0)) - goto out; - - context->alt_path.pkey_index = attr->alt_pkey_index; - context->alt_path.ackto = attr->alt_timeout << 3; - context->alt_path.counter_index = dev->counters[attr->alt_port_num - 1].counter_index; - optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH; - } - - pd = get_pd(qp); - get_cqs(qp, &send_cq, &recv_cq); - context->pd = cpu_to_be32(pd->pdn); - context->cqn_send = cpu_to_be32(send_cq->mcq.cqn); - context->cqn_recv = cpu_to_be32(recv_cq->mcq.cqn); - context->params1 = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28); - - /* Set "fast registration enabled" for all kernel QPs */ - if (!qp->ibqp.uobject) - context->params1 |= cpu_to_be32(1 << 11); - - if (attr_mask & IB_QP_RNR_RETRY) { - context->params1 |= cpu_to_be32(attr->rnr_retry << 13); - optpar |= MLX4_QP_OPTPAR_RNR_RETRY; - } - - if (attr_mask & IB_QP_RETRY_CNT) { - context->params1 |= cpu_to_be32(attr->retry_cnt << 16); - optpar |= MLX4_QP_OPTPAR_RETRY_COUNT; - } - - if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { - if (attr->max_rd_atomic) - context->params1 |= - cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21); - optpar |= MLX4_QP_OPTPAR_SRA_MAX; - } - - if (attr_mask & IB_QP_SQ_PSN) - context->next_send_psn = cpu_to_be32(attr->sq_psn); - - if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { - if (attr->max_dest_rd_atomic) - context->params2 |= - cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21); - optpar |= MLX4_QP_OPTPAR_RRA_MAX; - } - - if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) { - context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask); - optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE; - } - - if (attr_mask & IB_M_EXT_CLASS_1) - context->params2 |= cpu_to_be32(MLX4_QP_BIT_COLL_MASTER); - - /* for now we enable also sqe on send */ - if (attr_mask & IB_M_EXT_CLASS_2) { - context->params2 |= cpu_to_be32(MLX4_QP_BIT_COLL_SYNC_SQ); - context->params2 |= cpu_to_be32(MLX4_QP_BIT_COLL_MASTER); - } - - if (attr_mask & IB_M_EXT_CLASS_3) - context->params2 |= cpu_to_be32(MLX4_QP_BIT_COLL_SYNC_RQ); - - if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { - context->params2 |= (qp->flags & MLX4_IB_QP_CAP_CROSS_CHANNEL ? - cpu_to_be32(MLX4_QP_BIT_COLL_MASTER) : 0); - context->params2 |= (qp->flags & MLX4_IB_QP_CAP_MANAGED_SEND ? - cpu_to_be32(MLX4_QP_BIT_COLL_MASTER | MLX4_QP_BIT_COLL_SYNC_SQ) : 0); - context->params2 |= (qp->flags & MLX4_IB_QP_CAP_MANAGED_RECV ? - cpu_to_be32(MLX4_QP_BIT_COLL_MASTER | MLX4_QP_BIT_COLL_SYNC_RQ) : 0); - } - - if (ibqp->srq) - context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC); - - if (attr_mask & IB_QP_MIN_RNR_TIMER) { - context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24); - optpar |= MLX4_QP_OPTPAR_RNR_TIMEOUT; - } - if (attr_mask & IB_QP_RQ_PSN) - context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn); - - /* proxy and tunnel qp qkeys will be changed in modify-qp wrappers */ - if (attr_mask & IB_QP_QKEY) { - if (qp->mlx4_ib_qp_type & - (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) - context->qkey = cpu_to_be32(IB_QP_SET_QKEY); - else { - if (mlx4_is_mfunc(dev->dev) && - !(qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) && - (attr->qkey & MLX4_RESERVED_QKEY_MASK) == - MLX4_RESERVED_QKEY_BASE) { - pr_err("Cannot use reserved QKEY" - " 0x%x (range 0xffff0000..0xffffffff" - " is reserved)\n", attr->qkey); - err = -EINVAL; - goto out; - } - context->qkey = cpu_to_be32(attr->qkey); - } - optpar |= MLX4_QP_OPTPAR_Q_KEY; - } - - if (ibqp->srq) - context->srqn = cpu_to_be32(1 << 24 | to_msrq(ibqp->srq)->msrq.srqn); - - if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) - context->db_rec_addr = cpu_to_be64(qp->db.dma); - - if (cur_state == IB_QPS_INIT && - new_state == IB_QPS_RTR && - (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI || - ibqp->qp_type == IB_QPT_UD || - ibqp->qp_type == IB_QPT_RAW_PACKET)) { - context->pri_path.sched_queue = (qp->port - 1) << 6; - if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI || - qp->mlx4_ib_qp_type & - (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) { - context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE; - if (qp->mlx4_ib_qp_type != MLX4_IB_QPT_SMI) - context->pri_path.fl = 0x80; - } else { - if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) - context->pri_path.fl = 0x80; - context->pri_path.sched_queue |= MLX4_IB_DEFAULT_SCHED_QUEUE; - } - if (ibqp->qp_type == IB_QPT_RAW_PACKET && - (attr_mask & IB_QP_AV)) { - context->pri_path.sched_queue |= - ((attr->ah_attr.sl & 0xf) << 3); - context->pri_path.feup = 1 << 6; - } - is_eth = rdma_port_get_link_layer(&dev->ib_dev, qp->port) == - IB_LINK_LAYER_ETHERNET; - if (is_eth) { - if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI || - qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) - context->pri_path.feup = 1 << 7; /* don't fsm */ - /* handle smac_index */ - if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_UD || - qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI || - qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI) { - err = handle_eth_ud_smac_index(dev, qp, (const u8 *)attr->smac, context); - if (err) - return -EINVAL; - } - } - } - - if (ibqp->qp_type == IB_QPT_UD) - if (is_eth && (new_state == IB_QPS_RTR)) { - context->pri_path.ackto = MLX4_IB_LINK_TYPE_ETH; - optpar |= MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH; - } - - if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD && - attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify) - sqd_event = 1; - else - sqd_event = 0; - - if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) - context->rlkey |= (1 << 4); - - if ((attr_mask & IB_QP_GROUP_RSS) && - (qp->qpg_data->rss_child_count > 1)) { - struct mlx4_ib_qpg_data *qpg_data = qp->qpg_data; - void *rss_context_base = &context->pri_path; - struct mlx4_rss_context *rss_context = - (struct mlx4_rss_context *) (rss_context_base - + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH); - - context->flags |= cpu_to_be32(1 << MLX4_RSS_QPC_FLAG_OFFSET); - - /* This should be tbl_sz_base_qpn */ - rss_context->base_qpn = cpu_to_be32(qpg_data->rss_qpn_base | - (ilog2(qpg_data->rss_child_count) << 24)); - rss_context->default_qpn = cpu_to_be32(qpg_data->rss_qpn_base); - /* This should be flags_hash_fn */ - rss_context->flags = MLX4_RSS_TCP_IPV6 | - MLX4_RSS_TCP_IPV4; - if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UDP_RSS) { - rss_context->base_qpn_udp = rss_context->default_qpn; - rss_context->flags |= MLX4_RSS_IPV6 | - MLX4_RSS_IPV4 | - MLX4_RSS_UDP_IPV6 | - MLX4_RSS_UDP_IPV4; - } - if (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_TOP) { - static const u32 rsskey[10] = { 0xD181C62C, 0xF7F4DB5B, - 0x1983A2FC, 0x943E1ADB, 0xD9389E6B, 0xD1039C2C, - 0xA74499AD, 0x593D56D9, 0xF3253C06, 0x2ADC1FFC}; - rss_context->hash_fn = MLX4_RSS_HASH_TOP; - memcpy(rss_context->rss_key, rsskey, - sizeof(rss_context->rss_key)); - } else { - rss_context->hash_fn = MLX4_RSS_HASH_XOR; - memset(rss_context->rss_key, 0, - sizeof(rss_context->rss_key)); - } - } - /* - * Before passing a kernel QP to the HW, make sure that the - * ownership bits of the send queue are set and the SQ - * headroom is stamped so that the hardware doesn't start - * processing stale work requests. - */ - if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { - struct mlx4_wqe_ctrl_seg *ctrl; - int i; - - for (i = 0; i < qp->sq.wqe_cnt; ++i) { - ctrl = get_send_wqe(qp, i); - ctrl->owner_opcode = cpu_to_be32(1 << 31); - if (qp->sq_max_wqes_per_wr == 1) - ctrl->fence_size = 1 << (qp->sq.wqe_shift - 4); - - stamp_send_wqe(qp, i, 1 << qp->sq.wqe_shift); - } - } - - if ((qp->port && rdma_port_get_link_layer(&dev->ib_dev, qp->port) == - IB_LINK_LAYER_ETHERNET) && (qp->ibqp.qp_type == IB_QPT_RAW_PACKET)) - context->pri_path.ackto = (context->pri_path.ackto & 0xf8) | - MLX4_IB_LINK_TYPE_ETH; - - err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state), - to_mlx4_state(new_state), context, optpar, - sqd_event, &qp->mqp); - if (err) - goto out; - - qp->state = new_state; - - if (attr_mask & IB_QP_ACCESS_FLAGS) - qp->atomic_rd_en = attr->qp_access_flags; - if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) - qp->resp_depth = attr->max_dest_rd_atomic; - if (attr_mask & IB_QP_PORT) { - qp->port = attr->port_num; - update_mcg_macs(dev, qp); - } - if (attr_mask & IB_QP_ALT_PATH) - qp->alt_port = attr->alt_port_num; - - if (is_sqp(dev, qp)) - store_sqp_attrs(to_msqp(qp), attr, attr_mask); - - /* Set 'ignore_cq_overrun' bits for collectives offload */ - if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { - if (attr_mask & (IB_M_EXT_CLASS_2 | IB_M_EXT_CLASS_3)) { - err = mlx4_ib_ignore_overrun_cq(ibqp->send_cq); - if (err) { - pr_err("Failed to set ignore CQ " - "overrun for QP 0x%x's send CQ\n", - ibqp->qp_num); - goto out; - } - - if (ibqp->recv_cq != ibqp->send_cq) { - err = mlx4_ib_ignore_overrun_cq(ibqp->recv_cq); - if (err) { - pr_err("Failed to set ignore " - "CQ overrun for QP 0x%x's recv " - "CQ\n", ibqp->qp_num); - goto out; - } - } - } - } - - /* - * If we moved QP0 to RTR, bring the IB link up; if we moved - * QP0 to RESET or ERROR, bring the link back down. - */ - if (is_qp0(dev, qp)) { - if (cur_state != IB_QPS_RTR && new_state == IB_QPS_RTR) - if (mlx4_INIT_PORT(dev->dev, qp->port)) - pr_warn("INIT_PORT failed for port %d\n", - qp->port); - - if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR && - (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR)) - mlx4_CLOSE_PORT(dev->dev, qp->port); - } - - /* - * If we moved a kernel QP to RESET, clean up all old CQ - * entries and reinitialize the QP. - */ - if (new_state == IB_QPS_RESET) { - if (!ibqp->uobject) { - mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn, - ibqp->srq ? to_msrq(ibqp->srq) : NULL); - if (send_cq != recv_cq) - mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL); - - qp->rq.head = 0; - qp->rq.tail = 0; - qp->sq.head = 0; - qp->sq.tail = 0; - qp->sq_next_wqe = 0; - if (qp->rq.wqe_cnt) - *qp->db.db = 0; - - if (qp->flags & MLX4_IB_QP_NETIF && - (qp->qpg_type == IB_QPG_NONE || - qp->qpg_type == IB_QPG_PARENT)) - mlx4_ib_steer_qp_reg(dev, qp, 0); - } - if (qp->pri.smac) { - mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac); - qp->pri.smac = 0; - } - if (qp->alt.smac) { - mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac); - qp->alt.smac = 0; - } - if (qp->pri.vid < 0x1000) { - mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid); - qp->pri.vid = 0xFFFF; - qp->pri.candidate_vid = 0xFFFF; - qp->pri.update_vid = 0; - } - - if (qp->alt.vid < 0x1000) { - mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid); - qp->alt.vid = 0xFFFF; - qp->alt.candidate_vid = 0xFFFF; - qp->alt.update_vid = 0; - } - } - -out: - if (err && steer_qp) - mlx4_ib_steer_qp_reg(dev, qp, 0); - kfree(context); - if (qp->pri.candidate_smac) { - if (err) - mlx4_unregister_mac(dev->dev, qp->pri.candidate_smac_port, qp->pri.candidate_smac); - else { - if (qp->pri.smac) { - mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac); - } - qp->pri.smac = qp->pri.candidate_smac; - qp->pri.smac_index = qp->pri.candidate_smac_index; - qp->pri.smac_port = qp->pri.candidate_smac_port; - - } - qp->pri.candidate_smac = 0; - qp->pri.candidate_smac_index = 0; - qp->pri.candidate_smac_port = 0; - } - if (qp->alt.candidate_smac) { - if (err) - mlx4_unregister_mac(dev->dev, qp->alt.candidate_smac_port, qp->pri.candidate_smac); - else { - if (qp->pri.smac) { - mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac); - } - qp->alt.smac = qp->alt.candidate_smac; - qp->alt.smac_index = qp->alt.candidate_smac_index; - qp->alt.smac_port = qp->alt.candidate_smac_port; - - } - qp->pri.candidate_smac = 0; - qp->pri.candidate_smac_index = 0; - qp->pri.candidate_smac_port = 0; - } - - if (qp->pri.update_vid) { - if (err) { - if (qp->pri.candidate_vid < 0x1000) - mlx4_unregister_vlan(dev->dev, qp->pri.candidate_vlan_port, - qp->pri.candidate_vid); - } else { - if (qp->pri.vid < 0x1000) - mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, - qp->pri.vid); - qp->pri.vid = qp->pri.candidate_vid; - qp->pri.vlan_port = qp->pri.candidate_vlan_port; - qp->pri.vlan_index = qp->pri.candidate_vlan_index; - } - qp->pri.candidate_vid = 0xFFFF; - qp->pri.update_vid = 0; - } - - if (qp->alt.update_vid) { - if (err) { - if (qp->alt.candidate_vid < 0x1000) - mlx4_unregister_vlan(dev->dev, qp->alt.candidate_vlan_port, - qp->alt.candidate_vid); - } else { - if (qp->alt.vid < 0x1000) - mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, - qp->alt.vid); - qp->alt.vid = qp->alt.candidate_vid; - qp->alt.vlan_port = qp->alt.candidate_vlan_port; - qp->alt.vlan_index = qp->alt.candidate_vlan_index; - } - qp->alt.candidate_vid = 0xFFFF; - qp->alt.update_vid = 0; - } - - return err; -} - -int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, - int attr_mask, struct ib_udata *udata) -{ - struct mlx4_ib_dev *dev = to_mdev(ibqp->device); - struct mlx4_ib_qp *qp = to_mqp(ibqp); - enum ib_qp_state cur_state, new_state; - int err = -EINVAL; - int ll; - - mutex_lock(&qp->mutex); - - cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state; - new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; - - if (cur_state == new_state && cur_state == IB_QPS_RESET) { - ll = IB_LINK_LAYER_UNSPECIFIED; - } else { - int port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port; - ll = rdma_port_get_link_layer(&dev->ib_dev, port); - } - - if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, - attr_mask & ~IB_M_QP_MOD_VEND_MASK, ll)) { - pr_debug("qpn 0x%x: invalid attribute mask specified " - "for transition %d to %d. qp_type %d," - " attr_mask 0x%x\n", - ibqp->qp_num, cur_state, new_state, - ibqp->qp_type, attr_mask); - goto out; - } - - if ((attr_mask & IB_M_QP_MOD_VEND_MASK) && !dev->dev->caps.sync_qp) { - pr_err("extended verbs are not supported by %s\n", - dev->ib_dev.name); - goto out; - } - - if ((attr_mask & IB_QP_PORT) && - (attr->port_num == 0 || attr->port_num > dev->num_ports)) { - pr_debug("qpn 0x%x: invalid port number (%d) specified " - "for transition %d to %d. qp_type %d\n", - ibqp->qp_num, attr->port_num, cur_state, - new_state, ibqp->qp_type); - goto out; - } - - if (attr_mask & IB_QP_PKEY_INDEX) { - int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port; - if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p]) { - pr_debug("qpn 0x%x: invalid pkey index (%d) specified " - "for transition %d to %d. qp_type %d\n", - ibqp->qp_num, attr->pkey_index, cur_state, - new_state, ibqp->qp_type); - goto out; - } - } - - if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && - attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) { - pr_debug("qpn 0x%x: max_rd_atomic (%d) too large. " - "Transition %d to %d. qp_type %d\n", - ibqp->qp_num, attr->max_rd_atomic, cur_state, - new_state, ibqp->qp_type); - goto out; - } - - if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && - attr->max_dest_rd_atomic > dev->dev->caps.max_qp_dest_rdma) { - pr_debug("qpn 0x%x: max_dest_rd_atomic (%d) too large. " - "Transition %d to %d. qp_type %d\n", - ibqp->qp_num, attr->max_dest_rd_atomic, cur_state, - new_state, ibqp->qp_type); - goto out; - } - - if (cur_state == new_state && cur_state == IB_QPS_RESET) { - err = 0; - goto out; - } - - err = __mlx4_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state); - -out: - mutex_unlock(&qp->mutex); - return err; -} - -static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp, - struct ib_send_wr *wr, - void *wqe, unsigned *mlx_seg_len) -{ - struct mlx4_ib_dev *mdev = to_mdev(sqp->qp.ibqp.device); - struct ib_device *ib_dev = &mdev->ib_dev; - struct mlx4_wqe_mlx_seg *mlx = wqe; - struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx; - struct mlx4_ib_ah *ah = to_mah(wr->wr.ud.ah); - u16 pkey; - u32 qkey; - int send_size; - int header_size; - int spc; - int i; - - if (wr->opcode != IB_WR_SEND) - return -EINVAL; - - send_size = 0; - - for (i = 0; i < wr->num_sge; ++i) - send_size += wr->sg_list[i].length; - - /* for proxy-qp0 sends, need to add in size of tunnel header */ - /* for tunnel-qp0 sends, tunnel header is already in s/g list */ - if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) - send_size += sizeof (struct mlx4_ib_tunnel_header); - - ib_ud_header_init(send_size, 1, 0, 0, 0, 0, &sqp->ud_header); - - if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) { - sqp->ud_header.lrh.service_level = - be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28; - sqp->ud_header.lrh.destination_lid = - cpu_to_be16(ah->av.ib.g_slid & 0x7f); - sqp->ud_header.lrh.source_lid = - cpu_to_be16(ah->av.ib.g_slid & 0x7f); - } - - mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); - - /* force loopback */ - mlx->flags |= cpu_to_be32(MLX4_WQE_MLX_VL15 | 0x1 | MLX4_WQE_MLX_SLR); - mlx->rlid = sqp->ud_header.lrh.destination_lid; - - sqp->ud_header.lrh.virtual_lane = 0; - sqp->ud_header.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED); - ib_get_cached_pkey(ib_dev, sqp->qp.port, 0, &pkey); - sqp->ud_header.bth.pkey = cpu_to_be16(pkey); - if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER) - sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn); - else - sqp->ud_header.bth.destination_qpn = - cpu_to_be32(mdev->dev->caps.qp0_tunnel[sqp->qp.port - 1]); - - sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1)); - if (mlx4_get_parav_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey)) - return -EINVAL; - sqp->ud_header.deth.qkey = cpu_to_be32(qkey); - sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.mqp.qpn); - - sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY; - sqp->ud_header.immediate_present = 0; - - header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf); - - /* - * Inline data segments may not cross a 64 byte boundary. If - * our UD header is bigger than the space available up to the - * next 64 byte boundary in the WQE, use two inline data - * segments to hold the UD header. - */ - spc = MLX4_INLINE_ALIGN - - ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1)); - if (header_size <= spc) { - inl->byte_count = cpu_to_be32(1 << 31 | header_size); - memcpy(inl + 1, sqp->header_buf, header_size); - i = 1; - } else { - inl->byte_count = cpu_to_be32(1 << 31 | spc); - memcpy(inl + 1, sqp->header_buf, spc); - - inl = (void *) (inl + 1) + spc; - memcpy(inl + 1, sqp->header_buf + spc, header_size - spc); - /* - * Need a barrier here to make sure all the data is - * visible before the byte_count field is set. - * Otherwise the HCA prefetcher could grab the 64-byte - * chunk with this inline segment and get a valid (!= - * 0xffffffff) byte count but stale data, and end up - * generating a packet with bad headers. - * - * The first inline segment's byte_count field doesn't - * need a barrier, because it comes after a - * control/MLX segment and therefore is at an offset - * of 16 mod 64. - */ - wmb(); - inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc)); - i = 2; - } - - *mlx_seg_len = - ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16); - return 0; -} - -static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr, - void *wqe, unsigned *mlx_seg_len) -{ - struct ib_device *ib_dev = sqp->qp.ibqp.device; - struct mlx4_wqe_mlx_seg *mlx = wqe; - struct mlx4_wqe_ctrl_seg *ctrl = wqe; - struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx; - struct mlx4_ib_ah *ah = to_mah(wr->wr.ud.ah); - union ib_gid sgid; - u16 pkey; - int send_size; - int header_size; - int spc; - int i; - int is_eth; - int is_vlan = 0; - int is_grh; - u16 uninitialized_var(vlan); - int err = 0; - - send_size = 0; - for (i = 0; i < wr->num_sge; ++i) - send_size += wr->sg_list[i].length; - - is_eth = rdma_port_get_link_layer(sqp->qp.ibqp.device, sqp->qp.port) == IB_LINK_LAYER_ETHERNET; - is_grh = mlx4_ib_ah_grh_present(ah); - if (is_eth) { - if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) { - /* When multi-function is enabled, the ib_core gid - * indexes don't necessarily match the hw ones, so - * we must use our own cache */ - err = mlx4_get_roce_gid_from_slave(to_mdev(ib_dev)->dev, - be32_to_cpu(ah->av.ib.port_pd) >> 24, - ah->av.ib.gid_index, &sgid.raw[0]); - if (err) - return err; - } else { - err = ib_get_cached_gid(ib_dev, - be32_to_cpu(ah->av.ib.port_pd) >> 24, - ah->av.ib.gid_index, &sgid); - if (err) - return err; - } - - if (is_eth && ah->av.eth.vlan != 0xffff) { - vlan = cpu_to_be16(ah->av.eth.vlan) & 0x0fff; - is_vlan = 1; - } - } - ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh, 0, &sqp->ud_header); - - if (!is_eth) { - sqp->ud_header.lrh.service_level = - be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28; - sqp->ud_header.lrh.destination_lid = ah->av.ib.dlid; - sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.ib.g_slid & 0x7f); - } - - if (is_grh) { - sqp->ud_header.grh.traffic_class = - (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff; - sqp->ud_header.grh.flow_label = - ah->av.ib.sl_tclass_flowlabel & cpu_to_be32(0xfffff); - sqp->ud_header.grh.hop_limit = ah->av.ib.hop_limit; - if (is_eth) - memcpy(sqp->ud_header.grh.source_gid.raw, sgid.raw, 16); - else { - if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) { - /* When multi-function is enabled, the ib_core gid - * indexes don't necessarily match the hw ones, so - * we must use our own cache */ - sqp->ud_header.grh.source_gid.global.subnet_prefix = - to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1]. - subnet_prefix; - sqp->ud_header.grh.source_gid.global.interface_id = - to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1]. - guid_cache[ah->av.ib.gid_index]; - } else - ib_get_cached_gid(ib_dev, - be32_to_cpu(ah->av.ib.port_pd) >> 24, - ah->av.ib.gid_index, - &sqp->ud_header.grh.source_gid); - } - memcpy(sqp->ud_header.grh.destination_gid.raw, - ah->av.ib.dgid, 16); - } - - mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); - - if (!is_eth) { - mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) | - (sqp->ud_header.lrh.destination_lid == - IB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) | - (sqp->ud_header.lrh.service_level << 8)); - if (ah->av.ib.port_pd & cpu_to_be32(0x80000000)) - mlx->flags |= cpu_to_be32(0x1); /* force loopback */ - mlx->rlid = sqp->ud_header.lrh.destination_lid; - } - - switch (wr->opcode) { - case IB_WR_SEND: - sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY; - sqp->ud_header.immediate_present = 0; - break; - case IB_WR_SEND_WITH_IMM: - sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; - sqp->ud_header.immediate_present = 1; - sqp->ud_header.immediate_data = wr->ex.imm_data; - break; - default: - return -EINVAL; - } - - if (is_eth) { - u8 *smac; - struct in6_addr in6; - - u16 pcp = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 29) << 13; - - mlx->sched_prio = cpu_to_be16(pcp); - - memcpy(sqp->ud_header.eth.dmac_h, ah->av.eth.mac, 6); - /* FIXME: cache smac value? */ - memcpy(&ctrl->srcrb_flags16[0], ah->av.eth.mac, 2); - memcpy(&ctrl->imm, ah->av.eth.mac + 2, 4); - memcpy(&in6, sgid.raw, sizeof(in6)); - - if (!mlx4_is_mfunc(to_mdev(ib_dev)->dev)) - smac = IF_LLADDR(to_mdev(sqp->qp.ibqp.device)->iboe.netdevs[sqp->qp.port - 1]); - else - smac = ah->av.eth.s_mac; /* use the src mac of the tunnel */ - memcpy(sqp->ud_header.eth.smac_h, smac, 6); - - if (!memcmp(sqp->ud_header.eth.smac_h, sqp->ud_header.eth.dmac_h, 6)) - mlx->flags |= cpu_to_be32(MLX4_WQE_CTRL_FORCE_LOOPBACK); - if (!is_vlan) { - sqp->ud_header.eth.type = cpu_to_be16(MLX4_IB_IBOE_ETHERTYPE); - } else { - sqp->ud_header.vlan.type = cpu_to_be16(MLX4_IB_IBOE_ETHERTYPE); - sqp->ud_header.vlan.tag = cpu_to_be16(vlan | pcp); - } - } else { - sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0; - if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE) - sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE; - } - sqp->ud_header.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED); - if (!sqp->qp.ibqp.qp_num) - ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey); - else - ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->wr.ud.pkey_index, &pkey); - sqp->ud_header.bth.pkey = cpu_to_be16(pkey); - sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn); - sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1)); - sqp->ud_header.deth.qkey = cpu_to_be32(wr->wr.ud.remote_qkey & 0x80000000 ? - sqp->qkey : wr->wr.ud.remote_qkey); - sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num); - - header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf); - - if (0) { - pr_err("built UD header of size %d:\n", header_size); - for (i = 0; i < header_size / 4; ++i) { - if (i % 8 == 0) - pr_err(" [%02x] ", i * 4); - pr_cont(" %08x", - be32_to_cpu(((__be32 *) sqp->header_buf)[i])); - if ((i + 1) % 8 == 0) - pr_cont("\n"); - } - pr_err("\n"); - } - - /* - * Inline data segments may not cross a 64 byte boundary. If - * our UD header is bigger than the space available up to the - * next 64 byte boundary in the WQE, use two inline data - * segments to hold the UD header. - */ - spc = MLX4_INLINE_ALIGN - - ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1)); - if (header_size <= spc) { - inl->byte_count = cpu_to_be32(1 << 31 | header_size); - memcpy(inl + 1, sqp->header_buf, header_size); - i = 1; - } else { - inl->byte_count = cpu_to_be32(1 << 31 | spc); - memcpy(inl + 1, sqp->header_buf, spc); - - inl = (void *) (inl + 1) + spc; - memcpy(inl + 1, sqp->header_buf + spc, header_size - spc); - /* - * Need a barrier here to make sure all the data is - * visible before the byte_count field is set. - * Otherwise the HCA prefetcher could grab the 64-byte - * chunk with this inline segment and get a valid (!= - * 0xffffffff) byte count but stale data, and end up - * generating a packet with bad headers. - * - * The first inline segment's byte_count field doesn't - * need a barrier, because it comes after a - * control/MLX segment and therefore is at an offset - * of 16 mod 64. - */ - wmb(); - inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc)); - i = 2; - } - - *mlx_seg_len = - ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16); - return 0; -} - -static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq) -{ - unsigned cur; - struct mlx4_ib_cq *cq; - - cur = wq->head - wq->tail; - if (likely(cur + nreq < wq->max_post)) - return 0; - - cq = to_mcq(ib_cq); - spin_lock(&cq->lock); - cur = wq->head - wq->tail; - spin_unlock(&cq->lock); - - return cur + nreq >= wq->max_post; -} - -static __be32 convert_access(int acc) -{ - return (acc & IB_ACCESS_REMOTE_ATOMIC ? - cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_ATOMIC) : 0) | - (acc & IB_ACCESS_REMOTE_WRITE ? - cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_WRITE) : 0) | - (acc & IB_ACCESS_REMOTE_READ ? - cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_READ) : 0) | - (acc & IB_ACCESS_LOCAL_WRITE ? cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_WRITE) : 0) | - cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_READ); -} - -static void set_fmr_seg(struct mlx4_wqe_fmr_seg *fseg, struct ib_send_wr *wr) -{ - struct mlx4_ib_fast_reg_page_list *mfrpl = to_mfrpl(wr->wr.fast_reg.page_list); - int i; - - for (i = 0; i < wr->wr.fast_reg.page_list_len; ++i) - mfrpl->mapped_page_list[i] = - cpu_to_be64(wr->wr.fast_reg.page_list->page_list[i] | - MLX4_MTT_FLAG_PRESENT); - - fseg->flags = convert_access(wr->wr.fast_reg.access_flags); - fseg->mem_key = cpu_to_be32(wr->wr.fast_reg.rkey); - fseg->buf_list = cpu_to_be64(mfrpl->map); - fseg->start_addr = cpu_to_be64(wr->wr.fast_reg.iova_start); - fseg->reg_len = cpu_to_be64(wr->wr.fast_reg.length); - fseg->offset = 0; /* XXX -- is this just for ZBVA? */ - fseg->page_size = cpu_to_be32(wr->wr.fast_reg.page_shift); - fseg->reserved[0] = 0; - fseg->reserved[1] = 0; -} - -static void set_bind_seg(struct mlx4_wqe_bind_seg *bseg, struct ib_send_wr *wr) -{ - bseg->flags1 = - convert_access(wr->wr.bind_mw.bind_info.mw_access_flags) & - cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_READ | - MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_WRITE | - MLX4_WQE_FMR_AND_BIND_PERM_ATOMIC); - bseg->flags2 = 0; - if (wr->wr.bind_mw.mw->type == IB_MW_TYPE_2) - bseg->flags2 |= cpu_to_be32(MLX4_WQE_BIND_TYPE_2); - if (wr->wr.bind_mw.bind_info.mw_access_flags & IB_ZERO_BASED) - bseg->flags2 |= cpu_to_be32(MLX4_WQE_BIND_ZERO_BASED); - bseg->new_rkey = cpu_to_be32(wr->wr.bind_mw.rkey); - bseg->lkey = cpu_to_be32(wr->wr.bind_mw.bind_info.mr->lkey); - bseg->addr = cpu_to_be64(wr->wr.bind_mw.bind_info.addr); - bseg->length = cpu_to_be64(wr->wr.bind_mw.bind_info.length); -} - -static void set_local_inv_seg(struct mlx4_wqe_local_inval_seg *iseg, u32 rkey) -{ - iseg->mem_key = cpu_to_be32(rkey); - - iseg->reserved1 = 0; - iseg->reserved2 = 0; - iseg->reserved3[0] = 0; - iseg->reserved3[1] = 0; -} - -static __always_inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg, - u64 remote_addr, u32 rkey) -{ - rseg->raddr = cpu_to_be64(remote_addr); - rseg->rkey = cpu_to_be32(rkey); - rseg->reserved = 0; -} - -static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg, struct ib_send_wr *wr) -{ - if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) { - aseg->swap_add = cpu_to_be64(wr->wr.atomic.swap); - aseg->compare = cpu_to_be64(wr->wr.atomic.compare_add); - } else if (wr->opcode == IB_WR_MASKED_ATOMIC_FETCH_AND_ADD) { - aseg->swap_add = cpu_to_be64(wr->wr.atomic.compare_add); - aseg->compare = cpu_to_be64(wr->wr.atomic.compare_add_mask); - } else { - aseg->swap_add = cpu_to_be64(wr->wr.atomic.compare_add); - aseg->compare = 0; - } - -} - -static void set_masked_atomic_seg(struct mlx4_wqe_masked_atomic_seg *aseg, - struct ib_send_wr *wr) -{ - aseg->swap_add = cpu_to_be64(wr->wr.atomic.swap); - aseg->swap_add_mask = cpu_to_be64(wr->wr.atomic.swap_mask); - aseg->compare = cpu_to_be64(wr->wr.atomic.compare_add); - aseg->compare_mask = cpu_to_be64(wr->wr.atomic.compare_add_mask); -} - -static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg, - struct ib_send_wr *wr) -{ - memcpy(dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof (struct mlx4_av)); - dseg->dqpn = cpu_to_be32(wr->wr.ud.remote_qpn); - dseg->qkey = cpu_to_be32(wr->wr.ud.remote_qkey); - dseg->vlan = to_mah(wr->wr.ud.ah)->av.eth.vlan; - memcpy(dseg->mac, to_mah(wr->wr.ud.ah)->av.eth.mac, 6); -} - -static void set_tunnel_datagram_seg(struct mlx4_ib_dev *dev, - struct mlx4_wqe_datagram_seg *dseg, - struct ib_send_wr *wr, enum ib_qp_type qpt) -{ - union mlx4_ext_av *av = &to_mah(wr->wr.ud.ah)->av; - struct mlx4_av sqp_av = {0}; - int port = *((u8 *) &av->ib.port_pd) & 0x3; - - /* force loopback */ - sqp_av.port_pd = av->ib.port_pd | cpu_to_be32(0x80000000); - sqp_av.g_slid = av->ib.g_slid & 0x7f; /* no GRH */ - sqp_av.sl_tclass_flowlabel = av->ib.sl_tclass_flowlabel & - cpu_to_be32(0xf0000000); - - memcpy(dseg->av, &sqp_av, sizeof (struct mlx4_av)); - /* This function used only for sending on QP1 proxies */ - dseg->dqpn = cpu_to_be32(dev->dev->caps.qp1_tunnel[port - 1]); - /* Use QKEY from the QP context, which is set by master */ - dseg->qkey = cpu_to_be32(IB_QP_SET_QKEY); -} - -static void build_tunnel_header(struct ib_send_wr *wr, void *wqe, unsigned *mlx_seg_len) -{ - struct mlx4_wqe_inline_seg *inl = wqe; - struct mlx4_ib_tunnel_header hdr; - struct mlx4_ib_ah *ah = to_mah(wr->wr.ud.ah); - int spc; - int i; - - memcpy(&hdr.av, &ah->av, sizeof hdr.av); - hdr.remote_qpn = cpu_to_be32(wr->wr.ud.remote_qpn); - hdr.pkey_index = cpu_to_be16(wr->wr.ud.pkey_index); - hdr.qkey = cpu_to_be32(wr->wr.ud.remote_qkey); - memcpy(hdr.mac, ah->av.eth.mac, 6); - hdr.vlan = cpu_to_be16(ah->av.eth.vlan); - - spc = MLX4_INLINE_ALIGN - - ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1)); - if (sizeof (hdr) <= spc) { - memcpy(inl + 1, &hdr, sizeof (hdr)); - wmb(); - inl->byte_count = cpu_to_be32(1 << 31 | sizeof (hdr)); - i = 1; - } else { - memcpy(inl + 1, &hdr, spc); - wmb(); - inl->byte_count = cpu_to_be32(1 << 31 | spc); - - inl = (void *) (inl + 1) + spc; - memcpy(inl + 1, (void *) &hdr + spc, sizeof (hdr) - spc); - wmb(); - inl->byte_count = cpu_to_be32(1 << 31 | (sizeof (hdr) - spc)); - i = 2; - } - - *mlx_seg_len = - ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + sizeof (hdr), 16); -} - -static void set_mlx_icrc_seg(void *dseg) -{ - u32 *t = dseg; - struct mlx4_wqe_inline_seg *iseg = dseg; - - t[1] = 0; - - /* - * Need a barrier here before writing the byte_count field to - * make sure that all the data is visible before the - * byte_count field is set. Otherwise, if the segment begins - * a new cacheline, the HCA prefetcher could grab the 64-byte - * chunk and get a valid (!= * 0xffffffff) byte count but - * stale data, and end up sending the wrong data. - */ - wmb(); - - iseg->byte_count = cpu_to_be32((1 << 31) | 4); -} - -static void set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg) -{ - dseg->lkey = cpu_to_be32(sg->lkey); - dseg->addr = cpu_to_be64(sg->addr); - - /* - * Need a barrier here before writing the byte_count field to - * make sure that all the data is visible before the - * byte_count field is set. Otherwise, if the segment begins - * a new cacheline, the HCA prefetcher could grab the 64-byte - * chunk and get a valid (!= * 0xffffffff) byte count but - * stale data, and end up sending the wrong data. - */ - wmb(); - - dseg->byte_count = cpu_to_be32(sg->length); -} - -static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg) -{ - dseg->byte_count = cpu_to_be32(sg->length); - dseg->lkey = cpu_to_be32(sg->lkey); - dseg->addr = cpu_to_be64(sg->addr); -} - -static int build_lso_seg(struct mlx4_wqe_lso_seg *wqe, struct ib_send_wr *wr, - struct mlx4_ib_qp *qp, unsigned *lso_seg_len, - __be32 *lso_hdr_sz, __be32 *blh) -{ - unsigned halign = ALIGN(sizeof *wqe + wr->wr.ud.hlen, 16); - - if (unlikely(halign > MLX4_IB_CACHE_LINE_SIZE)) - *blh = cpu_to_be32(1 << 6); - - if (unlikely(!(qp->flags & MLX4_IB_QP_LSO) && - wr->num_sge > qp->sq.max_gs - (halign >> 4))) - return -EINVAL; - - memcpy(wqe->header, wr->wr.ud.header, wr->wr.ud.hlen); - - *lso_hdr_sz = cpu_to_be32((wr->wr.ud.mss - wr->wr.ud.hlen) << 16 | - wr->wr.ud.hlen); - *lso_seg_len = halign; - return 0; -} - -static __be32 send_ieth(struct ib_send_wr *wr) -{ - switch (wr->opcode) { - case IB_WR_SEND_WITH_IMM: - case IB_WR_RDMA_WRITE_WITH_IMM: - return wr->ex.imm_data; - - case IB_WR_SEND_WITH_INV: - return cpu_to_be32(wr->ex.invalidate_rkey); - - default: - return 0; - } -} - -static void add_zero_len_inline(void *wqe) -{ - struct mlx4_wqe_inline_seg *inl = wqe; - memset(wqe, 0, 16); - inl->byte_count = cpu_to_be32(1 << 31); -} - -static int lay_inline_data(struct mlx4_ib_qp *qp, struct ib_send_wr *wr, - void *wqe, int *sz) -{ - struct mlx4_wqe_inline_seg *seg; - void *addr; - int len, seg_len; - int num_seg; - int off, to_copy; - int i; - int inl = 0; - - seg = wqe; - wqe += sizeof *seg; - off = ((unsigned long)wqe) & (unsigned long)(MLX4_INLINE_ALIGN - 1); - num_seg = 0; - seg_len = 0; - - for (i = 0; i < wr->num_sge; ++i) { - addr = (void *) (unsigned long)(wr->sg_list[i].addr); - len = wr->sg_list[i].length; - inl += len; - - if (inl > qp->max_inline_data) { - inl = 0; - return -1; - } - - while (len >= MLX4_INLINE_ALIGN - off) { - to_copy = MLX4_INLINE_ALIGN - off; - memcpy(wqe, addr, to_copy); - len -= to_copy; - wqe += to_copy; - addr += to_copy; - seg_len += to_copy; - wmb(); /* see comment below */ - seg->byte_count = htonl(MLX4_INLINE_SEG | seg_len); - seg_len = 0; - seg = wqe; - wqe += sizeof *seg; - off = sizeof *seg; - ++num_seg; - } - - memcpy(wqe, addr, len); - wqe += len; - seg_len += len; - off += len; - } - - if (seg_len) { - ++num_seg; - /* - * Need a barrier here to make sure - * all the data is visible before the - * byte_count field is set. Otherwise - * the HCA prefetcher could grab the - * 64-byte chunk with this inline - * segment and get a valid (!= - * 0xffffffff) byte count but stale - * data, and end up sending the wrong - * data. - */ - wmb(); - seg->byte_count = htonl(MLX4_INLINE_SEG | seg_len); - } - - *sz = (inl + num_seg * sizeof *seg + 15) / 16; - - return 0; -} - -/* - * Avoid using memcpy() to copy to BlueFlame page, since memcpy() - * implementations may use move-string-buffer assembler instructions, - * which do not guarantee order of copying. - */ -static void mlx4_bf_copy(unsigned long *dst, unsigned long *src, - unsigned bytecnt) -{ - __iowrite64_copy(dst, src, bytecnt / 8); -} - -int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, - struct ib_send_wr **bad_wr) -{ - struct mlx4_ib_qp *qp = to_mqp(ibqp); - void *wqe; - struct mlx4_wqe_ctrl_seg *uninitialized_var(ctrl); - struct mlx4_wqe_data_seg *dseg; - unsigned long flags; - int nreq; - int err = 0; - unsigned ind; - int uninitialized_var(stamp); - int uninitialized_var(size); - unsigned uninitialized_var(seglen); - __be32 dummy; - __be32 *lso_wqe; - __be32 uninitialized_var(lso_hdr_sz); - __be32 blh; - int i; - int inl = 0; - spin_lock_irqsave(&qp->sq.lock, flags); - - ind = qp->sq_next_wqe; - - for (nreq = 0; wr; ++nreq, wr = wr->next) { - lso_wqe = &dummy; - blh = 0; - - if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { - err = -ENOMEM; - *bad_wr = wr; - goto out; - } - - if (unlikely(wr->num_sge > qp->sq.max_gs)) { - err = -EINVAL; - *bad_wr = wr; - goto out; - } - - ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1)); - *((u32 *) (&ctrl->vlan_tag)) = 0; - qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = wr->wr_id; - - ctrl->srcrb_flags = - (wr->send_flags & IB_SEND_SIGNALED ? - cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) | - (wr->send_flags & IB_SEND_SOLICITED ? - cpu_to_be32(MLX4_WQE_CTRL_SOLICITED) : 0) | - ((wr->send_flags & IB_SEND_IP_CSUM) ? - cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM | - MLX4_WQE_CTRL_TCP_UDP_CSUM) : 0) | - qp->sq_signal_bits; - - ctrl->imm = send_ieth(wr); - - wqe += sizeof *ctrl; - size = sizeof *ctrl / 16; - - switch (qp->mlx4_ib_qp_type) { - case MLX4_IB_QPT_RC: - case MLX4_IB_QPT_UC: - switch (wr->opcode) { - case IB_WR_ATOMIC_CMP_AND_SWP: - case IB_WR_ATOMIC_FETCH_AND_ADD: - case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD: - set_raddr_seg(wqe, wr->wr.atomic.remote_addr, - wr->wr.atomic.rkey); - wqe += sizeof (struct mlx4_wqe_raddr_seg); - - set_atomic_seg(wqe, wr); - wqe += sizeof (struct mlx4_wqe_atomic_seg); - - size += (sizeof (struct mlx4_wqe_raddr_seg) + - sizeof (struct mlx4_wqe_atomic_seg)) / 16; - - break; - - case IB_WR_MASKED_ATOMIC_CMP_AND_SWP: - set_raddr_seg(wqe, wr->wr.atomic.remote_addr, - wr->wr.atomic.rkey); - wqe += sizeof (struct mlx4_wqe_raddr_seg); - - set_masked_atomic_seg(wqe, wr); - wqe += sizeof (struct mlx4_wqe_masked_atomic_seg); - - size += (sizeof (struct mlx4_wqe_raddr_seg) + - sizeof (struct mlx4_wqe_masked_atomic_seg)) / 16; - - break; - - case IB_WR_RDMA_READ: - case IB_WR_RDMA_WRITE: - case IB_WR_RDMA_WRITE_WITH_IMM: - set_raddr_seg(wqe, wr->wr.rdma.remote_addr, - wr->wr.rdma.rkey); - wqe += sizeof (struct mlx4_wqe_raddr_seg); - size += sizeof (struct mlx4_wqe_raddr_seg) / 16; - break; - - case IB_WR_LOCAL_INV: - ctrl->srcrb_flags |= - cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER); - set_local_inv_seg(wqe, wr->ex.invalidate_rkey); - wqe += sizeof (struct mlx4_wqe_local_inval_seg); - size += sizeof (struct mlx4_wqe_local_inval_seg) / 16; - break; - - case IB_WR_FAST_REG_MR: - ctrl->srcrb_flags |= - cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER); - set_fmr_seg(wqe, wr); - wqe += sizeof (struct mlx4_wqe_fmr_seg); - size += sizeof (struct mlx4_wqe_fmr_seg) / 16; - break; - - case IB_WR_BIND_MW: - ctrl->srcrb_flags |= - cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER); - set_bind_seg(wqe, wr); - wqe += sizeof(struct mlx4_wqe_bind_seg); - size += sizeof(struct mlx4_wqe_bind_seg) / 16; - default: - /* No extra segments required for sends */ - break; - } - break; - - case MLX4_IB_QPT_TUN_SMI_OWNER: - err = build_sriov_qp0_header(to_msqp(qp), wr, ctrl, &seglen); - if (unlikely(err)) { - *bad_wr = wr; - goto out; - } - wqe += seglen; - size += seglen / 16; - break; - case MLX4_IB_QPT_TUN_SMI: - case MLX4_IB_QPT_TUN_GSI: - /* this is a UD qp used in MAD responses to slaves. */ - set_datagram_seg(wqe, wr); - /* set the forced-loopback bit in the data seg av */ - *(__be32 *) wqe |= cpu_to_be32(0x80000000); - wqe += sizeof (struct mlx4_wqe_datagram_seg); - size += sizeof (struct mlx4_wqe_datagram_seg) / 16; - break; - case MLX4_IB_QPT_UD: - set_datagram_seg(wqe, wr); - wqe += sizeof (struct mlx4_wqe_datagram_seg); - size += sizeof (struct mlx4_wqe_datagram_seg) / 16; - - if (wr->opcode == IB_WR_LSO) { - err = build_lso_seg(wqe, wr, qp, &seglen, &lso_hdr_sz, &blh); - if (unlikely(err)) { - *bad_wr = wr; - goto out; - } - lso_wqe = (__be32 *) wqe; - wqe += seglen; - size += seglen / 16; - } - break; - - case MLX4_IB_QPT_PROXY_SMI_OWNER: - if (unlikely(!mlx4_is_master(to_mdev(ibqp->device)->dev))) { - err = -ENOSYS; - *bad_wr = wr; - goto out; - } - err = build_sriov_qp0_header(to_msqp(qp), wr, ctrl, &seglen); - if (unlikely(err)) { - *bad_wr = wr; - goto out; - } - wqe += seglen; - size += seglen / 16; - /* to start tunnel header on a cache-line boundary */ - add_zero_len_inline(wqe); - wqe += 16; - size++; - build_tunnel_header(wr, wqe, &seglen); - wqe += seglen; - size += seglen / 16; - break; - case MLX4_IB_QPT_PROXY_SMI: - /* don't allow QP0 sends on guests */ - err = -ENOSYS; - *bad_wr = wr; - goto out; - case MLX4_IB_QPT_PROXY_GSI: - /* If we are tunneling special qps, this is a UD qp. - * In this case we first add a UD segment targeting - * the tunnel qp, and then add a header with address - * information */ - set_tunnel_datagram_seg(to_mdev(ibqp->device), wqe, wr, ibqp->qp_type); - wqe += sizeof (struct mlx4_wqe_datagram_seg); - size += sizeof (struct mlx4_wqe_datagram_seg) / 16; - build_tunnel_header(wr, wqe, &seglen); - wqe += seglen; - size += seglen / 16; - break; - - case MLX4_IB_QPT_SMI: - case MLX4_IB_QPT_GSI: - err = build_mlx_header(to_msqp(qp), wr, ctrl, &seglen); - if (unlikely(err)) { - *bad_wr = wr; - goto out; - } - wqe += seglen; - size += seglen / 16; - break; - - default: - break; - } - - /* - * Write data segments in reverse order, so as to - * overwrite cacheline stamp last within each - * cacheline. This avoids issues with WQE - * prefetching. - */ - dseg = wqe; - dseg += wr->num_sge - 1; - - /* Add one more inline data segment for ICRC for MLX sends */ - if (unlikely(qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI || - qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI || - qp->mlx4_ib_qp_type & - (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER))) { - set_mlx_icrc_seg(dseg + 1); - size += sizeof (struct mlx4_wqe_data_seg) / 16; - } - - if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) { - int sz; - err = lay_inline_data(qp, wr, wqe, &sz); - if (!err) { - inl = 1; - size += sz; - } - } else { - size += wr->num_sge * - (sizeof(struct mlx4_wqe_data_seg) / 16); - for (i = wr->num_sge - 1; i >= 0; --i, --dseg) - set_data_seg(dseg, wr->sg_list + i); - } - - /* - * Possibly overwrite stamping in cacheline with LSO - * segment only after making sure all data segments - * are written. - */ - wmb(); - *lso_wqe = lso_hdr_sz; - ctrl->fence_size = (wr->send_flags & IB_SEND_FENCE ? - MLX4_WQE_CTRL_FENCE : 0) | size; - - /* - * Make sure descriptor is fully written before - * setting ownership bit (because HW can start - * executing as soon as we do). - */ - wmb(); - - if (wr->opcode < 0 || wr->opcode >= ARRAY_SIZE(mlx4_ib_opcode)) { - *bad_wr = wr; - err = -EINVAL; - goto out; - } - - ctrl->owner_opcode = mlx4_ib_opcode[wr->opcode] | - (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0) | blh; - - stamp = ind + qp->sq_spare_wqes; - ind += DIV_ROUND_UP(size * 16, 1U << qp->sq.wqe_shift); - - /* - * We can improve latency by not stamping the last - * send queue WQE until after ringing the doorbell, so - * only stamp here if there are still more WQEs to post. - * - * Same optimization applies to padding with NOP wqe - * in case of WQE shrinking (used to prevent wrap-around - * in the middle of WR). - */ - if (wr->next) { - stamp_send_wqe(qp, stamp, size * 16); - ind = pad_wraparound(qp, ind); - } - } - -out: - if (nreq == 1 && inl && size > 1 && size < qp->bf.buf_size / 16) { - ctrl->owner_opcode |= htonl((qp->sq_next_wqe & 0xffff) << 8); - /* We set above doorbell_qpn bits to 0 as part of vlan - * tag initialization, so |= should be correct. - */ - *(u32 *) (&ctrl->vlan_tag) |= qp->doorbell_qpn; - /* - * Make sure that descriptor is written to memory - * before writing to BlueFlame page. - */ - wmb(); - - ++qp->sq.head; - - mlx4_bf_copy(qp->bf.reg + qp->bf.offset, (unsigned long *) ctrl, - ALIGN(size * 16, 64)); - wc_wmb(); - - qp->bf.offset ^= qp->bf.buf_size; - - } else if (nreq) { - qp->sq.head += nreq; - - /* - * Make sure that descriptors are written before - * doorbell record. - */ - wmb(); - - writel(qp->doorbell_qpn, qp->bf.uar->map + MLX4_SEND_DOORBELL); - - /* - * Make sure doorbells don't leak out of SQ spinlock - * and reach the HCA out of order. - */ - mmiowb(); - - } - - if (likely(nreq)) { - stamp_send_wqe(qp, stamp, size * 16); - ind = pad_wraparound(qp, ind); - qp->sq_next_wqe = ind; - } - - spin_unlock_irqrestore(&qp->sq.lock, flags); - - return err; -} - -int mlx4_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, - struct ib_recv_wr **bad_wr) -{ - struct mlx4_ib_qp *qp = to_mqp(ibqp); - struct mlx4_wqe_data_seg *scat; - unsigned long flags; - int err = 0; - int nreq; - int ind; - int max_gs; - int i; - - max_gs = qp->rq.max_gs; - spin_lock_irqsave(&qp->rq.lock, flags); - - ind = qp->rq.head & (qp->rq.wqe_cnt - 1); - - for (nreq = 0; wr; ++nreq, wr = wr->next) { - if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) { - err = -ENOMEM; - *bad_wr = wr; - goto out; - } - - if (unlikely(wr->num_sge > qp->rq.max_gs)) { - err = -EINVAL; - *bad_wr = wr; - goto out; - } - - scat = get_recv_wqe(qp, ind); - - if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER | - MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) { - ib_dma_sync_single_for_device(ibqp->device, - qp->sqp_proxy_rcv[ind].map, - sizeof (struct mlx4_ib_proxy_sqp_hdr), - DMA_FROM_DEVICE); - scat->byte_count = - cpu_to_be32(sizeof (struct mlx4_ib_proxy_sqp_hdr)); - /* use dma lkey from upper layer entry */ - scat->lkey = cpu_to_be32(wr->sg_list->lkey); - scat->addr = cpu_to_be64(qp->sqp_proxy_rcv[ind].map); - scat++; - max_gs--; - } - - for (i = 0; i < wr->num_sge; ++i) - __set_data_seg(scat + i, wr->sg_list + i); - - if (i < max_gs) { - scat[i].byte_count = 0; - scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY); - scat[i].addr = 0; - } - - qp->rq.wrid[ind] = wr->wr_id; - - ind = (ind + 1) & (qp->rq.wqe_cnt - 1); - } - -out: - if (likely(nreq)) { - qp->rq.head += nreq; - - /* - * Make sure that descriptors are written before - * doorbell record. - */ - wmb(); - - *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff); - } - - spin_unlock_irqrestore(&qp->rq.lock, flags); - - return err; -} - -static inline enum ib_qp_state to_ib_qp_state(enum mlx4_qp_state mlx4_state) -{ - switch (mlx4_state) { - case MLX4_QP_STATE_RST: return IB_QPS_RESET; - case MLX4_QP_STATE_INIT: return IB_QPS_INIT; - case MLX4_QP_STATE_RTR: return IB_QPS_RTR; - case MLX4_QP_STATE_RTS: return IB_QPS_RTS; - case MLX4_QP_STATE_SQ_DRAINING: - case MLX4_QP_STATE_SQD: return IB_QPS_SQD; - case MLX4_QP_STATE_SQER: return IB_QPS_SQE; - case MLX4_QP_STATE_ERR: return IB_QPS_ERR; - default: return -1; - } -} - -static inline enum ib_mig_state to_ib_mig_state(int mlx4_mig_state) -{ - switch (mlx4_mig_state) { - case MLX4_QP_PM_ARMED: return IB_MIG_ARMED; - case MLX4_QP_PM_REARM: return IB_MIG_REARM; - case MLX4_QP_PM_MIGRATED: return IB_MIG_MIGRATED; - default: return -1; - } -} - -static int to_ib_qp_access_flags(int mlx4_flags) -{ - int ib_flags = 0; - - if (mlx4_flags & MLX4_QP_BIT_RRE) - ib_flags |= IB_ACCESS_REMOTE_READ; - if (mlx4_flags & MLX4_QP_BIT_RWE) - ib_flags |= IB_ACCESS_REMOTE_WRITE; - if (mlx4_flags & MLX4_QP_BIT_RAE) - ib_flags |= IB_ACCESS_REMOTE_ATOMIC; - - return ib_flags; -} - -static void to_ib_ah_attr(struct mlx4_ib_dev *ibdev, struct ib_ah_attr *ib_ah_attr, - struct mlx4_qp_path *path) -{ - struct mlx4_dev *dev = ibdev->dev; - int is_eth; - - memset(ib_ah_attr, 0, sizeof *ib_ah_attr); - ib_ah_attr->port_num = path->sched_queue & 0x40 ? 2 : 1; - - if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->caps.num_ports) - return; - - is_eth = rdma_port_get_link_layer(&ibdev->ib_dev, ib_ah_attr->port_num) == - IB_LINK_LAYER_ETHERNET; - if (is_eth) - ib_ah_attr->sl = ((path->sched_queue >> 3) & 0x7) | - ((path->sched_queue & 4) << 1); - else - ib_ah_attr->sl = (path->sched_queue >> 2) & 0xf; - - ib_ah_attr->dlid = be16_to_cpu(path->rlid); - ib_ah_attr->src_path_bits = path->grh_mylmc & 0x7f; - ib_ah_attr->static_rate = path->static_rate ? path->static_rate - 5 : 0; - ib_ah_attr->ah_flags = (path->grh_mylmc & (1 << 7)) ? IB_AH_GRH : 0; - if (ib_ah_attr->ah_flags) { - ib_ah_attr->grh.sgid_index = path->mgid_index; - ib_ah_attr->grh.hop_limit = path->hop_limit; - ib_ah_attr->grh.traffic_class = - (be32_to_cpu(path->tclass_flowlabel) >> 20) & 0xff; - ib_ah_attr->grh.flow_label = - be32_to_cpu(path->tclass_flowlabel) & 0xfffff; - memcpy(ib_ah_attr->grh.dgid.raw, - path->rgid, sizeof ib_ah_attr->grh.dgid.raw); - } -} - -int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, - struct ib_qp_init_attr *qp_init_attr) -{ - struct mlx4_ib_dev *dev = to_mdev(ibqp->device); - struct mlx4_ib_qp *qp = to_mqp(ibqp); - struct mlx4_qp_context context; - int mlx4_state; - int err = 0; - - mutex_lock(&qp->mutex); - - if (qp->state == IB_QPS_RESET) { - qp_attr->qp_state = IB_QPS_RESET; - goto done; - } - - err = mlx4_qp_query(dev->dev, &qp->mqp, &context); - if (err) { - err = -EINVAL; - goto out; - } - - mlx4_state = be32_to_cpu(context.flags) >> 28; - - qp->state = to_ib_qp_state(mlx4_state); - qp_attr->qp_state = qp->state; - qp_attr->path_mtu = context.mtu_msgmax >> 5; - qp_attr->path_mig_state = - to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3); - qp_attr->qkey = be32_to_cpu(context.qkey); - qp_attr->rq_psn = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff; - qp_attr->sq_psn = be32_to_cpu(context.next_send_psn) & 0xffffff; - qp_attr->dest_qp_num = be32_to_cpu(context.remote_qpn) & 0xffffff; - qp_attr->qp_access_flags = - to_ib_qp_access_flags(be32_to_cpu(context.params2)); - - if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) { - to_ib_ah_attr(dev, &qp_attr->ah_attr, &context.pri_path); - to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context.alt_path); - qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f; - qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num; - } - - qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f; - if (qp_attr->qp_state == IB_QPS_INIT) - qp_attr->port_num = qp->port; - else - qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1; - - /* qp_attr->en_sqd_async_notify is only applicable in modify qp */ - qp_attr->sq_draining = mlx4_state == MLX4_QP_STATE_SQ_DRAINING; - - qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context.params1) >> 21) & 0x7); - - qp_attr->max_dest_rd_atomic = - 1 << ((be32_to_cpu(context.params2) >> 21) & 0x7); - qp_attr->min_rnr_timer = - (be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f; - qp_attr->timeout = context.pri_path.ackto >> 3; - qp_attr->retry_cnt = (be32_to_cpu(context.params1) >> 16) & 0x7; - qp_attr->rnr_retry = (be32_to_cpu(context.params1) >> 13) & 0x7; - qp_attr->alt_timeout = context.alt_path.ackto >> 3; - -done: - qp_attr->cur_qp_state = qp_attr->qp_state; - qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt; - qp_attr->cap.max_recv_sge = qp->rq.max_gs; - - if (!ibqp->uobject) { - qp_attr->cap.max_send_wr = qp->sq.wqe_cnt; - qp_attr->cap.max_send_sge = qp->sq.max_gs; - } else { - qp_attr->cap.max_send_wr = 0; - qp_attr->cap.max_send_sge = 0; - } - - /* - * We don't support inline sends for kernel QPs (yet), and we - * don't know what userspace's value should be. - */ - qp_attr->cap.max_inline_data = 0; - - qp_init_attr->cap = qp_attr->cap; - - qp_init_attr->create_flags = 0; - if (qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) - qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK; - - if (qp->flags & MLX4_IB_QP_LSO) - qp_init_attr->create_flags |= IB_QP_CREATE_IPOIB_UD_LSO; - - if (qp->flags & MLX4_IB_QP_NETIF) - qp_init_attr->create_flags |= IB_QP_CREATE_NETIF_QP; - - qp_init_attr->sq_sig_type = - qp->sq_signal_bits == cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) ? - IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; - - if (qp->flags & MLX4_IB_QP_CAP_CROSS_CHANNEL) - qp_init_attr->create_flags |= IB_QP_CREATE_CROSS_CHANNEL; - - if (qp->flags & MLX4_IB_QP_CAP_MANAGED_SEND) - qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_SEND; - - if (qp->flags & MLX4_IB_QP_CAP_MANAGED_RECV) - qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_RECV; - - qp_init_attr->qpg_type = ibqp->qpg_type; - if (ibqp->qpg_type == IB_QPG_PARENT) - qp_init_attr->cap.qpg_tss_mask_sz = qp->qpg_data->qpg_tss_mask_sz; - else - qp_init_attr->cap.qpg_tss_mask_sz = 0; - -out: - mutex_unlock(&qp->mutex); - return err; -} Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/qp.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/alias_GUID.c =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/alias_GUID.c (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/alias_GUID.c (nonexistent) @@ -1,702 +0,0 @@ -/* - * Copyright (c) 2012 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - /***********************************************************/ -/*This file support the handling of the Alias GUID feature. */ -/***********************************************************/ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "mlx4_ib.h" - -/* -The driver keeps the current state of all guids, as they are in the HW. -Whenever we receive an smp mad GUIDInfo record, the data will be cached. -*/ - -struct mlx4_alias_guid_work_context { - u8 port; - struct mlx4_ib_dev *dev ; - struct ib_sa_query *sa_query; - struct completion done; - int query_id; - struct list_head list; - int block_num; - u8 method; -}; - -struct mlx4_next_alias_guid_work { - u8 port; - u8 block_num; - struct mlx4_sriov_alias_guid_info_rec_det rec_det; -}; - - -void mlx4_ib_update_cache_on_guid_change(struct mlx4_ib_dev *dev, int block_num, - u8 port_num, u8 *p_data) -{ - int i; - u64 guid_indexes; - int slave_id; - int port_index = port_num - 1; - - if (!mlx4_is_master(dev->dev)) - return; - - guid_indexes = be64_to_cpu((__force __be64) dev->sriov.alias_guid. - ports_guid[port_num - 1]. - all_rec_per_port[block_num].guid_indexes); - pr_debug("port: %d, guid_indexes: 0x%llx\n", port_num, - (unsigned long long)guid_indexes); - - for (i = 0; i < NUM_ALIAS_GUID_IN_REC; i++) { - /* The location of the specific index starts from bit number 4 - * until bit num 11 */ - if (test_bit(i + 4, (unsigned long *)&guid_indexes)) { - slave_id = (block_num * NUM_ALIAS_GUID_IN_REC) + i ; - if (slave_id >= dev->dev->num_slaves) { - pr_debug("The last slave: %d\n", slave_id); - return; - } - - /* cache the guid: */ - memcpy(&dev->sriov.demux[port_index].guid_cache[slave_id], - &p_data[i * GUID_REC_SIZE], - GUID_REC_SIZE); - } else - pr_debug("Guid number: %d in block: %d" - " was not updated\n", i, block_num); - } -} - -static __be64 get_cached_alias_guid(struct mlx4_ib_dev *dev, int port, int index) -{ - if (index >= NUM_ALIAS_GUID_PER_PORT) { - pr_err("%s: ERROR: asked for index:%d\n", __func__, index); - return (__force __be64) -1; - } - return *(__be64 *)&dev->sriov.demux[port - 1].guid_cache[index]; -} - - -ib_sa_comp_mask mlx4_ib_get_aguid_comp_mask_from_ix(int index) -{ - return IB_SA_COMP_MASK(4 + index); -} - -/* - * Whenever new GUID is set/unset (guid table change) create event and - * notify the relevant slave (master also should be notified). - * If the GUID value is not as we have in the cache the slave will not be - * updated; in this case it waits for the smp_snoop or the port management - * event to call the function and to update the slave. - * block_number - the index of the block (16 blocks available) - * port_number - 1 or 2 - */ -void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev, - int block_num, u8 port_num, - u8 *p_data) -{ - int i; - u64 guid_indexes; - int slave_id; - enum slave_port_state new_state; - enum slave_port_state prev_state; - __be64 tmp_cur_ag, form_cache_ag; - enum slave_port_gen_event gen_event; - - if (!mlx4_is_master(dev->dev)) - return; - - guid_indexes = be64_to_cpu((__force __be64) dev->sriov.alias_guid. - ports_guid[port_num - 1]. - all_rec_per_port[block_num].guid_indexes); - pr_debug("port: %d, guid_indexes: 0x%llx\n", port_num, - (unsigned long long)guid_indexes); - - /*calculate the slaves and notify them*/ - for (i = 0; i < NUM_ALIAS_GUID_IN_REC; i++) { - /* the location of the specific index runs from bits 4..11 */ - if (!(test_bit(i + 4, (unsigned long *)&guid_indexes))) - continue; - - slave_id = (block_num * NUM_ALIAS_GUID_IN_REC) + i ; - if (slave_id >= dev->dev->num_slaves) - return; - tmp_cur_ag = *(__be64 *)&p_data[i * GUID_REC_SIZE]; - form_cache_ag = get_cached_alias_guid(dev, port_num, - (NUM_ALIAS_GUID_IN_REC * block_num) + i); - /* - * Check if guid is not the same as in the cache, - * If it is different, wait for the snoop_smp or the port mgmt - * change event to update the slave on its port state change - */ - if (tmp_cur_ag != form_cache_ag) - continue; - mlx4_gen_guid_change_eqe(dev->dev, slave_id, port_num); - - /*2 cases: Valid GUID, and Invalid Guid*/ - - if (tmp_cur_ag != MLX4_NOT_SET_GUID) { /*valid GUID*/ - prev_state = mlx4_get_slave_port_state(dev->dev, slave_id, port_num); - new_state = set_and_calc_slave_port_state(dev->dev, slave_id, port_num, - MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID, - &gen_event); - pr_debug("slave: %d, port: %d prev_port_state: %d," - " new_port_state: %d, gen_event: %d\n", - slave_id, port_num, prev_state, new_state, gen_event); - if (gen_event == SLAVE_PORT_GEN_EVENT_UP) { - pr_debug("sending PORT_UP event to slave: %d, port: %d\n", - slave_id, port_num); - mlx4_gen_port_state_change_eqe(dev->dev, slave_id, - port_num, MLX4_PORT_CHANGE_SUBTYPE_ACTIVE); - } - } else { /* request to invalidate GUID */ - set_and_calc_slave_port_state(dev->dev, slave_id, port_num, - MLX4_PORT_STATE_IB_EVENT_GID_INVALID, - &gen_event); - pr_debug("sending PORT DOWN event to slave: %d, port: %d\n", - slave_id, port_num); - mlx4_gen_port_state_change_eqe(dev->dev, slave_id, port_num, - MLX4_PORT_CHANGE_SUBTYPE_DOWN); - } - } -} - -static void aliasguid_query_handler(int status, - struct ib_sa_guidinfo_rec *guid_rec, - void *context) -{ - struct mlx4_ib_dev *dev; - struct mlx4_alias_guid_work_context *cb_ctx = context; - u8 port_index; - int i; - struct mlx4_sriov_alias_guid_info_rec_det *rec; - unsigned long flags, flags1; - - if (!context) - return; - - dev = cb_ctx->dev; - port_index = cb_ctx->port - 1; - rec = &dev->sriov.alias_guid.ports_guid[port_index]. - all_rec_per_port[cb_ctx->block_num]; - - if (status) { - rec->status = MLX4_GUID_INFO_STATUS_IDLE; - pr_debug("(port: %d) failed: status = %d\n", - cb_ctx->port, status); - goto out; - } - - if (guid_rec->block_num != cb_ctx->block_num) { - pr_err("block num mismatch: %d != %d\n", - cb_ctx->block_num, guid_rec->block_num); - goto out; - } - - pr_debug("lid/port: %d/%d, block_num: %d\n", - be16_to_cpu(guid_rec->lid), cb_ctx->port, - guid_rec->block_num); - - rec = &dev->sriov.alias_guid.ports_guid[port_index]. - all_rec_per_port[guid_rec->block_num]; - - rec->status = MLX4_GUID_INFO_STATUS_SET; - rec->method = MLX4_GUID_INFO_RECORD_SET; - - for (i = 0 ; i < NUM_ALIAS_GUID_IN_REC; i++) { - __be64 tmp_cur_ag; - tmp_cur_ag = *(__be64 *)&guid_rec->guid_info_list[i * GUID_REC_SIZE]; - if ((cb_ctx->method == MLX4_GUID_INFO_RECORD_DELETE) - && (MLX4_NOT_SET_GUID == tmp_cur_ag)) { - pr_debug("%s:Record num %d in block_num:%d " - "was deleted by SM,ownership by %d " - "(0 = driver, 1=sysAdmin, 2=None)\n", - __func__, i, guid_rec->block_num, - rec->ownership); - rec->guid_indexes = rec->guid_indexes & - ~mlx4_ib_get_aguid_comp_mask_from_ix(i); - continue; - } - - /* check if the SM didn't assign one of the records. - * if it didn't, if it was not sysadmin request: - * ask the SM to give a new GUID, (instead of the driver request). - */ - if (tmp_cur_ag == MLX4_NOT_SET_GUID) { - mlx4_ib_warn(&dev->ib_dev, "%s:Record num %d in " - "block_num: %d was declined by SM, " - "ownership by %d (0 = driver, 1=sysAdmin," - " 2=None)\n", __func__, i, - guid_rec->block_num, rec->ownership); - if (rec->ownership == MLX4_GUID_DRIVER_ASSIGN) { - /* if it is driver assign, asks for new GUID from SM*/ - *(__be64 *)&rec->all_recs[i * GUID_REC_SIZE] = - MLX4_NOT_SET_GUID; - - /* Mark the record as not assigned, and let it - * be sent again in the next work sched.*/ - rec->status = MLX4_GUID_INFO_STATUS_IDLE; - rec->guid_indexes |= mlx4_ib_get_aguid_comp_mask_from_ix(i); - } - } else { - /* properly assigned record. */ - /* We save the GUID we just got from the SM in the - * admin_guid in order to be persistent, and in the - * request from the sm the process will ask for the same GUID */ - if (rec->ownership == MLX4_GUID_SYSADMIN_ASSIGN && - tmp_cur_ag != *(__be64 *)&rec->all_recs[i * GUID_REC_SIZE]) { - /* the sysadmin assignment failed.*/ - mlx4_ib_warn(&dev->ib_dev, "%s: Failed to set" - " admin guid after SysAdmin " - "configuration. " - "Record num %d in block_num:%d " - "was declined by SM, " - "new val(0x%llx) was kept\n", - __func__, i, - guid_rec->block_num, - (long long)be64_to_cpu(*(__be64 *) & - rec->all_recs[i * GUID_REC_SIZE])); - } else { - memcpy(&rec->all_recs[i * GUID_REC_SIZE], - &guid_rec->guid_info_list[i * GUID_REC_SIZE], - GUID_REC_SIZE); - } - } - } - /* - The func is call here to close the cases when the - sm doesn't send smp, so in the sa response the driver - notifies the slave. - */ - mlx4_ib_notify_slaves_on_guid_change(dev, guid_rec->block_num, - cb_ctx->port, - guid_rec->guid_info_list); -out: - spin_lock_irqsave(&dev->sriov.going_down_lock, flags); - spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); - if (!dev->sriov.is_going_down) - queue_delayed_work(dev->sriov.alias_guid.ports_guid[port_index].wq, - &dev->sriov.alias_guid.ports_guid[port_index]. - alias_guid_work, 0); - if (cb_ctx->sa_query) { - list_del(&cb_ctx->list); - kfree(cb_ctx); - } else - complete(&cb_ctx->done); - spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); - spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); -} - -static void invalidate_guid_record(struct mlx4_ib_dev *dev, u8 port, int index) -{ - int i; - u64 cur_admin_val; - ib_sa_comp_mask comp_mask = 0; - - dev->sriov.alias_guid.ports_guid[port - 1].all_rec_per_port[index].status - = MLX4_GUID_INFO_STATUS_IDLE; - dev->sriov.alias_guid.ports_guid[port - 1].all_rec_per_port[index].method - = MLX4_GUID_INFO_RECORD_SET; - - /* calculate the comp_mask for that record.*/ - for (i = 0; i < NUM_ALIAS_GUID_IN_REC; i++) { - cur_admin_val = - *(u64 *)&dev->sriov.alias_guid.ports_guid[port - 1]. - all_rec_per_port[index].all_recs[GUID_REC_SIZE * i]; - /* - check the admin value: if it's for delete (~00LL) or - it is the first guid of the first record (hw guid) or - the records is not in ownership of the sysadmin and the sm doesn't - need to assign GUIDs, then don't put it up for assignment. - */ - if (MLX4_GUID_FOR_DELETE_VAL == cur_admin_val || - (!index && !i) || - MLX4_GUID_NONE_ASSIGN == dev->sriov.alias_guid. - ports_guid[port - 1].all_rec_per_port[index].ownership) - continue; - comp_mask |= mlx4_ib_get_aguid_comp_mask_from_ix(i); - } - dev->sriov.alias_guid.ports_guid[port - 1]. - all_rec_per_port[index].guid_indexes = comp_mask; -} - -static int set_guid_rec(struct ib_device *ibdev, - u8 port, int index, - struct mlx4_sriov_alias_guid_info_rec_det *rec_det) -{ - int err; - struct mlx4_ib_dev *dev = to_mdev(ibdev); - struct ib_sa_guidinfo_rec guid_info_rec; - ib_sa_comp_mask comp_mask; - struct ib_port_attr attr; - struct mlx4_alias_guid_work_context *callback_context; - unsigned long resched_delay, flags, flags1; - struct list_head *head = - &dev->sriov.alias_guid.ports_guid[port - 1].cb_list; - - err = __mlx4_ib_query_port(ibdev, port, &attr, 1); - if (err) { - pr_debug("mlx4_ib_query_port failed (err: %d), port: %d\n", - err, port); - return err; - } - /*check the port was configured by the sm, otherwise no need to send */ - if (attr.state != IB_PORT_ACTIVE) { - pr_debug("port %d not active...rescheduling\n", port); - resched_delay = 5 * HZ; - err = -EAGAIN; - goto new_schedule; - } - - callback_context = kmalloc(sizeof *callback_context, GFP_KERNEL); - if (!callback_context) { - err = -ENOMEM; - resched_delay = HZ * 5; - goto new_schedule; - } - callback_context->port = port; - callback_context->dev = dev; - callback_context->block_num = index; - callback_context->method = rec_det->method; - memset(&guid_info_rec, 0, sizeof (struct ib_sa_guidinfo_rec)); - - guid_info_rec.lid = cpu_to_be16(attr.lid); - guid_info_rec.block_num = index; - - memcpy(guid_info_rec.guid_info_list, rec_det->all_recs, - GUID_REC_SIZE * NUM_ALIAS_GUID_IN_REC); - comp_mask = IB_SA_GUIDINFO_REC_LID | IB_SA_GUIDINFO_REC_BLOCK_NUM | - rec_det->guid_indexes; - - init_completion(&callback_context->done); - spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); - list_add_tail(&callback_context->list, head); - spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); - - callback_context->query_id = - ib_sa_guid_info_rec_query(dev->sriov.alias_guid.sa_client, - ibdev, port, &guid_info_rec, - comp_mask, rec_det->method, 1000, - GFP_KERNEL, aliasguid_query_handler, - callback_context, - &callback_context->sa_query); - if (callback_context->query_id < 0) { - pr_debug("ib_sa_guid_info_rec_query failed, query_id: " - "%d. will reschedule to the next 1 sec.\n", - callback_context->query_id); - spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); - list_del(&callback_context->list); - kfree(callback_context); - spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); - resched_delay = 1 * HZ; - err = -EAGAIN; - goto new_schedule; - } - err = 0; - goto out; - -new_schedule: - spin_lock_irqsave(&dev->sriov.going_down_lock, flags); - spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); - invalidate_guid_record(dev, port, index); - if (!dev->sriov.is_going_down) { - queue_delayed_work(dev->sriov.alias_guid.ports_guid[port - 1].wq, - &dev->sriov.alias_guid.ports_guid[port - 1].alias_guid_work, - resched_delay); - } - spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); - spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); - -out: - return err; -} - -void mlx4_ib_invalidate_all_guid_record(struct mlx4_ib_dev *dev, int port) -{ - int i; - unsigned long flags, flags1; - - pr_debug("port %d\n", port); - - spin_lock_irqsave(&dev->sriov.going_down_lock, flags); - spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); - for (i = 0; i < NUM_ALIAS_GUID_REC_IN_PORT; i++) - invalidate_guid_record(dev, port, i); - - if (mlx4_is_master(dev->dev) && !dev->sriov.is_going_down) { - /* - make sure no work waits in the queue, if the work is already - queued(not on the timer) the cancel will fail. That is not a problem - because we just want the work started. - */ - cancel_delayed_work(&dev->sriov.alias_guid. - ports_guid[port - 1].alias_guid_work); - queue_delayed_work(dev->sriov.alias_guid.ports_guid[port - 1].wq, - &dev->sriov.alias_guid.ports_guid[port - 1].alias_guid_work, - 0); - } - spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); - spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); -} - -/* The function returns the next record that was - * not configured (or failed to be configured) */ -static int get_next_record_to_update(struct mlx4_ib_dev *dev, u8 port, - struct mlx4_next_alias_guid_work *rec) -{ - int j; - unsigned long flags; - - for (j = 0; j < NUM_ALIAS_GUID_REC_IN_PORT; j++) { - spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags); - if (dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[j].status == - MLX4_GUID_INFO_STATUS_IDLE) { - memcpy(&rec->rec_det, - &dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[j], - sizeof (struct mlx4_sriov_alias_guid_info_rec_det)); - rec->port = port; - rec->block_num = j; - dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[j].status = - MLX4_GUID_INFO_STATUS_PENDING; - spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags); - return 0; - } - spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags); - } - return -ENOENT; -} - -static void set_administratively_guid_record(struct mlx4_ib_dev *dev, int port, - int rec_index, - struct mlx4_sriov_alias_guid_info_rec_det *rec_det) -{ - dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[rec_index].guid_indexes = - rec_det->guid_indexes; - memcpy(dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[rec_index].all_recs, - rec_det->all_recs, NUM_ALIAS_GUID_IN_REC * GUID_REC_SIZE); - dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[rec_index].status = - rec_det->status; -} - -static void set_all_slaves_guids(struct mlx4_ib_dev *dev, int port) -{ - int j; - struct mlx4_sriov_alias_guid_info_rec_det rec_det ; - - for (j = 0 ; j < NUM_ALIAS_GUID_REC_IN_PORT ; j++) { - memset(rec_det.all_recs, 0, NUM_ALIAS_GUID_IN_REC * GUID_REC_SIZE); - rec_det.guid_indexes = (!j ? 0 : IB_SA_GUIDINFO_REC_GID0) | - IB_SA_GUIDINFO_REC_GID1 | IB_SA_GUIDINFO_REC_GID2 | - IB_SA_GUIDINFO_REC_GID3 | IB_SA_GUIDINFO_REC_GID4 | - IB_SA_GUIDINFO_REC_GID5 | IB_SA_GUIDINFO_REC_GID6 | - IB_SA_GUIDINFO_REC_GID7; - rec_det.status = MLX4_GUID_INFO_STATUS_IDLE; - set_administratively_guid_record(dev, port, j, &rec_det); - } -} - -static void alias_guid_work(struct work_struct *work) -{ - struct delayed_work *delay = to_delayed_work(work); - int ret = 0; - struct mlx4_next_alias_guid_work *rec; - struct mlx4_sriov_alias_guid_port_rec_det *sriov_alias_port = - container_of(delay, struct mlx4_sriov_alias_guid_port_rec_det, - alias_guid_work); - struct mlx4_sriov_alias_guid *sriov_alias_guid = sriov_alias_port->parent; - struct mlx4_ib_sriov *ib_sriov = container_of(sriov_alias_guid, - struct mlx4_ib_sriov, - alias_guid); - struct mlx4_ib_dev *dev = container_of(ib_sriov, struct mlx4_ib_dev, sriov); - - rec = kzalloc(sizeof *rec, GFP_KERNEL); - if (!rec) { - pr_err("alias_guid_work: No Memory\n"); - return; - } - - pr_debug("starting [port: %d]...\n", sriov_alias_port->port + 1); - ret = get_next_record_to_update(dev, sriov_alias_port->port, rec); - if (ret) { - pr_debug("No more records to update.\n"); - goto out; - } - - set_guid_rec(&dev->ib_dev, rec->port + 1, rec->block_num, - &rec->rec_det); - -out: - kfree(rec); -} - - -void mlx4_ib_init_alias_guid_work(struct mlx4_ib_dev *dev, int port) -{ - unsigned long flags, flags1; - - if (!mlx4_is_master(dev->dev)) - return; - spin_lock_irqsave(&dev->sriov.going_down_lock, flags); - spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); - if (!dev->sriov.is_going_down) { - queue_delayed_work(dev->sriov.alias_guid.ports_guid[port].wq, - &dev->sriov.alias_guid.ports_guid[port].alias_guid_work, 0); - } - spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); - spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); -} - -void mlx4_ib_destroy_alias_guid_service(struct mlx4_ib_dev *dev) -{ - int i; - struct mlx4_ib_sriov *sriov = &dev->sriov; - struct mlx4_alias_guid_work_context *cb_ctx; - struct mlx4_sriov_alias_guid_port_rec_det *det; - struct ib_sa_query *sa_query; - unsigned long flags; - - for (i = 0 ; i < dev->num_ports; i++) { - cancel_delayed_work(&dev->sriov.alias_guid.ports_guid[i].alias_guid_work); - det = &sriov->alias_guid.ports_guid[i]; - spin_lock_irqsave(&sriov->alias_guid.ag_work_lock, flags); - while (!list_empty(&det->cb_list)) { - cb_ctx = list_entry(det->cb_list.next, - struct mlx4_alias_guid_work_context, - list); - sa_query = cb_ctx->sa_query; - cb_ctx->sa_query = NULL; - list_del(&cb_ctx->list); - spin_unlock_irqrestore(&sriov->alias_guid.ag_work_lock, flags); - ib_sa_cancel_query(cb_ctx->query_id, sa_query); - wait_for_completion(&cb_ctx->done); - kfree(cb_ctx); - spin_lock_irqsave(&sriov->alias_guid.ag_work_lock, flags); - } - spin_unlock_irqrestore(&sriov->alias_guid.ag_work_lock, flags); - } - for (i = 0 ; i < dev->num_ports; i++) { - flush_workqueue(dev->sriov.alias_guid.ports_guid[i].wq); - destroy_workqueue(dev->sriov.alias_guid.ports_guid[i].wq); - } - ib_sa_unregister_client(dev->sriov.alias_guid.sa_client); - kfree(dev->sriov.alias_guid.sa_client); -} - -int mlx4_ib_init_alias_guid_service(struct mlx4_ib_dev *dev) -{ - char alias_wq_name[15]; - int ret = 0; - int i, j, k; - union ib_gid gid; - - if (!mlx4_is_master(dev->dev)) - return 0; - dev->sriov.alias_guid.sa_client = - kzalloc(sizeof *dev->sriov.alias_guid.sa_client, GFP_KERNEL); - if (!dev->sriov.alias_guid.sa_client) - return -ENOMEM; - - ib_sa_register_client(dev->sriov.alias_guid.sa_client); - - spin_lock_init(&dev->sriov.alias_guid.ag_work_lock); - - for (i = 1; i <= dev->num_ports; ++i) { - if (dev->ib_dev.query_gid(&dev->ib_dev , i, 0, &gid)) { - ret = -EFAULT; - goto err_unregister; - } - } - - for (i = 0 ; i < dev->num_ports; i++) { - memset(&dev->sriov.alias_guid.ports_guid[i], 0, - sizeof (struct mlx4_sriov_alias_guid_port_rec_det)); - /*Check if the SM doesn't need to assign the GUIDs*/ - for (j = 0; j < NUM_ALIAS_GUID_REC_IN_PORT; j++) { - if (mlx4_ib_sm_guid_assign) { - dev->sriov.alias_guid.ports_guid[i]. - all_rec_per_port[j]. - ownership = MLX4_GUID_DRIVER_ASSIGN; - continue; - } - dev->sriov.alias_guid.ports_guid[i].all_rec_per_port[j]. - ownership = MLX4_GUID_NONE_ASSIGN; - /*mark each val as it was deleted, - till the sysAdmin will give it valid val*/ - for (k = 0; k < NUM_ALIAS_GUID_IN_REC; k++) { - *(__be64 *)&dev->sriov.alias_guid.ports_guid[i]. - all_rec_per_port[j].all_recs[GUID_REC_SIZE * k] = - cpu_to_be64(MLX4_GUID_FOR_DELETE_VAL); - } - } - INIT_LIST_HEAD(&dev->sriov.alias_guid.ports_guid[i].cb_list); - /*prepare the records, set them to be allocated by sm*/ - for (j = 0 ; j < NUM_ALIAS_GUID_REC_IN_PORT; j++) - invalidate_guid_record(dev, i + 1, j); - - dev->sriov.alias_guid.ports_guid[i].parent = &dev->sriov.alias_guid; - dev->sriov.alias_guid.ports_guid[i].port = i; - if (mlx4_ib_sm_guid_assign) - set_all_slaves_guids(dev, i); - - snprintf(alias_wq_name, sizeof alias_wq_name, "alias_guid%d", i); - dev->sriov.alias_guid.ports_guid[i].wq = - create_singlethread_workqueue(alias_wq_name); - if (!dev->sriov.alias_guid.ports_guid[i].wq) { - ret = -ENOMEM; - goto err_thread; - } - INIT_DELAYED_WORK(&dev->sriov.alias_guid.ports_guid[i].alias_guid_work, - alias_guid_work); - } - return 0; - -err_thread: - for (--i; i >= 0; i--) { - destroy_workqueue(dev->sriov.alias_guid.ports_guid[i].wq); - dev->sriov.alias_guid.ports_guid[i].wq = NULL; - } - -err_unregister: - ib_sa_unregister_client(dev->sriov.alias_guid.sa_client); - kfree(dev->sriov.alias_guid.sa_client); - dev->sriov.alias_guid.sa_client = NULL; - pr_err("init_alias_guid_service: Failed. (ret:%d)\n", ret); - return ret; -} Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/alias_GUID.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/wc.h =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/wc.h (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/wc.h (nonexistent) @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2006-2007 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef mlx4_WC_H -#define mlx4_WC_H - -#include - -int mlx4_wc_enabled(void); -pgprot_t pgprot_wc(pgprot_t _prot); - -#endif Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/wc.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/wc.c =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/wc.c (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/wc.c (nonexistent) @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2006-2007 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include "wc.h" - -#if defined(__i386__) || defined(__x86_64__) - -pgprot_t pgprot_wc(pgprot_t _prot) -{ - return pgprot_writecombine(_prot); -} - -int mlx4_wc_enabled(void) -{ - return 1; -} - -#elif defined(CONFIG_PPC64) - -pgprot_t pgprot_wc(pgprot_t _prot) -{ - return __pgprot((pgprot_val(_prot) | _PAGE_NO_CACHE) & - ~(pgprot_t)_PAGE_GUARDED); -} - -int mlx4_wc_enabled(void) -{ - return 1; -} - -#else /* !(defined(__i386__) || defined(__x86_64__)) */ - -pgprot_t pgprot_wc(pgprot_t _prot) -{ - return pgprot_noncached(_prot); -} - -int mlx4_wc_enabled(void) -{ - return 0; -} - -#endif Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/wc.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/srq.c =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/srq.c (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/srq.c (nonexistent) @@ -1,368 +0,0 @@ -/* - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include - -#include "mlx4_ib.h" -#include "user.h" - -static void *get_wqe(struct mlx4_ib_srq *srq, int n) -{ - return mlx4_buf_offset(&srq->buf, n << srq->msrq.wqe_shift); -} - -static void mlx4_ib_srq_event(struct mlx4_srq *srq, enum mlx4_event type) -{ - struct ib_event event; - struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq; - - if (ibsrq->event_handler) { - event.device = ibsrq->device; - event.element.srq = ibsrq; - switch (type) { - case MLX4_EVENT_TYPE_SRQ_LIMIT: - event.event = IB_EVENT_SRQ_LIMIT_REACHED; - break; - case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR: - event.event = IB_EVENT_SRQ_ERR; - break; - default: - pr_warn("Unexpected event type %d " - "on SRQ %06x\n", type, srq->srqn); - return; - } - - ibsrq->event_handler(&event, ibsrq->srq_context); - } -} - -struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd, - struct ib_srq_init_attr *init_attr, - struct ib_udata *udata) -{ - struct mlx4_ib_dev *dev = to_mdev(pd->device); - struct mlx4_ib_srq *srq; - struct mlx4_wqe_srq_next_seg *next; - struct mlx4_wqe_data_seg *scatter; - u32 cqn; - u16 xrcdn; - int desc_size; - int buf_size; - int err; - int i; - - /* Sanity check SRQ size before proceeding */ - if (init_attr->attr.max_wr >= dev->dev->caps.max_srq_wqes || - init_attr->attr.max_sge > dev->dev->caps.max_srq_sge) - return ERR_PTR(-EINVAL); - - srq = kmalloc(sizeof *srq, GFP_KERNEL); - if (!srq) - return ERR_PTR(-ENOMEM); - - mutex_init(&srq->mutex); - spin_lock_init(&srq->lock); - srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); - srq->msrq.max_gs = init_attr->attr.max_sge; - - desc_size = max(32UL, - roundup_pow_of_two(sizeof (struct mlx4_wqe_srq_next_seg) + - srq->msrq.max_gs * - sizeof (struct mlx4_wqe_data_seg))); - srq->msrq.wqe_shift = ilog2(desc_size); - - buf_size = srq->msrq.max * desc_size; - - if (pd->uobject) { - struct mlx4_ib_create_srq ucmd; - - if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { - err = -EFAULT; - goto err_srq; - } - - srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, - buf_size, 0, 0); - if (IS_ERR(srq->umem)) { - err = PTR_ERR(srq->umem); - goto err_srq; - } - - err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem), - ilog2(srq->umem->page_size), &srq->mtt); - if (err) - goto err_buf; - - err = mlx4_ib_umem_write_mtt(dev, &srq->mtt, srq->umem); - if (err) - goto err_mtt; - - err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context), - ucmd.db_addr, &srq->db); - if (err) - goto err_mtt; - } else { - err = mlx4_db_alloc(dev->dev, &srq->db, 0); - if (err) - goto err_srq; - - *srq->db.db = 0; - - if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &srq->buf)) { - err = -ENOMEM; - goto err_db; - } - - srq->head = 0; - srq->tail = srq->msrq.max - 1; - srq->wqe_ctr = 0; - - for (i = 0; i < srq->msrq.max; ++i) { - next = get_wqe(srq, i); - next->next_wqe_index = - cpu_to_be16((i + 1) & (srq->msrq.max - 1)); - - for (scatter = (void *) (next + 1); - (void *) scatter < (void *) next + desc_size; - ++scatter) - scatter->lkey = cpu_to_be32(MLX4_INVALID_LKEY); - } - - err = mlx4_mtt_init(dev->dev, srq->buf.npages, srq->buf.page_shift, - &srq->mtt); - if (err) - goto err_buf; - - err = mlx4_buf_write_mtt(dev->dev, &srq->mtt, &srq->buf); - if (err) - goto err_mtt; - - srq->wrid = kmalloc(srq->msrq.max * sizeof (u64), GFP_KERNEL); - if (!srq->wrid) { - err = -ENOMEM; - goto err_mtt; - } - } - - cqn = (init_attr->srq_type == IB_SRQT_XRC) ? - to_mcq(init_attr->ext.xrc.cq)->mcq.cqn : 0; - xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? - to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn : - (u16) dev->dev->caps.reserved_xrcds; - err = mlx4_srq_alloc(dev->dev, to_mpd(pd)->pdn, cqn, xrcdn, &srq->mtt, - srq->db.dma, &srq->msrq); - if (err) - goto err_wrid; - - srq->msrq.event = mlx4_ib_srq_event; - srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; - - if (pd->uobject) - if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) { - err = -EFAULT; - goto err_wrid; - } - - init_attr->attr.max_wr = srq->msrq.max - 1; - - return &srq->ibsrq; - -err_wrid: - if (pd->uobject) - mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db); - else - kfree(srq->wrid); - -err_mtt: - mlx4_mtt_cleanup(dev->dev, &srq->mtt); - -err_buf: - if (pd->uobject) - ib_umem_release(srq->umem); - else - mlx4_buf_free(dev->dev, buf_size, &srq->buf); - -err_db: - if (!pd->uobject) - mlx4_db_free(dev->dev, &srq->db); - -err_srq: - kfree(srq); - - return ERR_PTR(err); -} - -int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, - enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) -{ - struct mlx4_ib_dev *dev = to_mdev(ibsrq->device); - struct mlx4_ib_srq *srq = to_msrq(ibsrq); - int ret; - - /* We don't support resizing SRQs (yet?) */ - if (attr_mask & IB_SRQ_MAX_WR) - return -EINVAL; - - if (attr_mask & IB_SRQ_LIMIT) { - if (attr->srq_limit >= srq->msrq.max) - return -EINVAL; - - mutex_lock(&srq->mutex); - ret = mlx4_srq_arm(dev->dev, &srq->msrq, attr->srq_limit); - mutex_unlock(&srq->mutex); - - if (ret) - return ret; - } - - return 0; -} - -int mlx4_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) -{ - struct mlx4_ib_dev *dev = to_mdev(ibsrq->device); - struct mlx4_ib_srq *srq = to_msrq(ibsrq); - int ret; - int limit_watermark; - - ret = mlx4_srq_query(dev->dev, &srq->msrq, &limit_watermark); - if (ret) - return ret; - - srq_attr->srq_limit = limit_watermark; - srq_attr->max_wr = srq->msrq.max - 1; - srq_attr->max_sge = srq->msrq.max_gs; - - return 0; -} - -int mlx4_ib_destroy_srq(struct ib_srq *srq) -{ - struct mlx4_ib_dev *dev = to_mdev(srq->device); - struct mlx4_ib_srq *msrq = to_msrq(srq); - - mlx4_srq_free(dev->dev, &msrq->msrq); - mlx4_mtt_cleanup(dev->dev, &msrq->mtt); - - if (srq->uobject) { - mlx4_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db); - ib_umem_release(msrq->umem); - } else { - kfree(msrq->wrid); - mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift, - &msrq->buf); - mlx4_db_free(dev->dev, &msrq->db); - } - - kfree(msrq); - - return 0; -} - -void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index) -{ - struct mlx4_wqe_srq_next_seg *next; - - /* always called with interrupts disabled. */ - spin_lock(&srq->lock); - - next = get_wqe(srq, srq->tail); - next->next_wqe_index = cpu_to_be16(wqe_index); - srq->tail = wqe_index; - - spin_unlock(&srq->lock); -} - -int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, - struct ib_recv_wr **bad_wr) -{ - struct mlx4_ib_srq *srq = to_msrq(ibsrq); - struct mlx4_wqe_srq_next_seg *next; - struct mlx4_wqe_data_seg *scat; - unsigned long flags; - int err = 0; - int nreq; - int i; - - spin_lock_irqsave(&srq->lock, flags); - - for (nreq = 0; wr; ++nreq, wr = wr->next) { - if (unlikely(wr->num_sge > srq->msrq.max_gs)) { - err = -EINVAL; - *bad_wr = wr; - break; - } - - if (unlikely(srq->head == srq->tail)) { - err = -ENOMEM; - *bad_wr = wr; - break; - } - - srq->wrid[srq->head] = wr->wr_id; - - next = get_wqe(srq, srq->head); - srq->head = be16_to_cpu(next->next_wqe_index); - scat = (struct mlx4_wqe_data_seg *) (next + 1); - - for (i = 0; i < wr->num_sge; ++i) { - scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length); - scat[i].lkey = cpu_to_be32(wr->sg_list[i].lkey); - scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); - } - - if (i < srq->msrq.max_gs) { - scat[i].byte_count = 0; - scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY); - scat[i].addr = 0; - } - } - - if (likely(nreq)) { - srq->wqe_ctr += nreq; - - /* - * Make sure that descriptors are written before - * doorbell record. - */ - wmb(); - - *srq->db.db = cpu_to_be32(srq->wqe_ctr); - } - - spin_unlock_irqrestore(&srq->lock, flags); - - return err; -} Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/srq.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/sysfs.c =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/sysfs.c (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/sysfs.c (nonexistent) @@ -1,801 +0,0 @@ -/* - * Copyright (c) 2012 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -/*#include "core_priv.h"*/ -#include "mlx4_ib.h" -#include -#include -#include - -#include -/*show_admin_alias_guid returns the administratively assigned value of that GUID. - * Values returned in buf parameter string: - * 0 - requests opensm to assign a value. - * ffffffffffffffff - delete this entry. - * other - value assigned by administrator. - */ -static ssize_t show_admin_alias_guid(struct device *dev, - struct device_attribute *attr, char *buf) -{ - int record_num;/*0-15*/ - int guid_index_in_rec; /*0 - 7*/ - struct mlx4_ib_iov_sysfs_attr *mlx4_ib_iov_dentry = - container_of(attr, struct mlx4_ib_iov_sysfs_attr, dentry); - struct mlx4_ib_iov_port *port = mlx4_ib_iov_dentry->ctx; - struct mlx4_ib_dev *mdev = port->dev; - - record_num = mlx4_ib_iov_dentry->entry_num / 8 ; - guid_index_in_rec = mlx4_ib_iov_dentry->entry_num % 8 ; - - return sprintf(buf, "%llx\n", - (long long)be64_to_cpu(*(__be64 *)&mdev->sriov.alias_guid. - ports_guid[port->num - 1]. - all_rec_per_port[record_num]. - all_recs[8 * guid_index_in_rec])); -} - -/* store_admin_alias_guid stores the (new) administratively assigned value of that GUID. - * Values in buf parameter string: - * 0 - requests opensm to assign a value. - * 0xffffffffffffffff - delete this entry. - * other - guid value assigned by the administrator. - */ -static ssize_t store_admin_alias_guid(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - int record_num;/*0-15*/ - int guid_index_in_rec; /*0 - 7*/ - struct mlx4_ib_iov_sysfs_attr *mlx4_ib_iov_dentry = - container_of(attr, struct mlx4_ib_iov_sysfs_attr, dentry); - struct mlx4_ib_iov_port *port = mlx4_ib_iov_dentry->ctx; - struct mlx4_ib_dev *mdev = port->dev; - unsigned long long sysadmin_ag_val; - - record_num = mlx4_ib_iov_dentry->entry_num / 8; - guid_index_in_rec = mlx4_ib_iov_dentry->entry_num % 8; - if (0 == record_num && 0 == guid_index_in_rec) { - pr_err("GUID 0 block 0 is RO\n"); - return count; - } - sscanf(buf, "%llx", &sysadmin_ag_val); - *(__be64 *)&mdev->sriov.alias_guid.ports_guid[port->num - 1]. - all_rec_per_port[record_num]. - all_recs[GUID_REC_SIZE * guid_index_in_rec] = - cpu_to_be64(sysadmin_ag_val); - - /* Change the state to be pending for update */ - mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].status - = MLX4_GUID_INFO_STATUS_IDLE ; - - mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].method - = MLX4_GUID_INFO_RECORD_SET; - - switch (sysadmin_ag_val) { - case MLX4_GUID_FOR_DELETE_VAL: - mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].method - = MLX4_GUID_INFO_RECORD_DELETE; - mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].ownership - = MLX4_GUID_SYSADMIN_ASSIGN; - break; - /* The sysadmin requests the SM to re-assign */ - case MLX4_NOT_SET_GUID: - mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].ownership - = MLX4_GUID_DRIVER_ASSIGN; - break; - /* The sysadmin requests a specific value.*/ - default: - mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].ownership - = MLX4_GUID_SYSADMIN_ASSIGN; - break; - } - - /* set the record index */ - mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].guid_indexes - = mlx4_ib_get_aguid_comp_mask_from_ix(guid_index_in_rec); - - mlx4_ib_init_alias_guid_work(mdev, port->num - 1); - - return count; -} - -static ssize_t show_port_gid(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct mlx4_ib_iov_sysfs_attr *mlx4_ib_iov_dentry = - container_of(attr, struct mlx4_ib_iov_sysfs_attr, dentry); - struct mlx4_ib_iov_port *port = mlx4_ib_iov_dentry->ctx; - struct mlx4_ib_dev *mdev = port->dev; - union ib_gid gid; - ssize_t ret; - - ret = __mlx4_ib_query_gid(&mdev->ib_dev, port->num, - mlx4_ib_iov_dentry->entry_num, &gid, 1); - if (ret) - return ret; - ret = sprintf(buf, "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n", - be16_to_cpu(((__be16 *) gid.raw)[0]), - be16_to_cpu(((__be16 *) gid.raw)[1]), - be16_to_cpu(((__be16 *) gid.raw)[2]), - be16_to_cpu(((__be16 *) gid.raw)[3]), - be16_to_cpu(((__be16 *) gid.raw)[4]), - be16_to_cpu(((__be16 *) gid.raw)[5]), - be16_to_cpu(((__be16 *) gid.raw)[6]), - be16_to_cpu(((__be16 *) gid.raw)[7])); - return ret; -} - -static ssize_t show_phys_port_pkey(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct mlx4_ib_iov_sysfs_attr *mlx4_ib_iov_dentry = - container_of(attr, struct mlx4_ib_iov_sysfs_attr, dentry); - struct mlx4_ib_iov_port *port = mlx4_ib_iov_dentry->ctx; - struct mlx4_ib_dev *mdev = port->dev; - u16 pkey; - ssize_t ret; - - ret = __mlx4_ib_query_pkey(&mdev->ib_dev, port->num, - mlx4_ib_iov_dentry->entry_num, &pkey, 1); - if (ret) - return ret; - - return sprintf(buf, "0x%04x\n", pkey); -} - -#define DENTRY_REMOVE(_dentry) \ -do { \ - sysfs_remove_file((_dentry)->kobj, &(_dentry)->dentry.attr); \ -} while (0); - -static int create_sysfs_entry(void *_ctx, struct mlx4_ib_iov_sysfs_attr *_dentry, - char *_name, struct kobject *_kobj, - ssize_t (*show)(struct device *dev, - struct device_attribute *attr, - char *buf), - ssize_t (*store)(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) - ) -{ - int ret = 0; - struct mlx4_ib_iov_sysfs_attr *vdentry = _dentry; - - vdentry->ctx = _ctx; - vdentry->dentry.show = show; - vdentry->dentry.store = store; - sysfs_attr_init(&vdentry->dentry.attr); - vdentry->dentry.attr.name = vdentry->name; - vdentry->dentry.attr.mode = 0; - vdentry->kobj = _kobj; - snprintf(vdentry->name, 15, "%s", _name); - - if (vdentry->dentry.store) - vdentry->dentry.attr.mode |= S_IWUSR; - - if (vdentry->dentry.show) - vdentry->dentry.attr.mode |= S_IRUGO; - - ret = sysfs_create_file(vdentry->kobj, &vdentry->dentry.attr); - if (ret) { - pr_err("failed to create %s\n", vdentry->dentry.attr.name); - vdentry->ctx = NULL; - return ret; - } - - return ret; -} - -int add_sysfs_port_mcg_attr(struct mlx4_ib_dev *device, int port_num, - struct attribute *attr) -{ - struct mlx4_ib_iov_port *port = &device->iov_ports[port_num - 1]; - int ret; - - ret = sysfs_create_file(port->mcgs_parent, attr); - if (ret) - pr_err("failed to create %s\n", attr->name); - - return ret; -} - -void del_sysfs_port_mcg_attr(struct mlx4_ib_dev *device, int port_num, - struct attribute *attr) -{ - struct mlx4_ib_iov_port *port = &device->iov_ports[port_num - 1]; - - sysfs_remove_file(port->mcgs_parent, attr); -} - -static int add_port_entries(struct mlx4_ib_dev *device, int port_num) -{ - int i; - char buff[10]; - struct mlx4_ib_iov_port *port = NULL; - int ret = 0 ; - struct ib_port_attr attr; - - /* get the physical gid and pkey table sizes.*/ - ret = __mlx4_ib_query_port(&device->ib_dev, port_num, &attr, 1); - if (ret) - goto err; - - port = &device->iov_ports[port_num - 1]; - port->dev = device; - port->num = port_num; - /* Directory structure: - * iov - - * port num - - * admin_guids - * gids (operational) - * mcg_table - */ - port->dentr_ar = kzalloc(sizeof (struct mlx4_ib_iov_sysfs_attr_ar), - GFP_KERNEL); - if (!port->dentr_ar) { - ret = -ENOMEM; - goto err; - } - sprintf(buff, "%d", port_num); - port->cur_port = kobject_create_and_add(buff, - kobject_get(device->ports_parent)); - if (!port->cur_port) { - ret = -ENOMEM; - goto kobj_create_err; - } - /* admin GUIDs */ - port->admin_alias_parent = kobject_create_and_add("admin_guids", - kobject_get(port->cur_port)); - if (!port->admin_alias_parent) { - ret = -ENOMEM; - goto err_admin_guids; - } - for (i = 0 ; i < attr.gid_tbl_len; i++) { - sprintf(buff, "%d", i); - port->dentr_ar->dentries[i].entry_num = i; - ret = create_sysfs_entry(port, &port->dentr_ar->dentries[i], - buff, port->admin_alias_parent, - show_admin_alias_guid, store_admin_alias_guid); - if (ret) - goto err_admin_alias_parent; - } - - /* gids subdirectory (operational gids) */ - port->gids_parent = kobject_create_and_add("gids", - kobject_get(port->cur_port)); - if (!port->gids_parent) { - ret = -ENOMEM; - goto err_gids; - } - - for (i = 0 ; i < attr.gid_tbl_len; i++) { - sprintf(buff, "%d", i); - port->dentr_ar->dentries[attr.gid_tbl_len + i].entry_num = i; - ret = create_sysfs_entry(port, - &port->dentr_ar->dentries[attr.gid_tbl_len + i], - buff, - port->gids_parent, show_port_gid, NULL); - if (ret) - goto err_gids_parent; - } - - /* physical port pkey table */ - port->pkeys_parent = - kobject_create_and_add("pkeys", kobject_get(port->cur_port)); - if (!port->pkeys_parent) { - ret = -ENOMEM; - goto err_pkeys; - } - - for (i = 0 ; i < attr.pkey_tbl_len; i++) { - sprintf(buff, "%d", i); - port->dentr_ar->dentries[2 * attr.gid_tbl_len + i].entry_num = i; - ret = create_sysfs_entry(port, - &port->dentr_ar->dentries[2 * attr.gid_tbl_len + i], - buff, port->pkeys_parent, - show_phys_port_pkey, NULL); - if (ret) - goto err_pkeys_parent; - } - - /* MCGs table */ - port->mcgs_parent = - kobject_create_and_add("mcgs", kobject_get(port->cur_port)); - if (!port->mcgs_parent) { - ret = -ENOMEM; - goto err_mcgs; - } - return 0; - -err_mcgs: - kobject_put(port->cur_port); - -err_pkeys_parent: - kobject_put(port->pkeys_parent); - -err_pkeys: - kobject_put(port->cur_port); - -err_gids_parent: - kobject_put(port->gids_parent); - -err_gids: - kobject_put(port->cur_port); - -err_admin_alias_parent: - kobject_put(port->admin_alias_parent); - -err_admin_guids: - kobject_put(port->cur_port); - kobject_put(port->cur_port); /* once more for create_and_add buff */ - -kobj_create_err: - kobject_put(device->ports_parent); - kfree(port->dentr_ar); - -err: - pr_err("add_port_entries FAILED: for port:%d, error: %d\n", - port_num, ret); - return ret; -} - -static void get_name(struct mlx4_ib_dev *dev, char *name, int i, int max) -{ - char base_name[9]; - - /* pci_name format is: bus:dev:func -> xxxx:yy:zz.n */ - strlcpy(name, pci_name(dev->dev->pdev), max); - strncpy(base_name, name, 8); /*till xxxx:yy:*/ - base_name[8] = '\0'; - /* with no ARI only 3 last bits are used so when the fn is higher than 8 - * need to add it to the dev num, so count in the last number will be - * modulo 8 */ - sprintf(name, "%s%.2d.%d", base_name, (i/8), (i%8)); -} - -struct mlx4_port { - struct kobject kobj; - struct mlx4_ib_dev *dev; - struct attribute_group pkey_group; - struct attribute_group gid_group; - u8 port_num; - int slave; -}; - - -static void mlx4_port_release(struct kobject *kobj) -{ - struct mlx4_port *p = container_of(kobj, struct mlx4_port, kobj); - struct attribute *a; - int i; - - for (i = 0; (a = p->pkey_group.attrs[i]); ++i) - kfree(a); - kfree(p->pkey_group.attrs); - for (i = 0; (a = p->gid_group.attrs[i]); ++i) - kfree(a); - kfree(p->gid_group.attrs); - kfree(p); -} - -struct port_attribute { - struct attribute attr; - ssize_t (*show)(struct mlx4_port *, struct port_attribute *, char *buf); - ssize_t (*store)(struct mlx4_port *, struct port_attribute *, - const char *buf, size_t count); -}; - -static ssize_t port_attr_show(struct kobject *kobj, - struct attribute *attr, char *buf) -{ - struct port_attribute *port_attr = - container_of(attr, struct port_attribute, attr); - struct mlx4_port *p = container_of(kobj, struct mlx4_port, kobj); - - if (!port_attr->show) - return -EIO; - return port_attr->show(p, port_attr, buf); -} - -static ssize_t port_attr_store(struct kobject *kobj, - struct attribute *attr, - const char *buf, size_t size) -{ - struct port_attribute *port_attr = - container_of(attr, struct port_attribute, attr); - struct mlx4_port *p = container_of(kobj, struct mlx4_port, kobj); - - if (!port_attr->store) - return -EIO; - return port_attr->store(p, port_attr, buf, size); -} - -static const struct sysfs_ops port_sysfs_ops = { - .show = port_attr_show, - .store = port_attr_store, -}; - -static struct kobj_type port_type = { - .release = mlx4_port_release, - .sysfs_ops = &port_sysfs_ops, -}; - -struct port_table_attribute { - struct port_attribute attr; - char name[8]; - int index; -}; - -static ssize_t show_port_pkey(struct mlx4_port *p, struct port_attribute *attr, - char *buf) -{ - struct port_table_attribute *tab_attr = - container_of(attr, struct port_table_attribute, attr); - ssize_t ret = -ENODEV; - - if (p->dev->pkeys.virt2phys_pkey[p->slave][p->port_num - 1][tab_attr->index] >= - (p->dev->dev->caps.pkey_table_len[p->port_num])) - ret = sprintf(buf, "none\n"); - else - ret = sprintf(buf, "%d\n", - p->dev->pkeys.virt2phys_pkey[p->slave] - [p->port_num - 1][tab_attr->index]); - return ret; -} - -static ssize_t store_port_pkey(struct mlx4_port *p, struct port_attribute *attr, - const char *buf, size_t count) -{ - struct port_table_attribute *tab_attr = - container_of(attr, struct port_table_attribute, attr); - int idx; - int err; - - /* do not allow remapping Dom0 virtual pkey table */ - if (p->slave == mlx4_master_func_num(p->dev->dev)) - return -EINVAL; - - if (!strncasecmp(buf, "no", 2)) - idx = p->dev->dev->phys_caps.pkey_phys_table_len[p->port_num] - 1; - else if (sscanf(buf, "%i", &idx) != 1 || - idx >= p->dev->dev->caps.pkey_table_len[p->port_num] || - idx < 0) - return -EINVAL; - - p->dev->pkeys.virt2phys_pkey[p->slave][p->port_num - 1] - [tab_attr->index] = idx; - mlx4_sync_pkey_table(p->dev->dev, p->slave, p->port_num, - tab_attr->index, idx); - err = mlx4_gen_pkey_eqe(p->dev->dev, p->slave, p->port_num); - if (err) { - pr_err("mlx4_gen_pkey_eqe failed for slave %d," - " port %d, index %d\n", p->slave, p->port_num, idx); - return err; - } - return count; -} - -static ssize_t show_port_gid_idx(struct mlx4_port *p, - struct port_attribute *attr, char *buf) -{ - return sprintf(buf, "%d\n", p->slave); -} - -static struct attribute ** -alloc_group_attrs(ssize_t (*show)(struct mlx4_port *, - struct port_attribute *, char *buf), - ssize_t (*store)(struct mlx4_port *, struct port_attribute *, - const char *buf, size_t count), - int len) -{ - struct attribute **tab_attr; - struct port_table_attribute *element; - int i; - - tab_attr = kcalloc(1 + len, sizeof (struct attribute *), GFP_KERNEL); - if (!tab_attr) - return NULL; - - for (i = 0; i < len; i++) { - element = kzalloc(sizeof (struct port_table_attribute), - GFP_KERNEL); - if (!element) - goto err; - if (snprintf(element->name, sizeof (element->name), - "%d", i) >= sizeof (element->name)) { - kfree(element); - goto err; - } - sysfs_attr_init(&element->attr.attr); - element->attr.attr.name = element->name; - if (store) { - element->attr.attr.mode = S_IWUSR | S_IRUGO; - element->attr.store = store; - } else - element->attr.attr.mode = S_IRUGO; - - element->attr.show = show; - element->index = i; - tab_attr[i] = &element->attr.attr; - } - return tab_attr; - -err: - while (--i >= 0) - kfree(tab_attr[i]); - kfree(tab_attr); - return NULL; -} - -static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave) -{ - struct mlx4_port *p; - int i; - int ret; - int is_eth = rdma_port_get_link_layer(&dev->ib_dev, port_num) == - IB_LINK_LAYER_ETHERNET; - - p = kzalloc(sizeof *p, GFP_KERNEL); - if (!p) - return -ENOMEM; - - p->dev = dev; - p->port_num = port_num; - p->slave = slave; - - ret = kobject_init_and_add(&p->kobj, &port_type, - kobject_get(dev->dev_ports_parent[slave]), - "%d", port_num); - if (ret) - goto err_alloc; - - p->pkey_group.name = "pkey_idx"; - if (is_eth) - p->pkey_group.attrs = - alloc_group_attrs(show_port_pkey, NULL, - dev->dev->caps.pkey_table_len[port_num]); - else - p->pkey_group.attrs = - alloc_group_attrs(show_port_pkey, store_port_pkey, - dev->dev->caps.pkey_table_len[port_num]); - if (!p->pkey_group.attrs) - goto err_alloc; - - ret = sysfs_create_group(&p->kobj, &p->pkey_group); - if (ret) - goto err_free_pkey; - - p->gid_group.name = "gid_idx"; - p->gid_group.attrs = alloc_group_attrs(show_port_gid_idx, NULL, 1); - if (!p->gid_group.attrs) - goto err_free_pkey; - - ret = sysfs_create_group(&p->kobj, &p->gid_group); - if (ret) - goto err_free_gid; - - list_add_tail(&p->kobj.entry, &dev->pkeys.pkey_port_list[slave]); - return 0; - -err_free_gid: - kfree(p->gid_group.attrs[0]); - kfree(p->gid_group.attrs); - -err_free_pkey: - for (i = 0; i < dev->dev->caps.pkey_table_len[port_num]; ++i) - kfree(p->pkey_group.attrs[i]); - kfree(p->pkey_group.attrs); - -err_alloc: - kobject_put(dev->dev_ports_parent[slave]); - kfree(p); - return ret; -} - -static int register_one_pkey_tree(struct mlx4_ib_dev *dev, int slave) -{ - char name[32]; - int err; - int port; - struct kobject *p, *t; - struct mlx4_port *mport; - - get_name(dev, name, slave, sizeof name); - - dev->pkeys.device_parent[slave] = - kobject_create_and_add(name, kobject_get(dev->iov_parent)); - - if (!dev->pkeys.device_parent[slave]) { - err = -ENOMEM; - goto fail_dev; - } - - INIT_LIST_HEAD(&dev->pkeys.pkey_port_list[slave]); - - dev->dev_ports_parent[slave] = - kobject_create_and_add("ports", - kobject_get(dev->pkeys.device_parent[slave])); - - if (!dev->dev_ports_parent[slave]) { - err = -ENOMEM; - goto err_ports; - } - - for (port = 1; port <= dev->dev->caps.num_ports; ++port) { - err = add_port(dev, port, slave); - if (err) - goto err_add; - } - return 0; - -err_add: - list_for_each_entry_safe(p, t, - &dev->pkeys.pkey_port_list[slave], - entry) { - list_del(&p->entry); - mport = container_of(p, struct mlx4_port, kobj); - sysfs_remove_group(p, &mport->pkey_group); - sysfs_remove_group(p, &mport->gid_group); - kobject_put(p); - } - kobject_put(dev->dev_ports_parent[slave]); - -err_ports: - kobject_put(dev->pkeys.device_parent[slave]); - /* extra put for the device_parent create_and_add */ - kobject_put(dev->pkeys.device_parent[slave]); - -fail_dev: - kobject_put(dev->iov_parent); - return err; -} - -static int register_pkey_tree(struct mlx4_ib_dev *device) -{ - int i; - - if (!mlx4_is_master(device->dev)) - return 0; - - for (i = 0; i <= device->dev->num_vfs; ++i) - register_one_pkey_tree(device, i); - - return 0; -} - -static void unregister_pkey_tree(struct mlx4_ib_dev *device) -{ - int slave; - struct kobject *p, *t; - struct mlx4_port *port; - - if (!mlx4_is_master(device->dev)) - return; - - for (slave = device->dev->num_vfs; slave >= 0; --slave) { - list_for_each_entry_safe(p, t, - &device->pkeys.pkey_port_list[slave], - entry) { - list_del(&p->entry); - port = container_of(p, struct mlx4_port, kobj); - sysfs_remove_group(p, &port->pkey_group); - sysfs_remove_group(p, &port->gid_group); - kobject_put(p); - kobject_put(device->dev_ports_parent[slave]); - } - kobject_put(device->dev_ports_parent[slave]); - kobject_put(device->pkeys.device_parent[slave]); - kobject_put(device->pkeys.device_parent[slave]); - kobject_put(device->iov_parent); - } -} - -int mlx4_ib_device_register_sysfs(struct mlx4_ib_dev *dev) -{ - int i; - int ret = 0; - - if (!mlx4_is_master(dev->dev)) - return 0; - - dev->iov_parent = - kobject_create_and_add("iov", - kobject_get(dev->ib_dev.ports_parent->parent)); - if (!dev->iov_parent) { - ret = -ENOMEM; - goto err; - } - dev->ports_parent = - kobject_create_and_add("ports", - kobject_get(dev->iov_parent)); - if (!dev->iov_parent) { - ret = -ENOMEM; - goto err_ports; - } - - for (i = 1; i <= dev->ib_dev.phys_port_cnt; ++i) { - ret = add_port_entries(dev, i); - if (ret) - goto err_add_entries; - } - - ret = register_pkey_tree(dev); - if (ret) - goto err_add_entries; - return 0; - -err_add_entries: - kobject_put(dev->ports_parent); - -err_ports: - kobject_put(dev->iov_parent); -err: - kobject_put(dev->ib_dev.ports_parent->parent); - pr_err("mlx4_ib_device_register_sysfs error (%d)\n", ret); - return ret; -} - -static void unregister_alias_guid_tree(struct mlx4_ib_dev *device) -{ - struct mlx4_ib_iov_port *p; - int i; - - if (!mlx4_is_master(device->dev)) - return; - - for (i = 0; i < device->dev->caps.num_ports; i++) { - p = &device->iov_ports[i]; - kobject_put(p->admin_alias_parent); - kobject_put(p->gids_parent); - kobject_put(p->pkeys_parent); - kobject_put(p->mcgs_parent); - kobject_put(p->cur_port); - kobject_put(p->cur_port); - kobject_put(p->cur_port); - kobject_put(p->cur_port); - kobject_put(p->cur_port); - kobject_put(p->dev->ports_parent); - kfree(p->dentr_ar); - } -} - -void mlx4_ib_device_unregister_sysfs(struct mlx4_ib_dev *device) -{ - unregister_alias_guid_tree(device); - unregister_pkey_tree(device); - kobject_put(device->ports_parent); - kobject_put(device->iov_parent); - kobject_put(device->iov_parent); - kobject_put(device->ib_dev.ports_parent->parent); -} Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/sysfs.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/main.c =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/main.c (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/main.c (nonexistent) @@ -1,2929 +0,0 @@ -/* - * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#define LINUXKPI_PARAM_PREFIX mlx4_ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include "mlx4_ib.h" -#include "mlx4_exp.h" -#include "user.h" -#include "wc.h" - -#define DRV_NAME MLX4_IB_DRV_NAME -#define DRV_VERSION "1.0" -#define DRV_RELDATE __DATE__ - -#define MLX4_IB_DRIVER_PROC_DIR_NAME "driver/mlx4_ib" -#define MLX4_IB_MRS_PROC_DIR_NAME "mrs" -#define MLX4_IB_FLOW_MAX_PRIO 0xFFF -#define MLX4_IB_FLOW_QPN_MASK 0xFFFFFF - -MODULE_AUTHOR("Roland Dreier"); -MODULE_DESCRIPTION("Mellanox ConnectX HCA InfiniBand driver"); -MODULE_LICENSE("Dual BSD/GPL"); -#ifdef __linux__ -MODULE_VERSION(DRV_VERSION); -#endif - -int mlx4_ib_sm_guid_assign = 1; - -module_param_named(sm_guid_assign, mlx4_ib_sm_guid_assign, int, 0444); -MODULE_PARM_DESC(sm_guid_assign, "Enable SM alias_GUID assignment if sm_guid_assign > 0 (Default: 1)"); - -enum { - MAX_NUM_STR_BITMAP = 1 << 15, - DEFAULT_TBL_VAL = -1 -}; - -static struct mlx4_dbdf2val_lst dev_assign_str = { - .name = "dev_assign_str param", - .num_vals = 1, - .def_val = {DEFAULT_TBL_VAL}, - .range = {0, MAX_NUM_STR_BITMAP - 1} -}; -module_param_string(dev_assign_str, dev_assign_str.str, - sizeof(dev_assign_str.str), 0444); -MODULE_PARM_DESC(dev_assign_str, - "Map device function numbers to IB device numbers (e.g. '0000:04:00.0-0,002b:1c:0b.a-1,...').\n" - "\t\tHexadecimal digits for the device function (e.g. 002b:1c:0b.a) and decimal for IB device numbers (e.g. 1).\n" - "\t\tMax supported devices - 32"); - - -static unsigned long *dev_num_str_bitmap; -static spinlock_t dev_num_str_lock; - -static const char mlx4_ib_version[] = - DRV_NAME ": Mellanox ConnectX InfiniBand driver v" - DRV_VERSION " (" DRV_RELDATE ")\n"; - -struct update_gid_work { - struct work_struct work; - union ib_gid gids[128]; - struct mlx4_ib_dev *dev; - int port; -}; - -struct dev_rec { - int bus; - int dev; - int func; - int nr; -}; - -static int dr_active; - -static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init); - -static void mlx4_ib_scan_netdevs(struct mlx4_ib_dev *ibdev, struct net_device*, - unsigned long); - -static u8 mlx4_ib_get_dev_port(struct net_device *dev, - struct mlx4_ib_dev *ibdev); - -static struct workqueue_struct *wq; - -static void init_query_mad(struct ib_smp *mad) -{ - mad->base_version = 1; - mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; - mad->class_version = 1; - mad->method = IB_MGMT_METHOD_GET; -} - -static union ib_gid zgid; - -static int check_flow_steering_support(struct mlx4_dev *dev) -{ - int eth_num_ports = 0; - int ib_num_ports = 0; - int dmfs = dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED; - - if (dmfs) { - int i; - mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) - eth_num_ports++; - mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB) - ib_num_ports++; - dmfs &= (!ib_num_ports || - (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_DMFS_IPOIB)) && - (!eth_num_ports || - (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FS_EN)); - if (ib_num_ports && mlx4_is_mfunc(dev)) { - dmfs = 0; - } - } - return dmfs; -} - -int mlx4_ib_query_device(struct ib_device *ibdev, - struct ib_device_attr *props) -{ - struct mlx4_ib_dev *dev = to_mdev(ibdev); - struct ib_smp *in_mad = NULL; - struct ib_smp *out_mad = NULL; - int err = -ENOMEM; - - in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); - out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); - if (!in_mad || !out_mad) - goto out; - - init_query_mad(in_mad); - in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; - - err = mlx4_MAD_IFC(to_mdev(ibdev), MLX4_MAD_IFC_IGNORE_KEYS, - 1, NULL, NULL, in_mad, out_mad); - if (err) - goto out; - - memset(props, 0, sizeof *props); - - props->fw_ver = dev->dev->caps.fw_ver; - props->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT | - IB_DEVICE_PORT_ACTIVE_EVENT | - IB_DEVICE_SYS_IMAGE_GUID | - IB_DEVICE_RC_RNR_NAK_GEN | - IB_DEVICE_BLOCK_MULTICAST_LOOPBACK | - IB_DEVICE_SHARED_MR; - - if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BAD_PKEY_CNTR) - props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR; - if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BAD_QKEY_CNTR) - props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR; - if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_APM) - props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG; - if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UD_AV_PORT) - props->device_cap_flags |= IB_DEVICE_UD_AV_PORT_ENFORCE; - if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_IPOIB_CSUM) - props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM; - if (dev->dev->caps.max_gso_sz && dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BLH) - props->device_cap_flags |= IB_DEVICE_UD_TSO; - if (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_RESERVED_LKEY) - props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY; - if ((dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_LOCAL_INV) && - (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_REMOTE_INV) && - (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_FAST_REG_WR)) - props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; - if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) - props->device_cap_flags |= IB_DEVICE_XRC; - if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_CROSS_CHANNEL) - props->device_cap_flags |= IB_DEVICE_CROSS_CHANNEL; - - if (check_flow_steering_support(dev->dev)) - props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING; - - - props->device_cap_flags |= IB_DEVICE_QPG; - if (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS) { - props->device_cap_flags |= IB_DEVICE_UD_RSS; - props->max_rss_tbl_sz = dev->dev->caps.max_rss_tbl_sz; - } - if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW) - props->device_cap_flags |= IB_DEVICE_MEM_WINDOW; - if (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN) { - if (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_WIN_TYPE_2B) - props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2B; - else - props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2A; - } - props->vendor_id = be32_to_cpup((__be32 *) (out_mad->data + 36)) & - 0xffffff; - props->vendor_part_id = dev->dev->pdev->device; - props->hw_ver = be32_to_cpup((__be32 *) (out_mad->data + 32)); - memcpy(&props->sys_image_guid, out_mad->data + 4, 8); - - props->max_mr_size = ~0ull; - props->page_size_cap = dev->dev->caps.page_size_cap; - props->max_qp = dev->dev->quotas.qp; - props->max_qp_wr = dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE; - props->max_sge = min(dev->dev->caps.max_sq_sg, - dev->dev->caps.max_rq_sg); - props->max_cq = dev->dev->quotas.cq; - props->max_cqe = dev->dev->caps.max_cqes; - props->max_mr = dev->dev->quotas.mpt; - props->max_pd = dev->dev->caps.num_pds - dev->dev->caps.reserved_pds; - props->max_qp_rd_atom = dev->dev->caps.max_qp_dest_rdma; - props->max_qp_init_rd_atom = dev->dev->caps.max_qp_init_rdma; - props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; - props->max_srq = dev->dev->quotas.srq; - props->max_srq_wr = dev->dev->caps.max_srq_wqes - 1; - props->max_srq_sge = dev->dev->caps.max_srq_sge; - props->max_fast_reg_page_list_len = MLX4_MAX_FAST_REG_PAGES; - props->local_ca_ack_delay = dev->dev->caps.local_ca_ack_delay; - props->atomic_cap = dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_ATOMIC ? - IB_ATOMIC_HCA : IB_ATOMIC_NONE; - props->masked_atomic_cap = props->atomic_cap; - props->max_pkeys = dev->dev->caps.pkey_table_len[1]; - props->max_mcast_grp = dev->dev->caps.num_mgms + dev->dev->caps.num_amgms; - props->max_mcast_qp_attach = dev->dev->caps.num_qp_per_mgm; - props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * - props->max_mcast_grp; - props->max_map_per_fmr = dev->dev->caps.max_fmr_maps; - props->hca_core_clock = dev->dev->caps.hca_core_clock; - if (dev->dev->caps.hca_core_clock > 0) - props->comp_mask |= IB_DEVICE_ATTR_WITH_HCA_CORE_CLOCK; - if (dev->dev->caps.cq_timestamp) { - props->timestamp_mask = 0xFFFFFFFFFFFF; - props->comp_mask |= IB_DEVICE_ATTR_WITH_TIMESTAMP_MASK; - } - -out: - kfree(in_mad); - kfree(out_mad); - - return err; -} - -static enum rdma_link_layer -mlx4_ib_port_link_layer(struct ib_device *device, u8 port_num) -{ - struct mlx4_dev *dev = to_mdev(device)->dev; - - return dev->caps.port_mask[port_num] == MLX4_PORT_TYPE_IB ? - IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET; -} - -static int ib_link_query_port(struct ib_device *ibdev, u8 port, - struct ib_port_attr *props, int netw_view) -{ - struct ib_smp *in_mad = NULL; - struct ib_smp *out_mad = NULL; - int ext_active_speed; - int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS; - int err = -ENOMEM; - - in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); - out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); - if (!in_mad || !out_mad) - goto out; - - init_query_mad(in_mad); - in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; - in_mad->attr_mod = cpu_to_be32(port); - - if (mlx4_is_mfunc(to_mdev(ibdev)->dev) && netw_view) - mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW; - - err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, NULL, NULL, - in_mad, out_mad); - if (err) - goto out; - - - props->lid = be16_to_cpup((__be16 *) (out_mad->data + 16)); - props->lmc = out_mad->data[34] & 0x7; - props->sm_lid = be16_to_cpup((__be16 *) (out_mad->data + 18)); - props->sm_sl = out_mad->data[36] & 0xf; - props->state = out_mad->data[32] & 0xf; - props->phys_state = out_mad->data[33] >> 4; - props->port_cap_flags = be32_to_cpup((__be32 *) (out_mad->data + 20)); - if (netw_view) - props->gid_tbl_len = out_mad->data[50]; - else - props->gid_tbl_len = to_mdev(ibdev)->dev->caps.gid_table_len[port]; - props->max_msg_sz = to_mdev(ibdev)->dev->caps.max_msg_sz; - props->pkey_tbl_len = to_mdev(ibdev)->dev->caps.pkey_table_len[port]; - props->bad_pkey_cntr = be16_to_cpup((__be16 *) (out_mad->data + 46)); - props->qkey_viol_cntr = be16_to_cpup((__be16 *) (out_mad->data + 48)); - props->active_width = out_mad->data[31] & 0xf; - props->active_speed = out_mad->data[35] >> 4; - props->max_mtu = out_mad->data[41] & 0xf; - props->active_mtu = out_mad->data[36] >> 4; - props->subnet_timeout = out_mad->data[51] & 0x1f; - props->max_vl_num = out_mad->data[37] >> 4; - props->init_type_reply = out_mad->data[41] >> 4; - - /* Check if extended speeds (EDR/FDR/...) are supported */ - if (props->port_cap_flags & IB_PORT_EXTENDED_SPEEDS_SUP) { - ext_active_speed = out_mad->data[62] >> 4; - - switch (ext_active_speed) { - case 1: - props->active_speed = IB_SPEED_FDR; - break; - case 2: - props->active_speed = IB_SPEED_EDR; - break; - } - } - - /* If reported active speed is QDR, check if is FDR-10 */ - if (props->active_speed == IB_SPEED_QDR) { - init_query_mad(in_mad); - in_mad->attr_id = MLX4_ATTR_EXTENDED_PORT_INFO; - in_mad->attr_mod = cpu_to_be32(port); - - err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, - NULL, NULL, in_mad, out_mad); - if (err) - goto out; - - /* Checking LinkSpeedActive for FDR-10 */ - if (out_mad->data[15] & 0x1) - props->active_speed = IB_SPEED_FDR10; - } - - /* Avoid wrong speed value returned by FW if the IB link is down. */ - if (props->state == IB_PORT_DOWN) - props->active_speed = IB_SPEED_SDR; - -out: - kfree(in_mad); - kfree(out_mad); - return err; -} - -static u8 state_to_phys_state(enum ib_port_state state) -{ - return state == IB_PORT_ACTIVE ? 5 : 3; -} - -static int eth_link_query_port(struct ib_device *ibdev, u8 port, - struct ib_port_attr *props, int netw_view) -{ - - struct mlx4_ib_dev *mdev = to_mdev(ibdev); - struct mlx4_ib_iboe *iboe = &mdev->iboe; - struct net_device *ndev; - enum ib_mtu tmp; - struct mlx4_cmd_mailbox *mailbox; - unsigned long flags; - int err = 0; - - mailbox = mlx4_alloc_cmd_mailbox(mdev->dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - err = mlx4_cmd_box(mdev->dev, 0, mailbox->dma, port, 0, - MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_WRAPPED); - if (err) - goto out; - - props->active_width = (((u8 *)mailbox->buf)[5] == 0x40) ? - IB_WIDTH_4X : IB_WIDTH_1X; - props->active_speed = IB_SPEED_QDR; - props->port_cap_flags = IB_PORT_CM_SUP; - if (netw_view) - props->gid_tbl_len = MLX4_ROCE_MAX_GIDS; - else - props->gid_tbl_len = mdev->dev->caps.gid_table_len[port]; - - props->max_msg_sz = mdev->dev->caps.max_msg_sz; - props->pkey_tbl_len = 1; - props->max_mtu = IB_MTU_4096; - props->max_vl_num = 2; - props->state = IB_PORT_DOWN; - props->phys_state = state_to_phys_state(props->state); - props->active_mtu = IB_MTU_256; - spin_lock_irqsave(&iboe->lock, flags); - ndev = iboe->netdevs[port - 1]; - if (!ndev) - goto out_unlock; - - tmp = iboe_get_mtu(ndev->if_mtu); - props->active_mtu = tmp ? min(props->max_mtu, tmp) : IB_MTU_256; - - props->state = (netif_running(ndev) && netif_carrier_ok(ndev)) ? - IB_PORT_ACTIVE : IB_PORT_DOWN; - props->phys_state = state_to_phys_state(props->state); -out_unlock: - spin_unlock_irqrestore(&iboe->lock, flags); -out: - mlx4_free_cmd_mailbox(mdev->dev, mailbox); - return err; -} - -int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port, - struct ib_port_attr *props, int netw_view) -{ - int err; - - memset(props, 0, sizeof *props); - - err = mlx4_ib_port_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND ? - ib_link_query_port(ibdev, port, props, netw_view) : - eth_link_query_port(ibdev, port, props, netw_view); - - return err; -} - -static int mlx4_ib_query_port(struct ib_device *ibdev, u8 port, - struct ib_port_attr *props) -{ - /* returns host view */ - return __mlx4_ib_query_port(ibdev, port, props, 0); -} - -int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index, - union ib_gid *gid, int netw_view) -{ - struct ib_smp *in_mad = NULL; - struct ib_smp *out_mad = NULL; - int err = -ENOMEM; - struct mlx4_ib_dev *dev = to_mdev(ibdev); - int clear = 0; - int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS; - - in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); - out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); - if (!in_mad || !out_mad) - goto out; - - init_query_mad(in_mad); - in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; - in_mad->attr_mod = cpu_to_be32(port); - - if (mlx4_is_mfunc(dev->dev) && netw_view) - mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW; - - err = mlx4_MAD_IFC(dev, mad_ifc_flags, port, NULL, NULL, in_mad, out_mad); - if (err) - goto out; - - memcpy(gid->raw, out_mad->data + 8, 8); - - if (mlx4_is_mfunc(dev->dev) && !netw_view) { - if (index) { - /* For any index > 0, return the null guid */ - err = 0; - clear = 1; - goto out; - } - } - - init_query_mad(in_mad); - in_mad->attr_id = IB_SMP_ATTR_GUID_INFO; - in_mad->attr_mod = cpu_to_be32(index / 8); - - err = mlx4_MAD_IFC(dev, mad_ifc_flags, port, - NULL, NULL, in_mad, out_mad); - if (err) - goto out; - - memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8); - -out: - if (clear) - memset(gid->raw + 8, 0, 8); - kfree(in_mad); - kfree(out_mad); - return err; -} - -static int iboe_query_gid(struct ib_device *ibdev, u8 port, int index, - union ib_gid *gid) -{ - struct mlx4_ib_dev *dev = to_mdev(ibdev); - - *gid = dev->iboe.gid_table[port - 1][index]; - - return 0; -} - -static int mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index, - union ib_gid *gid) -{ - if (rdma_port_get_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND) - return __mlx4_ib_query_gid(ibdev, port, index, gid, 0); - else - return iboe_query_gid(ibdev, port, index, gid); -} - -int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, - u16 *pkey, int netw_view) -{ - struct ib_smp *in_mad = NULL; - struct ib_smp *out_mad = NULL; - int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS; - int err = -ENOMEM; - - in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); - out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); - if (!in_mad || !out_mad) - goto out; - - init_query_mad(in_mad); - in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE; - in_mad->attr_mod = cpu_to_be32(index / 32); - - if (mlx4_is_mfunc(to_mdev(ibdev)->dev) && netw_view) - mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW; - - err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, NULL, NULL, - in_mad, out_mad); - if (err) - goto out; - - *pkey = be16_to_cpu(((__be16 *) out_mad->data)[index % 32]); - -out: - kfree(in_mad); - kfree(out_mad); - return err; -} - -static int mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) -{ - return __mlx4_ib_query_pkey(ibdev, port, index, pkey, 0); -} - -static int mlx4_ib_modify_device(struct ib_device *ibdev, int mask, - struct ib_device_modify *props) -{ - struct mlx4_cmd_mailbox *mailbox; - unsigned long flags; - - if (mask & ~IB_DEVICE_MODIFY_NODE_DESC) - return -EOPNOTSUPP; - - if (!(mask & IB_DEVICE_MODIFY_NODE_DESC)) - return 0; - - if (mlx4_is_slave(to_mdev(ibdev)->dev)) - return -EOPNOTSUPP; - - spin_lock_irqsave(&to_mdev(ibdev)->sm_lock, flags); - memcpy(ibdev->node_desc, props->node_desc, 64); - spin_unlock_irqrestore(&to_mdev(ibdev)->sm_lock, flags); - - /* - * If possible, pass node desc to FW, so it can generate - * a 144 trap. If cmd fails, just ignore. - */ - mailbox = mlx4_alloc_cmd_mailbox(to_mdev(ibdev)->dev); - if (IS_ERR(mailbox)) - return 0; - - memset(mailbox->buf, 0, 256); - memcpy(mailbox->buf, props->node_desc, 64); - mlx4_cmd(to_mdev(ibdev)->dev, mailbox->dma, 1, 0, - MLX4_CMD_SET_NODE, MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); - - mlx4_free_cmd_mailbox(to_mdev(ibdev)->dev, mailbox); - - return 0; -} - -static int mlx4_SET_PORT(struct mlx4_ib_dev *dev, u8 port, int reset_qkey_viols, - u32 cap_mask) -{ - struct mlx4_cmd_mailbox *mailbox; - int err; - u8 is_eth = dev->dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH; - - mailbox = mlx4_alloc_cmd_mailbox(dev->dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - memset(mailbox->buf, 0, 256); - - if (dev->dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { - *(u8 *) mailbox->buf = !!reset_qkey_viols << 6; - ((__be32 *) mailbox->buf)[2] = cpu_to_be32(cap_mask); - } else { - ((u8 *) mailbox->buf)[3] = !!reset_qkey_viols; - ((__be32 *) mailbox->buf)[1] = cpu_to_be32(cap_mask); - } - - err = mlx4_cmd(dev->dev, mailbox->dma, port, is_eth, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); - - mlx4_free_cmd_mailbox(dev->dev, mailbox); - return err; -} - -static int mlx4_ib_modify_port(struct ib_device *ibdev, u8 port, int mask, - struct ib_port_modify *props) -{ - struct ib_port_attr attr; - u32 cap_mask; - int err; - - mutex_lock(&to_mdev(ibdev)->cap_mask_mutex); - - err = mlx4_ib_query_port(ibdev, port, &attr); - if (err) - goto out; - - cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) & - ~props->clr_port_cap_mask; - - err = mlx4_SET_PORT(to_mdev(ibdev), port, - !!(mask & IB_PORT_RESET_QKEY_CNTR), - cap_mask); - -out: - mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex); - return err; -} - -static struct ib_ucontext *mlx4_ib_alloc_ucontext(struct ib_device *ibdev, - struct ib_udata *udata) -{ - struct mlx4_ib_dev *dev = to_mdev(ibdev); - struct mlx4_ib_ucontext *context; - struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3; - struct mlx4_ib_alloc_ucontext_resp resp; - int err; - - if (!dev->ib_active) - return ERR_PTR(-EAGAIN); - - if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) { - resp_v3.qp_tab_size = dev->dev->caps.num_qps; - if (mlx4_wc_enabled()) { - resp_v3.bf_reg_size = dev->dev->caps.bf_reg_size; - resp_v3.bf_regs_per_page = dev->dev->caps.bf_regs_per_page; - } else { - resp_v3.bf_reg_size = 0; - resp_v3.bf_regs_per_page = 0; - } - } else { - resp.dev_caps = dev->dev->caps.userspace_caps; - resp.qp_tab_size = dev->dev->caps.num_qps; - if (mlx4_wc_enabled()) { - resp.bf_reg_size = dev->dev->caps.bf_reg_size; - resp.bf_regs_per_page = dev->dev->caps.bf_regs_per_page; - } else { - resp.bf_reg_size = 0; - resp.bf_regs_per_page = 0; - } - resp.cqe_size = dev->dev->caps.cqe_size; - } - - context = kmalloc(sizeof *context, GFP_KERNEL); - if (!context) - return ERR_PTR(-ENOMEM); - - err = mlx4_uar_alloc(to_mdev(ibdev)->dev, &context->uar); - if (err) { - kfree(context); - return ERR_PTR(err); - } - - INIT_LIST_HEAD(&context->db_page_list); - mutex_init(&context->db_page_mutex); - - if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) - err = ib_copy_to_udata(udata, &resp_v3, sizeof(resp_v3)); - else - err = ib_copy_to_udata(udata, &resp, sizeof(resp)); - - if (err) { - mlx4_uar_free(to_mdev(ibdev)->dev, &context->uar); - kfree(context); - return ERR_PTR(-EFAULT); - } - - return &context->ibucontext; -} - -static int mlx4_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) -{ - struct mlx4_ib_ucontext *context = to_mucontext(ibcontext); - - mlx4_uar_free(to_mdev(ibcontext->device)->dev, &context->uar); - kfree(context); - - return 0; -} - -/* XXX FBSD has no support for get_unmapped_area function */ -#if 0 -static unsigned long mlx4_ib_get_unmapped_area(struct file *file, - unsigned long addr, - unsigned long len, unsigned long pgoff, - unsigned long flags) -{ - struct mm_struct *mm; - struct vm_area_struct *vma; - unsigned long start_addr; - unsigned long page_size_order; - unsigned long command; - - mm = current->mm; - if (addr) - return current->mm->get_unmapped_area(file, addr, len, - pgoff, flags); - - /* Last 8 bits hold the command others are data per that command */ - command = pgoff & MLX4_IB_MMAP_CMD_MASK; - if (command != MLX4_IB_MMAP_GET_CONTIGUOUS_PAGES) - return current->mm->get_unmapped_area(file, addr, len, - pgoff, flags); - - page_size_order = pgoff >> MLX4_IB_MMAP_CMD_BITS; - /* code is based on the huge-pages get_unmapped_area code */ - start_addr = mm->free_area_cache; - - if (len <= mm->cached_hole_size) - start_addr = TASK_UNMAPPED_BASE; - - -full_search: - addr = ALIGN(start_addr, 1 << page_size_order); - - for (vma = find_vma(mm, addr); ; vma = vma->vm_next) { - /* At this point: (!vma || addr < vma->vm_end). */ - if (TASK_SIZE - len < addr) { - /* - * Start a new search - just in case we missed - * some holes. - */ - if (start_addr != TASK_UNMAPPED_BASE) { - start_addr = TASK_UNMAPPED_BASE; - goto full_search; - } - return -ENOMEM; - } - - if (!vma || addr + len <= vma->vm_start) - return addr; - addr = ALIGN(vma->vm_end, 1 << page_size_order); - } -} -#endif - -static int mlx4_ib_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) -{ - struct mlx4_ib_dev *dev = to_mdev(context->device); - - /* Last 8 bits hold the command others are data per that command */ - unsigned long command = vma->vm_pgoff & MLX4_IB_MMAP_CMD_MASK; - - if (command < MLX4_IB_MMAP_GET_CONTIGUOUS_PAGES) { - /* compatibility handling for commands 0 & 1*/ - if (vma->vm_end - vma->vm_start != PAGE_SIZE) - return -EINVAL; - } - if (command == MLX4_IB_MMAP_UAR_PAGE) { - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - - if (io_remap_pfn_range(vma, vma->vm_start, - to_mucontext(context)->uar.pfn, - PAGE_SIZE, vma->vm_page_prot)) - return -EAGAIN; - } else if (command == MLX4_IB_MMAP_BLUE_FLAME_PAGE && - dev->dev->caps.bf_reg_size != 0) { - vma->vm_page_prot = pgprot_wc(vma->vm_page_prot); - - if (io_remap_pfn_range(vma, vma->vm_start, - to_mucontext(context)->uar.pfn + - dev->dev->caps.num_uars, - PAGE_SIZE, vma->vm_page_prot)) - return -EAGAIN; - } else if (command == MLX4_IB_MMAP_GET_HW_CLOCK) { - struct mlx4_clock_params params; - int ret; - - ret = mlx4_get_internal_clock_params(dev->dev, ¶ms); - if (ret) - return ret; - - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - - if (io_remap_pfn_range(vma, vma->vm_start, - (pci_resource_start(dev->dev->pdev, - params.bar) + params.offset) - >> PAGE_SHIFT, - PAGE_SIZE, vma->vm_page_prot)) - return -EAGAIN; - } else - return -EINVAL; - - return 0; -} - -static int mlx4_ib_ioctl(struct ib_ucontext *context, unsigned int cmd, - unsigned long arg) -{ - struct mlx4_ib_dev *dev = to_mdev(context->device); - int ret; - int offset; - - switch (cmd) { - case MLX4_IOCHWCLOCKOFFSET: { - struct mlx4_clock_params params; - int ret; - ret = mlx4_get_internal_clock_params(dev->dev, ¶ms); - if (!ret) { - offset = params.offset % PAGE_SIZE; - ret = put_user(offset, - (int *)arg); - return sizeof(int); - } else { - return ret; - } - } - default: { - pr_err("mlx4_ib: invalid ioctl %u command with arg %lX\n", - cmd, arg); - return -ENOTTY; - } - } - - return ret; -} - -static int mlx4_ib_query_values(struct ib_device *device, int q_values, - struct ib_device_values *values) -{ - struct mlx4_ib_dev *dev = to_mdev(device); - s64 cycles; - - values->values_mask = 0; - if (q_values & IBV_VALUES_HW_CLOCK) { - cycles = mlx4_read_clock(dev->dev); - if (cycles >= 0) { - values->hwclock = cycles & CORE_CLOCK_MASK; - values->values_mask |= IBV_VALUES_HW_CLOCK; - } - q_values &= ~IBV_VALUES_HW_CLOCK; - } - - if (q_values) - return -ENOTTY; - - return 0; -} - -static struct ib_pd *mlx4_ib_alloc_pd(struct ib_device *ibdev, - struct ib_ucontext *context, - struct ib_udata *udata) -{ - struct mlx4_ib_pd *pd; - int err; - - pd = kmalloc(sizeof *pd, GFP_KERNEL); - if (!pd) - return ERR_PTR(-ENOMEM); - - err = mlx4_pd_alloc(to_mdev(ibdev)->dev, &pd->pdn); - if (err) { - kfree(pd); - return ERR_PTR(err); - } - - if (context) - if (ib_copy_to_udata(udata, &pd->pdn, sizeof (__u32))) { - mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn); - kfree(pd); - return ERR_PTR(-EFAULT); - } - - return &pd->ibpd; -} - -static int mlx4_ib_dealloc_pd(struct ib_pd *pd) -{ - mlx4_pd_free(to_mdev(pd->device)->dev, to_mpd(pd)->pdn); - kfree(pd); - - return 0; -} - -static struct ib_xrcd *mlx4_ib_alloc_xrcd(struct ib_device *ibdev, - struct ib_ucontext *context, - struct ib_udata *udata) -{ - struct mlx4_ib_xrcd *xrcd; - int err; - - if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)) - return ERR_PTR(-ENOSYS); - - xrcd = kmalloc(sizeof *xrcd, GFP_KERNEL); - if (!xrcd) - return ERR_PTR(-ENOMEM); - - err = mlx4_xrcd_alloc(to_mdev(ibdev)->dev, &xrcd->xrcdn); - if (err) - goto err1; - - xrcd->pd = ib_alloc_pd(ibdev); - if (IS_ERR(xrcd->pd)) { - err = PTR_ERR(xrcd->pd); - goto err2; - } - - xrcd->cq = ib_create_cq(ibdev, NULL, NULL, xrcd, 1, 0); - if (IS_ERR(xrcd->cq)) { - err = PTR_ERR(xrcd->cq); - goto err3; - } - - return &xrcd->ibxrcd; - -err3: - ib_dealloc_pd(xrcd->pd); -err2: - mlx4_xrcd_free(to_mdev(ibdev)->dev, xrcd->xrcdn); -err1: - kfree(xrcd); - return ERR_PTR(err); -} - -static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd) -{ - ib_destroy_cq(to_mxrcd(xrcd)->cq); - ib_dealloc_pd(to_mxrcd(xrcd)->pd); - mlx4_xrcd_free(to_mdev(xrcd->device)->dev, to_mxrcd(xrcd)->xrcdn); - kfree(xrcd); - - return 0; -} - -static int add_gid_entry(struct ib_qp *ibqp, union ib_gid *gid) -{ - struct mlx4_ib_qp *mqp = to_mqp(ibqp); - struct mlx4_ib_dev *mdev = to_mdev(ibqp->device); - struct mlx4_ib_gid_entry *ge; - - ge = kzalloc(sizeof *ge, GFP_KERNEL); - if (!ge) - return -ENOMEM; - - ge->gid = *gid; - if (mlx4_ib_add_mc(mdev, mqp, gid)) { - ge->port = mqp->port; - ge->added = 1; - } - - mutex_lock(&mqp->mutex); - list_add_tail(&ge->list, &mqp->gid_list); - mutex_unlock(&mqp->mutex); - - return 0; -} - -int mlx4_ib_add_mc(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp, - union ib_gid *gid) -{ - u8 mac[6]; - struct net_device *ndev; - int ret = 0; - - if (!mqp->port) - return 0; - - spin_lock(&mdev->iboe.lock); - ndev = mdev->iboe.netdevs[mqp->port - 1]; - if (ndev) - dev_hold(ndev); - spin_unlock(&mdev->iboe.lock); - - if (ndev) { - rdma_get_mcast_mac((struct in6_addr *)gid, mac); - rtnl_lock(); - dev_mc_add(mdev->iboe.netdevs[mqp->port - 1], mac, 6, 0); - ret = 1; - rtnl_unlock(); - dev_put(ndev); - } - - return ret; -} - -struct mlx4_ib_steering { - struct list_head list; - u64 reg_id; - union ib_gid gid; -}; - -static int parse_flow_attr(struct mlx4_dev *dev, - union ib_flow_spec *ib_spec, - struct _rule_hw *mlx4_spec) -{ - enum mlx4_net_trans_rule_id type; - - switch (ib_spec->type) { - case IB_FLOW_SPEC_ETH: - type = MLX4_NET_TRANS_RULE_ID_ETH; - memcpy(mlx4_spec->eth.dst_mac, ib_spec->eth.val.dst_mac, - ETH_ALEN); - memcpy(mlx4_spec->eth.dst_mac_msk, ib_spec->eth.mask.dst_mac, - ETH_ALEN); - mlx4_spec->eth.vlan_tag = ib_spec->eth.val.vlan_tag; - mlx4_spec->eth.vlan_tag_msk = ib_spec->eth.mask.vlan_tag; - break; - - case IB_FLOW_SPEC_IB: - type = MLX4_NET_TRANS_RULE_ID_IB; - mlx4_spec->ib.l3_qpn = ib_spec->ib.val.l3_type_qpn; - mlx4_spec->ib.qpn_mask = ib_spec->ib.mask.l3_type_qpn; - memcpy(&mlx4_spec->ib.dst_gid, ib_spec->ib.val.dst_gid, 16); - memcpy(&mlx4_spec->ib.dst_gid_msk, - ib_spec->ib.mask.dst_gid, 16); - break; - - case IB_FLOW_SPEC_IPV4: - type = MLX4_NET_TRANS_RULE_ID_IPV4; - mlx4_spec->ipv4.src_ip = ib_spec->ipv4.val.src_ip; - mlx4_spec->ipv4.src_ip_msk = ib_spec->ipv4.mask.src_ip; - mlx4_spec->ipv4.dst_ip = ib_spec->ipv4.val.dst_ip; - mlx4_spec->ipv4.dst_ip_msk = ib_spec->ipv4.mask.dst_ip; - break; - - case IB_FLOW_SPEC_TCP: - case IB_FLOW_SPEC_UDP: - type = ib_spec->type == IB_FLOW_SPEC_TCP ? - MLX4_NET_TRANS_RULE_ID_TCP : - MLX4_NET_TRANS_RULE_ID_UDP; - mlx4_spec->tcp_udp.dst_port = ib_spec->tcp_udp.val.dst_port; - mlx4_spec->tcp_udp.dst_port_msk = - ib_spec->tcp_udp.mask.dst_port; - mlx4_spec->tcp_udp.src_port = ib_spec->tcp_udp.val.src_port; - mlx4_spec->tcp_udp.src_port_msk = - ib_spec->tcp_udp.mask.src_port; - break; - - default: - return -EINVAL; - } - if (map_sw_to_hw_steering_id(dev, type) < 0 || - hw_rule_sz(dev, type) < 0) - return -EINVAL; - mlx4_spec->id = cpu_to_be16(map_sw_to_hw_steering_id(dev, type)); - mlx4_spec->size = hw_rule_sz(dev, type) >> 2; - return hw_rule_sz(dev, type); -} - -static int __mlx4_ib_create_flow(struct ib_qp *qp, struct ib_flow_attr *flow_attr, - int domain, - enum mlx4_net_trans_promisc_mode flow_type, - u64 *reg_id) -{ - int ret, i; - int size = 0; - void *ib_flow; - struct mlx4_ib_dev *mdev = to_mdev(qp->device); - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_net_trans_rule_hw_ctrl *ctrl; - size_t rule_size = sizeof(struct mlx4_net_trans_rule_hw_ctrl) + - (sizeof(struct _rule_hw) * flow_attr->num_of_specs); - - static const u16 __mlx4_domain[] = { - [IB_FLOW_DOMAIN_USER] = MLX4_DOMAIN_UVERBS, - [IB_FLOW_DOMAIN_ETHTOOL] = MLX4_DOMAIN_ETHTOOL, - [IB_FLOW_DOMAIN_RFS] = MLX4_DOMAIN_RFS, - [IB_FLOW_DOMAIN_NIC] = MLX4_DOMAIN_NIC, - }; - - if (flow_attr->priority > MLX4_IB_FLOW_MAX_PRIO) { - pr_err("Invalid priority value.\n"); - return -EINVAL; - } - if (domain >= IB_FLOW_DOMAIN_NUM) { - pr_err("Invalid domain value.\n"); - return -EINVAL; - } - if (map_sw_to_hw_steering_mode(mdev->dev, flow_type) < 0) - return -EINVAL; - - mailbox = mlx4_alloc_cmd_mailbox(mdev->dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - memset(mailbox->buf, 0, rule_size); - ctrl = mailbox->buf; - - ctrl->prio = cpu_to_be16(__mlx4_domain[domain] | - flow_attr->priority); - ctrl->type = map_sw_to_hw_steering_mode(mdev->dev, flow_type); - ctrl->port = flow_attr->port; - ctrl->qpn = cpu_to_be32(qp->qp_num); - - if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_ALLOW_LOOP_BACK) - ctrl->flags = (1 << 3); - - ib_flow = flow_attr + 1; - size += sizeof(struct mlx4_net_trans_rule_hw_ctrl); - for (i = 0; i < flow_attr->num_of_specs; i++) { - ret = parse_flow_attr(mdev->dev, ib_flow, mailbox->buf + size); - if (ret < 0) { - mlx4_free_cmd_mailbox(mdev->dev, mailbox); - return -EINVAL; - } - ib_flow += ((union ib_flow_spec *)ib_flow)->size; - size += ret; - } - - ret = mlx4_cmd_imm(mdev->dev, mailbox->dma, reg_id, size >> 2, 0, - MLX4_QP_FLOW_STEERING_ATTACH, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - if (ret == -ENOMEM) - pr_err("mcg table is full. Fail to register network rule.\n"); - else if (ret == -ENXIO) - pr_err("Device managed flow steering is disabled. Fail to register network rule.\n"); - else if (ret) - pr_err("Invalid argumant. Fail to register network rule.\n"); - mlx4_free_cmd_mailbox(mdev->dev, mailbox); - return ret; -} - -static int __mlx4_ib_destroy_flow(struct mlx4_dev *dev, u64 reg_id) -{ - int err; - err = mlx4_cmd(dev, reg_id, 0, 0, - MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A, - MLX4_CMD_NATIVE); - if (err) - pr_err("Fail to detach network rule. registration id = 0x%llx\n", - (unsigned long long)reg_id); - return err; -} - -static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp, - struct ib_flow_attr *flow_attr, - int domain) -{ - int err = 0, i = 0; - struct mlx4_ib_flow *mflow; - enum mlx4_net_trans_promisc_mode type[2]; - - memset(type, 0, sizeof(type)); - - mflow = kzalloc(sizeof(struct mlx4_ib_flow), GFP_KERNEL); - if (!mflow) { - err = -ENOMEM; - goto err_free; - } - - switch (flow_attr->type) { - case IB_FLOW_ATTR_NORMAL: - type[0] = MLX4_FS_REGULAR; - break; - - case IB_FLOW_ATTR_ALL_DEFAULT: - type[0] = MLX4_FS_ALL_DEFAULT; - break; - - case IB_FLOW_ATTR_MC_DEFAULT: - type[0] = MLX4_FS_MC_DEFAULT; - break; - - case IB_FLOW_ATTR_SNIFFER: - type[0] = MLX4_FS_UC_SNIFFER; - type[1] = MLX4_FS_MC_SNIFFER; - break; - - default: - err = -EINVAL; - goto err_free; - } - - while (i < ARRAY_SIZE(type) && type[i]) { - err = __mlx4_ib_create_flow(qp, flow_attr, domain, type[i], - &mflow->reg_id[i]); - if (err) - goto err_free; - i++; - } - - return &mflow->ibflow; - -err_free: - kfree(mflow); - return ERR_PTR(err); -} - -static int mlx4_ib_destroy_flow(struct ib_flow *flow_id) -{ - int err, ret = 0; - int i = 0; - struct mlx4_ib_dev *mdev = to_mdev(flow_id->qp->device); - struct mlx4_ib_flow *mflow = to_mflow(flow_id); - - while (i < ARRAY_SIZE(mflow->reg_id) && mflow->reg_id[i]) { - err = __mlx4_ib_destroy_flow(mdev->dev, mflow->reg_id[i]); - if (err) - ret = err; - i++; - } - - kfree(mflow); - return ret; -} - -static struct mlx4_ib_gid_entry *find_gid_entry(struct mlx4_ib_qp *qp, u8 *raw) -{ - struct mlx4_ib_gid_entry *ge; - struct mlx4_ib_gid_entry *tmp; - struct mlx4_ib_gid_entry *ret = NULL; - - list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) { - if (!memcmp(raw, ge->gid.raw, 16)) { - ret = ge; - break; - } - } - - return ret; -} - - -static int del_gid_entry(struct ib_qp *ibqp, union ib_gid *gid) -{ - struct mlx4_ib_dev *mdev = to_mdev(ibqp->device); - struct mlx4_ib_qp *mqp = to_mqp(ibqp); - struct mlx4_ib_gid_entry *ge; - struct net_device *ndev; - u8 mac[6]; - - mutex_lock(&mqp->mutex); - ge = find_gid_entry(mqp, gid->raw); - if (ge) { - spin_lock(&mdev->iboe.lock); - ndev = ge->added ? mdev->iboe.netdevs[ge->port - 1] : NULL; - if (ndev) - dev_hold(ndev); - spin_unlock(&mdev->iboe.lock); - rdma_get_mcast_mac((struct in6_addr *)gid, mac); - if (ndev) { - rtnl_lock(); - dev_mc_delete(mdev->iboe.netdevs[ge->port - 1], mac, 6, 0); - rtnl_unlock(); - dev_put(ndev); - } - list_del(&ge->list); - kfree(ge); - } else - pr_warn("could not find mgid entry\n"); - - mutex_unlock(&mqp->mutex); - return ge != NULL ? 0 : -EINVAL; -} - -static int _mlx4_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid, - int count) -{ - int err; - struct mlx4_ib_dev *mdev = to_mdev(ibqp->device); - struct mlx4_ib_qp *mqp = to_mqp(ibqp); - u64 reg_id = 0; - int record_err = 0; - - if (mdev->dev->caps.steering_mode == - MLX4_STEERING_MODE_DEVICE_MANAGED) { - struct mlx4_ib_steering *ib_steering; - struct mlx4_ib_steering *tmp; - LIST_HEAD(temp); - - mutex_lock(&mqp->mutex); - list_for_each_entry_safe(ib_steering, tmp, &mqp->steering_rules, - list) { - if (memcmp(ib_steering->gid.raw, gid->raw, 16)) - continue; - - if (--count < 0) - break; - - list_del(&ib_steering->list); - list_add(&ib_steering->list, &temp); - } - mutex_unlock(&mqp->mutex); - list_for_each_entry_safe(ib_steering, tmp, &temp, - list) { - reg_id = ib_steering->reg_id; - - err = mlx4_multicast_detach(mdev->dev, &mqp->mqp, - gid->raw, - (ibqp->qp_type == IB_QPT_RAW_PACKET) ? - MLX4_PROT_ETH : MLX4_PROT_IB_IPV6, - reg_id); - if (err) { - record_err = record_err ?: err; - continue; - } - - err = del_gid_entry(ibqp, gid); - if (err) { - record_err = record_err ?: err; - continue; - } - - list_del(&ib_steering->list); - kfree(ib_steering); - } - mutex_lock(&mqp->mutex); - list_for_each_entry(ib_steering, &temp, list) { - list_add(&ib_steering->list, &mqp->steering_rules); - } - mutex_unlock(&mqp->mutex); - if (count) { - pr_warn("Couldn't release all reg_ids for mgid. Steering rule is left attached\n"); - return -EINVAL; - } - - } else { - if (mdev->dev->caps.steering_mode == MLX4_STEERING_MODE_B0 && - ibqp->qp_type == IB_QPT_RAW_PACKET) - gid->raw[5] = mqp->port; - - err = mlx4_multicast_detach(mdev->dev, &mqp->mqp, gid->raw, - (ibqp->qp_type == IB_QPT_RAW_PACKET) ? - MLX4_PROT_ETH : MLX4_PROT_IB_IPV6, - reg_id); - if (err) - return err; - - err = del_gid_entry(ibqp, gid); - - if (err) - return err; - } - - return record_err; -} - -static int mlx4_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) -{ - struct mlx4_ib_dev *mdev = to_mdev(ibqp->device); - int count = (mdev->dev->caps.steering_mode == - MLX4_STEERING_MODE_DEVICE_MANAGED) ? - mdev->dev->caps.num_ports : 1; - - return _mlx4_ib_mcg_detach(ibqp, gid, lid, count); -} - -static int mlx4_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) -{ - int err = -ENODEV; - struct mlx4_ib_dev *mdev = to_mdev(ibqp->device); - struct mlx4_ib_qp *mqp = to_mqp(ibqp); - DECLARE_BITMAP(ports, MLX4_MAX_PORTS); - int i = 0; - - if (mdev->dev->caps.steering_mode == MLX4_STEERING_MODE_B0 && - ibqp->qp_type == IB_QPT_RAW_PACKET) - gid->raw[5] = mqp->port; - - if (mdev->dev->caps.steering_mode == - MLX4_STEERING_MODE_DEVICE_MANAGED) { - bitmap_fill(ports, mdev->dev->caps.num_ports); - } else { - if (mqp->port <= mdev->dev->caps.num_ports) { - bitmap_zero(ports, mdev->dev->caps.num_ports); - set_bit(0, ports); - } else { - return -EINVAL; - } - } - - for (; i < mdev->dev->caps.num_ports; i++) { - u64 reg_id; - struct mlx4_ib_steering *ib_steering = NULL; - if (!test_bit(i, ports)) - continue; - if (mdev->dev->caps.steering_mode == - MLX4_STEERING_MODE_DEVICE_MANAGED) { - ib_steering = kmalloc(sizeof(*ib_steering), GFP_KERNEL); - if (!ib_steering) - goto err_add; - } - - err = mlx4_multicast_attach(mdev->dev, &mqp->mqp, - gid->raw, i + 1, - !!(mqp->flags & - MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK), - (ibqp->qp_type == IB_QPT_RAW_PACKET) ? - MLX4_PROT_ETH : MLX4_PROT_IB_IPV6, - ®_id); - if (err) { - kfree(ib_steering); - goto err_add; - } - - err = add_gid_entry(ibqp, gid); - if (err) { - mlx4_multicast_detach(mdev->dev, &mqp->mqp, gid->raw, - MLX4_PROT_IB_IPV6, reg_id); - kfree(ib_steering); - goto err_add; - } - - if (ib_steering) { - memcpy(ib_steering->gid.raw, gid->raw, 16); - mutex_lock(&mqp->mutex); - list_add(&ib_steering->list, &mqp->steering_rules); - mutex_unlock(&mqp->mutex); - ib_steering->reg_id = reg_id; - } - } - - - return 0; - -err_add: - if (i > 0) - _mlx4_ib_mcg_detach(ibqp, gid, lid, i); - - return err; -} - -static int init_node_data(struct mlx4_ib_dev *dev) -{ - struct ib_smp *in_mad = NULL; - struct ib_smp *out_mad = NULL; - int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS; - int err = -ENOMEM; - - in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); - out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); - if (!in_mad || !out_mad) - goto out; - - init_query_mad(in_mad); - in_mad->attr_id = IB_SMP_ATTR_NODE_DESC; - if (mlx4_is_master(dev->dev)) - mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW; - - err = mlx4_MAD_IFC(dev, mad_ifc_flags, 1, NULL, NULL, in_mad, out_mad); - if (err) - goto out; - - memcpy(dev->ib_dev.node_desc, out_mad->data, 64); - - in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; - - err = mlx4_MAD_IFC(dev, mad_ifc_flags, 1, NULL, NULL, in_mad, out_mad); - if (err) - goto out; - - dev->dev->rev_id = be32_to_cpup((__be32 *) (out_mad->data + 32)); - memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8); - -out: - kfree(in_mad); - kfree(out_mad); - return err; -} - -static ssize_t show_hca(struct device *device, struct device_attribute *attr, - char *buf) -{ - struct mlx4_ib_dev *dev = - container_of(device, struct mlx4_ib_dev, ib_dev.dev); - return sprintf(buf, "MT%d\n", dev->dev->pdev->device); -} - -static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr, - char *buf) -{ - struct mlx4_ib_dev *dev = - container_of(device, struct mlx4_ib_dev, ib_dev.dev); - return sprintf(buf, "%d.%d.%d\n", (int) (dev->dev->caps.fw_ver >> 32), - (int) (dev->dev->caps.fw_ver >> 16) & 0xffff, - (int) dev->dev->caps.fw_ver & 0xffff); -} - -static ssize_t show_rev(struct device *device, struct device_attribute *attr, - char *buf) -{ - struct mlx4_ib_dev *dev = - container_of(device, struct mlx4_ib_dev, ib_dev.dev); - return sprintf(buf, "%x\n", dev->dev->rev_id); -} - -static ssize_t show_board(struct device *device, struct device_attribute *attr, - char *buf) -{ - struct mlx4_ib_dev *dev = - container_of(device, struct mlx4_ib_dev, ib_dev.dev); - return sprintf(buf, "%.*s\n", MLX4_BOARD_ID_LEN, - dev->dev->board_id); -} - -static ssize_t show_vsd(struct device *device, struct device_attribute *attr, - char *buf) -{ - struct mlx4_ib_dev *dev = - container_of(device, struct mlx4_ib_dev, ib_dev.dev); - ssize_t len = MLX4_VSD_LEN; - - if (dev->dev->vsd_vendor_id == PCI_VENDOR_ID_MELLANOX) - len = sprintf(buf, "%.*s\n", MLX4_VSD_LEN, dev->dev->vsd); - else - memcpy(buf, dev->dev->vsd, MLX4_VSD_LEN); - - return len; -} - -static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL); -static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL); -static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL); -static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL); -static DEVICE_ATTR(vsd, S_IRUGO, show_vsd, NULL); - -static struct device_attribute *mlx4_class_attributes[] = { - &dev_attr_hw_rev, - &dev_attr_fw_ver, - &dev_attr_hca_type, - &dev_attr_board_id, - &dev_attr_vsd -}; - -static void mlx4_addrconf_ifid_eui48(u8 *eui, u16 vlan_id, struct net_device *dev, u8 port) -{ - memcpy(eui, IF_LLADDR(dev), 3); - memcpy(eui + 5, IF_LLADDR(dev) + 3, 3); - if (vlan_id < 0x1000) { - eui[3] = vlan_id >> 8; - eui[4] = vlan_id & 0xff; - } else { - eui[3] = 0xff; - eui[4] = 0xfe; - } - eui[0] ^= 2; -} - -static void update_gids_task(struct work_struct *work) -{ - struct update_gid_work *gw = container_of(work, struct update_gid_work, work); - struct mlx4_cmd_mailbox *mailbox; - union ib_gid *gids; - int err; - struct mlx4_dev *dev = gw->dev->dev; - - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) { - pr_warn("update gid table failed %ld\n", PTR_ERR(mailbox)); - goto free; - } - - gids = mailbox->buf; - memcpy(gids, gw->gids, sizeof gw->gids); - - if (mlx4_ib_port_link_layer(&gw->dev->ib_dev, gw->port) == - IB_LINK_LAYER_ETHERNET) { - err = mlx4_cmd(dev, mailbox->dma, - MLX4_SET_PORT_GID_TABLE << 8 | gw->port, - 1, MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_WRAPPED); - - if (err) - pr_warn("set port command failed\n"); - else - mlx4_ib_dispatch_event(gw->dev, gw->port, - IB_EVENT_GID_CHANGE); - } - - mlx4_free_cmd_mailbox(dev, mailbox); -free: - kfree(gw); -} - -static struct net_device *mlx4_ib_get_netdev(struct ib_device *device, u8 port_num) -{ - struct mlx4_ib_dev *ibdev = to_mdev(device); - return mlx4_get_protocol_dev(ibdev->dev, MLX4_PROT_ETH, port_num); -} - -static void reset_gids_task(struct work_struct *work) -{ - struct update_gid_work *gw = - container_of(work, struct update_gid_work, work); - struct mlx4_cmd_mailbox *mailbox; - union ib_gid *gids; - int err; - struct mlx4_dev *dev = gw->dev->dev; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) { - pr_warn("reset gid table failed\n"); - goto free; - } - - gids = mailbox->buf; - memcpy(gids, gw->gids, sizeof(gw->gids)); - - if (mlx4_ib_port_link_layer(&gw->dev->ib_dev, 1) == - IB_LINK_LAYER_ETHERNET && - dev->caps.num_ports > 0) { - err = mlx4_cmd(dev, mailbox->dma, - MLX4_SET_PORT_GID_TABLE << 8 | 1, - 1, MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_WRAPPED); - if (err) - pr_warn("set port 1 command failed\n"); - } - - if (mlx4_ib_port_link_layer(&gw->dev->ib_dev, 2) == - IB_LINK_LAYER_ETHERNET && - dev->caps.num_ports > 1) { - err = mlx4_cmd(dev, mailbox->dma, - MLX4_SET_PORT_GID_TABLE << 8 | 2, - 1, MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_WRAPPED); - if (err) - pr_warn("set port 2 command failed\n"); - } - - mlx4_free_cmd_mailbox(dev, mailbox); -free: - kfree(gw); -} - -static int update_gid_table(struct mlx4_ib_dev *dev, int port, - union ib_gid *gid, int clear, int default_gid) -{ - struct update_gid_work *work; - int i; - int need_update = 0; - int free = -1; - int found = -1; - int max_gids; - int start_index = !default_gid; - - max_gids = dev->dev->caps.gid_table_len[port]; - for (i = start_index; i < max_gids; ++i) { - if (!memcmp(&dev->iboe.gid_table[port - 1][i], gid, - sizeof(*gid))) - found = i; - - if (clear) { - if (found >= 0) { - need_update = 1; - dev->iboe.gid_table[port - 1][found] = zgid; - break; - } - } else { - if (found >= 0) - break; - - if (free < 0 && - !memcmp(&dev->iboe.gid_table[port - 1][i], - &zgid, sizeof(*gid))) - free = i; - } - } - - if (found == -1 && !clear && free < 0) { - pr_err("GID table of port %d is full. Can't add "GID_PRINT_FMT"\n", - port, GID_PRINT_ARGS(gid)); - return -ENOMEM; - } - if (found == -1 && clear) { - pr_err(GID_PRINT_FMT" is not in GID table of port %d\n", GID_PRINT_ARGS(gid), port); - return -EINVAL; - } - if (found == -1 && !clear && free >= 0) { - dev->iboe.gid_table[port - 1][free] = *gid; - need_update = 1; - } - - if (!need_update) - return 0; - - work = kzalloc(sizeof *work, GFP_ATOMIC); - if (!work) - return -ENOMEM; - - memcpy(work->gids, dev->iboe.gid_table[port - 1], sizeof(work->gids)); - INIT_WORK(&work->work, update_gids_task); - work->port = port; - work->dev = dev; - queue_work(wq, &work->work); - - return 0; -} - -static int reset_gid_table(struct mlx4_ib_dev *dev) -{ - struct update_gid_work *work; - - - work = kzalloc(sizeof(*work), GFP_ATOMIC); - if (!work) - return -ENOMEM; - - memset(dev->iboe.gid_table, 0, sizeof(dev->iboe.gid_table)); - memset(work->gids, 0, sizeof(work->gids)); - INIT_WORK(&work->work, reset_gids_task); - work->dev = dev; - queue_work(wq, &work->work); - return 0; -} - -/* XXX BOND Related - stub (no support for these flags in FBSD)*/ -static inline int netif_is_bond_master(struct net_device *dev) -{ -#if 0 - return (dev->flags & IFF_MASTER) && (dev->priv_flags & IFF_BONDING); -#endif - return 0; -} - -static void mlx4_make_default_gid(struct net_device *dev, union ib_gid *gid, u8 port) -{ - gid->global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL); - mlx4_addrconf_ifid_eui48(&gid->raw[8], 0xffff, dev, port); -} - -static u8 mlx4_ib_get_dev_port(struct net_device *dev, struct mlx4_ib_dev *ibdev) -{ - u8 port = 0; - struct mlx4_ib_iboe *iboe; - struct net_device *real_dev = rdma_vlan_dev_real_dev(dev) ? - rdma_vlan_dev_real_dev(dev) : dev; - - iboe = &ibdev->iboe; - - for (port = 1; port <= MLX4_MAX_PORTS; ++port) - if ((netif_is_bond_master(real_dev) && (real_dev == iboe->masters[port - 1])) || - (!netif_is_bond_master(real_dev) && (real_dev == iboe->netdevs[port - 1]))) - break; - - return port > MLX4_MAX_PORTS ? 0 : port; -} - -static void mlx4_ib_get_dev_addr(struct net_device *dev, struct mlx4_ib_dev *ibdev, u8 port) -{ - struct ifaddr *ifa; -#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) - struct inet6_dev *in6_dev; - union ib_gid *pgid; - struct inet6_ifaddr *ifp; -#endif - union ib_gid gid; - - - if ((port == 0) || (port > MLX4_MAX_PORTS)) - return; - - /* IPv4 gids */ - TAILQ_FOREACH(ifa, &dev->if_addrhead, ifa_link) { - if (ifa->ifa_addr && ifa->ifa_addr->sa_family == AF_INET){ - ipv6_addr_set_v4mapped( - ((struct sockaddr_in *) ifa->ifa_addr)->sin_addr.s_addr, - (struct in6_addr *)&gid); - update_gid_table(ibdev, port, &gid, 0, 0); - } - - } -#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) - /* IPv6 gids */ - in6_dev = in6_dev_get(dev); - if (in6_dev) { - read_lock_bh(&in6_dev->lock); - list_for_each_entry(ifp, &in6_dev->addr_list, if_list) { - pgid = (union ib_gid *)&ifp->addr; - update_gid_table(ibdev, port, pgid, 0, 0); - } - read_unlock_bh(&in6_dev->lock); - in6_dev_put(in6_dev); - } -#endif -} - -static void mlx4_set_default_gid(struct mlx4_ib_dev *ibdev, - struct net_device *dev, u8 port) -{ - union ib_gid gid; - mlx4_make_default_gid(dev, &gid, port); - update_gid_table(ibdev, port, &gid, 0, 1); -} - -static int mlx4_ib_init_gid_table(struct mlx4_ib_dev *ibdev) -{ - struct net_device *dev; - - if (reset_gid_table(ibdev)) - return -1; - - IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(dev, &V_ifnet, if_link) { - u8 port = mlx4_ib_get_dev_port(dev, ibdev); - if (port) { - if (!rdma_vlan_dev_real_dev(dev) && - !netif_is_bond_master(dev)) - mlx4_set_default_gid(ibdev, dev, port); - mlx4_ib_get_dev_addr(dev, ibdev, port); - } - } - - IFNET_RUNLOCK_NOSLEEP(); - - return 0; -} - -static void mlx4_ib_scan_netdevs(struct mlx4_ib_dev *ibdev, - struct net_device *dev, unsigned long event) -{ - struct mlx4_ib_iboe *iboe; - int port; - int init = 0; - unsigned long flags; - - iboe = &ibdev->iboe; - - spin_lock_irqsave(&iboe->lock, flags); - mlx4_foreach_ib_transport_port(port, ibdev->dev) { - struct net_device *old_netdev = iboe->netdevs[port - 1]; -/* XXX BOND related */ -#if 0 - struct net_device *old_master = iboe->masters[port - 1]; -#endif - iboe->masters[port - 1] = NULL; - iboe->netdevs[port - 1] = - mlx4_get_protocol_dev(ibdev->dev, MLX4_PROT_ETH, port); - - - if (old_netdev != iboe->netdevs[port - 1]) - init = 1; - if (dev == iboe->netdevs[port - 1] && - event == NETDEV_CHANGEADDR) - init = 1; -/* XXX BOND related */ -#if 0 - if (iboe->netdevs[port - 1] && netif_is_bond_slave(iboe->netdevs[port - 1])) - iboe->masters[port - 1] = iboe->netdevs[port - 1]->master; - - /* if bonding is used it is possible that we add it to masters only after - IP address is assigned to the net bonding interface */ - if (old_master != iboe->masters[port - 1]) - init = 1; -#endif - } - - spin_unlock_irqrestore(&iboe->lock, flags); - - if (init) - if (mlx4_ib_init_gid_table(ibdev)) - pr_warn("Fail to reset gid table\n"); -} - -static int mlx4_ib_netdev_event(struct notifier_block *this, unsigned long event, - void *ptr) -{ - struct net_device *dev = ptr; - struct mlx4_ib_dev *ibdev; - - ibdev = container_of(this, struct mlx4_ib_dev, iboe.nb); - - mlx4_ib_scan_netdevs(ibdev, dev, event); - - return NOTIFY_DONE; -} - -/* This function initializes the gid table only if the event_netdev real device is an iboe - * device, will be invoked by the inet/inet6 events */ -static int mlx4_ib_inet_event(struct notifier_block *this, unsigned long event, - void *ptr) -{ - struct net_device *event_netdev = ptr; - struct mlx4_ib_dev *ibdev; - struct mlx4_ib_iboe *ibdev_iboe; - int port = 0; - - ibdev = container_of(this, struct mlx4_ib_dev, iboe.nb_inet); - - struct net_device *real_dev = rdma_vlan_dev_real_dev(event_netdev) ? - rdma_vlan_dev_real_dev(event_netdev) : - event_netdev; - - ibdev_iboe = &ibdev->iboe; - - port = mlx4_ib_get_dev_port(real_dev, ibdev); - - /* Perform init_gid_table if the event real_dev is the net_device which represents this port, - * otherwise this event is not related and would be ignored.*/ - if(port && (real_dev == ibdev_iboe->netdevs[port - 1])) - if (mlx4_ib_init_gid_table(ibdev)) - pr_warn("Fail to reset gid table\n"); - - return NOTIFY_DONE; -} - - -static void init_pkeys(struct mlx4_ib_dev *ibdev) -{ - int port; - int slave; - int i; - - if (mlx4_is_master(ibdev->dev)) { - for (slave = 0; slave <= ibdev->dev->num_vfs; ++slave) { - for (port = 1; port <= ibdev->dev->caps.num_ports; ++port) { - for (i = 0; - i < ibdev->dev->phys_caps.pkey_phys_table_len[port]; - ++i) { - ibdev->pkeys.virt2phys_pkey[slave][port - 1][i] = - /* master has the identity virt2phys pkey mapping */ - (slave == mlx4_master_func_num(ibdev->dev) || !i) ? i : - ibdev->dev->phys_caps.pkey_phys_table_len[port] - 1; - mlx4_sync_pkey_table(ibdev->dev, slave, port, i, - ibdev->pkeys.virt2phys_pkey[slave][port - 1][i]); - } - } - } - /* initialize pkey cache */ - for (port = 1; port <= ibdev->dev->caps.num_ports; ++port) { - for (i = 0; - i < ibdev->dev->phys_caps.pkey_phys_table_len[port]; - ++i) - ibdev->pkeys.phys_pkey_cache[port-1][i] = - (i) ? 0 : 0xFFFF; - } - } -} - -static void mlx4_ib_alloc_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev) -{ - char name[32]; - int eq_per_port = 0; - int added_eqs = 0; - int total_eqs = 0; - int i, j, eq; - - /* Legacy mode or comp_pool is not large enough */ - if (dev->caps.comp_pool == 0 || - dev->caps.num_ports > dev->caps.comp_pool) - return; - - eq_per_port = rounddown_pow_of_two(dev->caps.comp_pool/ - dev->caps.num_ports); - - /* Init eq table */ - added_eqs = 0; - mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB) - added_eqs += eq_per_port; - - total_eqs = dev->caps.num_comp_vectors + added_eqs; - - ibdev->eq_table = kzalloc(total_eqs * sizeof(int), GFP_KERNEL); - if (!ibdev->eq_table) - return; - - ibdev->eq_added = added_eqs; - - eq = 0; - mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB) { - for (j = 0; j < eq_per_port; j++) { - sprintf(name, "mlx4-ib-%d-%d@%d:%d:%d:%d", i, j, - pci_get_domain(dev->pdev->dev.bsddev), - pci_get_bus(dev->pdev->dev.bsddev), - PCI_SLOT(dev->pdev->devfn), - PCI_FUNC(dev->pdev->devfn)); - - /* Set IRQ for specific name (per ring) */ - if (mlx4_assign_eq(dev, name, - &ibdev->eq_table[eq])) { - /* Use legacy (same as mlx4_en driver) */ - pr_warn("Can't allocate EQ %d; reverting to legacy\n", eq); - ibdev->eq_table[eq] = - (eq % dev->caps.num_comp_vectors); - } - eq++; - } - } - - /* Fill the reset of the vector with legacy EQ */ - for (i = 0, eq = added_eqs; i < dev->caps.num_comp_vectors; i++) - ibdev->eq_table[eq++] = i; - - /* Advertise the new number of EQs to clients */ - ibdev->ib_dev.num_comp_vectors = total_eqs; -} - -static void mlx4_ib_free_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev) -{ - int i; - - /* no additional eqs were added */ - if (!ibdev->eq_table) - return; - - /* Reset the advertised EQ number */ - ibdev->ib_dev.num_comp_vectors = dev->caps.num_comp_vectors; - - /* Free only the added eqs */ - for (i = 0; i < ibdev->eq_added; i++) { - /* Don't free legacy eqs if used */ - if (ibdev->eq_table[i] <= dev->caps.num_comp_vectors) - continue; - mlx4_release_eq(dev, ibdev->eq_table[i]); - } - - kfree(ibdev->eq_table); -} - -/* - * create show function and a device_attribute struct pointing to - * the function for _name - */ -#define DEVICE_DIAG_RPRT_ATTR(_name, _offset, _op_mod) \ -static ssize_t show_rprt_##_name(struct device *dev, \ - struct device_attribute *attr, \ - char *buf){ \ - return show_diag_rprt(dev, buf, _offset, _op_mod); \ -} \ -static DEVICE_ATTR(_name, S_IRUGO, show_rprt_##_name, NULL); - -#define MLX4_DIAG_RPRT_CLEAR_DIAGS 3 - -static size_t show_diag_rprt(struct device *device, char *buf, - u32 offset, u8 op_modifier) -{ - size_t ret; - u32 counter_offset = offset; - u32 diag_counter = 0; - struct mlx4_ib_dev *dev = container_of(device, struct mlx4_ib_dev, - ib_dev.dev); - - ret = mlx4_query_diag_counters(dev->dev, 1, op_modifier, - &counter_offset, &diag_counter); - if (ret) - return ret; - - return sprintf(buf, "%d\n", diag_counter); -} - -static ssize_t clear_diag_counters(struct device *device, - struct device_attribute *attr, - const char *buf, size_t length) -{ - size_t ret; - struct mlx4_ib_dev *dev = container_of(device, struct mlx4_ib_dev, - ib_dev.dev); - - ret = mlx4_query_diag_counters(dev->dev, 0, MLX4_DIAG_RPRT_CLEAR_DIAGS, - NULL, NULL); - if (ret) - return ret; - - return length; -} - -DEVICE_DIAG_RPRT_ATTR(rq_num_lle , 0x00, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_lle , 0x04, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_lqpoe , 0x08, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_lqpoe , 0x0C, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_lpe , 0x18, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_lpe , 0x1C, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_wrfe , 0x20, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_wrfe , 0x24, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_mwbe , 0x2C, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_bre , 0x34, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_lae , 0x38, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_rire , 0x44, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_rire , 0x48, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_rae , 0x4C, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_rae , 0x50, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_roe , 0x54, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_tree , 0x5C, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_rree , 0x64, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_rnr , 0x68, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_rnr , 0x6C, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_oos , 0x100, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_oos , 0x104, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_mce , 0x108, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_udsdprd , 0x118, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_ucsdprd , 0x120, 2); -DEVICE_DIAG_RPRT_ATTR(num_cqovf , 0x1A0, 2); -DEVICE_DIAG_RPRT_ATTR(num_eqovf , 0x1A4, 2); -DEVICE_DIAG_RPRT_ATTR(num_baddb , 0x1A8, 2); - -static DEVICE_ATTR(clear_diag, S_IWUSR, NULL, clear_diag_counters); - -static struct attribute *diag_rprt_attrs[] = { - &dev_attr_rq_num_lle.attr, - &dev_attr_sq_num_lle.attr, - &dev_attr_rq_num_lqpoe.attr, - &dev_attr_sq_num_lqpoe.attr, - &dev_attr_rq_num_lpe.attr, - &dev_attr_sq_num_lpe.attr, - &dev_attr_rq_num_wrfe.attr, - &dev_attr_sq_num_wrfe.attr, - &dev_attr_sq_num_mwbe.attr, - &dev_attr_sq_num_bre.attr, - &dev_attr_rq_num_lae.attr, - &dev_attr_sq_num_rire.attr, - &dev_attr_rq_num_rire.attr, - &dev_attr_sq_num_rae.attr, - &dev_attr_rq_num_rae.attr, - &dev_attr_sq_num_roe.attr, - &dev_attr_sq_num_tree.attr, - &dev_attr_sq_num_rree.attr, - &dev_attr_rq_num_rnr.attr, - &dev_attr_sq_num_rnr.attr, - &dev_attr_rq_num_oos.attr, - &dev_attr_sq_num_oos.attr, - &dev_attr_rq_num_mce.attr, - &dev_attr_rq_num_udsdprd.attr, - &dev_attr_rq_num_ucsdprd.attr, - &dev_attr_num_cqovf.attr, - &dev_attr_num_eqovf.attr, - &dev_attr_num_baddb.attr, - &dev_attr_clear_diag.attr, - NULL -}; - -static struct attribute_group diag_counters_group = { - .name = "diag_counters", - .attrs = diag_rprt_attrs -}; - -static void init_dev_assign(void) -{ - int i = 1; - - spin_lock_init(&dev_num_str_lock); - if (mlx4_fill_dbdf2val_tbl(&dev_assign_str)) - return; - dev_num_str_bitmap = - kmalloc(BITS_TO_LONGS(MAX_NUM_STR_BITMAP) * sizeof(long), - GFP_KERNEL); - if (!dev_num_str_bitmap) { - pr_warn("bitmap alloc failed -- cannot apply dev_assign_str parameter\n"); - return; - } - bitmap_zero(dev_num_str_bitmap, MAX_NUM_STR_BITMAP); - while ((i < MLX4_DEVS_TBL_SIZE) && (dev_assign_str.tbl[i].dbdf != - MLX4_ENDOF_TBL)) { - if (bitmap_allocate_region(dev_num_str_bitmap, - dev_assign_str.tbl[i].val[0], 0)) - goto err; - i++; - } - dr_active = 1; - return; - -err: - kfree(dev_num_str_bitmap); - dev_num_str_bitmap = NULL; - pr_warn("mlx4_ib: The value of 'dev_assign_str' parameter " - "is incorrect. The parameter value is discarded!"); -} - -static int mlx4_ib_dev_idx(struct mlx4_dev *dev) -{ - int i, val; - - if (!dr_active) - return -1; - if (!dev) - return -1; - if (mlx4_get_val(dev_assign_str.tbl, dev->pdev, 0, &val)) - return -1; - - if (val != DEFAULT_TBL_VAL) { - dev->flags |= MLX4_FLAG_DEV_NUM_STR; - return val; - } - - spin_lock(&dev_num_str_lock); - i = bitmap_find_free_region(dev_num_str_bitmap, MAX_NUM_STR_BITMAP, 0); - spin_unlock(&dev_num_str_lock); - if (i >= 0) - return i; - - return -1; -} - -static int mlx4_port_immutable(struct ib_device *ibdev, u8 port_num, - struct ib_port_immutable *immutable) -{ - struct ib_port_attr attr; - struct mlx4_ib_dev *mdev = to_mdev(ibdev); - int err; - - if (mlx4_ib_port_link_layer(ibdev, port_num) == IB_LINK_LAYER_INFINIBAND) { - immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB; - immutable->max_mad_size = IB_MGMT_MAD_SIZE; - } else { - if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_IBOE) - immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; - if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCEV2) - immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE | - RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; - immutable->core_cap_flags |= RDMA_CORE_PORT_RAW_PACKET; - if (immutable->core_cap_flags & (RDMA_CORE_PORT_IBA_ROCE | - RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP)) - immutable->max_mad_size = IB_MGMT_MAD_SIZE; - } - - err = ib_query_port(ibdev, port_num, &attr); - if (err) - return err; - - immutable->pkey_tbl_len = attr.pkey_tbl_len; - immutable->gid_tbl_len = attr.gid_tbl_len; - - return 0; -} - -static void *mlx4_ib_add(struct mlx4_dev *dev) -{ - struct mlx4_ib_dev *ibdev; - int num_ports = 0; - int i, j; - int err; - struct mlx4_ib_iboe *iboe; - int dev_idx; - - pr_info_once("%s", mlx4_ib_version); - - mlx4_foreach_ib_transport_port(i, dev) - num_ports++; - - /* No point in registering a device with no ports... */ - if (num_ports == 0) - return NULL; - - ibdev = (struct mlx4_ib_dev *) ib_alloc_device(sizeof *ibdev); - if (!ibdev) { - dev_err(&dev->pdev->dev, "Device struct alloc failed\n"); - return NULL; - } - - iboe = &ibdev->iboe; - - if (mlx4_pd_alloc(dev, &ibdev->priv_pdn)) - goto err_dealloc; - - if (mlx4_uar_alloc(dev, &ibdev->priv_uar)) - goto err_pd; - - ibdev->priv_uar.map = ioremap(ibdev->priv_uar.pfn << PAGE_SHIFT, - PAGE_SIZE); - - if (!ibdev->priv_uar.map) - goto err_uar; - - MLX4_INIT_DOORBELL_LOCK(&ibdev->uar_lock); - - ibdev->dev = dev; - - dev_idx = mlx4_ib_dev_idx(dev); - if (dev_idx >= 0) - sprintf(ibdev->ib_dev.name, "mlx4_%d", dev_idx); - else - strlcpy(ibdev->ib_dev.name, "mlx4_%d", IB_DEVICE_NAME_MAX); - - ibdev->ib_dev.owner = THIS_MODULE; - ibdev->ib_dev.node_type = RDMA_NODE_IB_CA; - ibdev->ib_dev.local_dma_lkey = dev->caps.reserved_lkey; - ibdev->num_ports = num_ports; - ibdev->ib_dev.phys_port_cnt = ibdev->num_ports; - ibdev->ib_dev.num_comp_vectors = dev->caps.num_comp_vectors; - ibdev->ib_dev.dma_device = &dev->pdev->dev; - - if (dev->caps.userspace_caps) - ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION; - else - ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION; - - ibdev->ib_dev.uverbs_cmd_mask = - (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | - (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | - (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | - (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | - (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | - (1ull << IB_USER_VERBS_CMD_REG_MR) | - (1ull << IB_USER_VERBS_CMD_DEREG_MR) | - (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | - (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | - (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | - (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | - (1ull << IB_USER_VERBS_CMD_CREATE_QP) | - (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | - (1ull << IB_USER_VERBS_CMD_QUERY_QP) | - (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | - (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) | - (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) | - (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | - (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | - (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | - (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | - (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ) | - (1ull << IB_USER_VERBS_CMD_OPEN_QP); - - ibdev->ib_dev.query_device = mlx4_ib_query_device; - ibdev->ib_dev.query_port = mlx4_ib_query_port; - ibdev->ib_dev.get_link_layer = mlx4_ib_port_link_layer; - ibdev->ib_dev.query_gid = mlx4_ib_query_gid; - ibdev->ib_dev.query_pkey = mlx4_ib_query_pkey; - ibdev->ib_dev.modify_device = mlx4_ib_modify_device; - ibdev->ib_dev.modify_port = mlx4_ib_modify_port; - ibdev->ib_dev.alloc_ucontext = mlx4_ib_alloc_ucontext; - ibdev->ib_dev.dealloc_ucontext = mlx4_ib_dealloc_ucontext; - ibdev->ib_dev.mmap = mlx4_ib_mmap; -/* XXX FBSD has no support for get_unmapped_area function */ -#if 0 - ibdev->ib_dev.get_unmapped_area = mlx4_ib_get_unmapped_area; -#endif - ibdev->ib_dev.alloc_pd = mlx4_ib_alloc_pd; - ibdev->ib_dev.dealloc_pd = mlx4_ib_dealloc_pd; - ibdev->ib_dev.create_ah = mlx4_ib_create_ah; - ibdev->ib_dev.query_ah = mlx4_ib_query_ah; - ibdev->ib_dev.destroy_ah = mlx4_ib_destroy_ah; - ibdev->ib_dev.create_srq = mlx4_ib_create_srq; - ibdev->ib_dev.modify_srq = mlx4_ib_modify_srq; - ibdev->ib_dev.query_srq = mlx4_ib_query_srq; - ibdev->ib_dev.destroy_srq = mlx4_ib_destroy_srq; - ibdev->ib_dev.post_srq_recv = mlx4_ib_post_srq_recv; - ibdev->ib_dev.create_qp = mlx4_ib_create_qp; - ibdev->ib_dev.modify_qp = mlx4_ib_modify_qp; - ibdev->ib_dev.query_qp = mlx4_ib_query_qp; - ibdev->ib_dev.destroy_qp = mlx4_ib_destroy_qp; - ibdev->ib_dev.post_send = mlx4_ib_post_send; - ibdev->ib_dev.post_recv = mlx4_ib_post_recv; - ibdev->ib_dev.create_cq = mlx4_ib_create_cq; - ibdev->ib_dev.modify_cq = mlx4_ib_modify_cq; - ibdev->ib_dev.resize_cq = mlx4_ib_resize_cq; - ibdev->ib_dev.destroy_cq = mlx4_ib_destroy_cq; - ibdev->ib_dev.poll_cq = mlx4_ib_poll_cq; - ibdev->ib_dev.req_notify_cq = mlx4_ib_arm_cq; - ibdev->ib_dev.get_dma_mr = mlx4_ib_get_dma_mr; - ibdev->ib_dev.reg_user_mr = mlx4_ib_reg_user_mr; - ibdev->ib_dev.dereg_mr = mlx4_ib_dereg_mr; - ibdev->ib_dev.alloc_fast_reg_mr = mlx4_ib_alloc_fast_reg_mr; - ibdev->ib_dev.alloc_fast_reg_page_list = mlx4_ib_alloc_fast_reg_page_list; - ibdev->ib_dev.free_fast_reg_page_list = mlx4_ib_free_fast_reg_page_list; - ibdev->ib_dev.attach_mcast = mlx4_ib_mcg_attach; - ibdev->ib_dev.detach_mcast = mlx4_ib_mcg_detach; - ibdev->ib_dev.process_mad = mlx4_ib_process_mad; - ibdev->ib_dev.get_port_immutable = mlx4_port_immutable; - ibdev->ib_dev.get_netdev = mlx4_ib_get_netdev; - ibdev->ib_dev.ioctl = mlx4_ib_ioctl; - ibdev->ib_dev.query_values = mlx4_ib_query_values; - - if (!mlx4_is_slave(ibdev->dev)) { - ibdev->ib_dev.alloc_fmr = mlx4_ib_fmr_alloc; - ibdev->ib_dev.map_phys_fmr = mlx4_ib_map_phys_fmr; - ibdev->ib_dev.unmap_fmr = mlx4_ib_unmap_fmr; - ibdev->ib_dev.dealloc_fmr = mlx4_ib_fmr_dealloc; - } - - if (dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW) { - ibdev->ib_dev.alloc_mw = mlx4_ib_alloc_mw; - ibdev->ib_dev.bind_mw = mlx4_ib_bind_mw; - ibdev->ib_dev.dealloc_mw = mlx4_ib_dealloc_mw; - - ibdev->ib_dev.uverbs_cmd_mask |= - (1ull << IB_USER_VERBS_CMD_ALLOC_MW) | - (1ull << IB_USER_VERBS_CMD_DEALLOC_MW); - } - - if (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) { - ibdev->ib_dev.alloc_xrcd = mlx4_ib_alloc_xrcd; - ibdev->ib_dev.dealloc_xrcd = mlx4_ib_dealloc_xrcd; - ibdev->ib_dev.uverbs_cmd_mask |= - (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) | - (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD); - } - - /* - * Set experimental data - */ - ibdev->ib_dev.uverbs_exp_cmd_mask = - (1ull << IB_USER_VERBS_EXP_CMD_CREATE_QP) | - (1ull << IB_USER_VERBS_EXP_CMD_MODIFY_CQ) | - (1ull << IB_USER_VERBS_EXP_CMD_QUERY_DEVICE) | - (1ull << IB_USER_VERBS_EXP_CMD_CREATE_CQ); - ibdev->ib_dev.exp_create_qp = mlx4_ib_exp_create_qp; - ibdev->ib_dev.exp_query_device = mlx4_ib_exp_query_device; - if (check_flow_steering_support(dev)) { - ibdev->ib_dev.uverbs_ex_cmd_mask |= - (1ull << IB_USER_VERBS_EX_CMD_CREATE_FLOW) | - (1ull << IB_USER_VERBS_EX_CMD_DESTROY_FLOW); - ibdev->ib_dev.create_flow = mlx4_ib_create_flow; - ibdev->ib_dev.destroy_flow = mlx4_ib_destroy_flow; - } else { - pr_debug("Device managed flow steering is unavailable for this configuration.\n"); - } - /* - * End of experimental data - */ - - mlx4_ib_alloc_eqs(dev, ibdev); - - spin_lock_init(&iboe->lock); - - if (init_node_data(ibdev)) - goto err_map; - - for (i = 0; i < ibdev->num_ports; ++i) { - if (mlx4_ib_port_link_layer(&ibdev->ib_dev, i + 1) == - IB_LINK_LAYER_ETHERNET) { - if (mlx4_is_slave(dev)) { - ibdev->counters[i].status = mlx4_counter_alloc(ibdev->dev, - i + 1, - &ibdev->counters[i].counter_index); - } else {/* allocating the PF IB default counter indices reserved in mlx4_init_counters_table */ - ibdev->counters[i].counter_index = ((i + 1) << 1) - 1; - ibdev->counters[i].status = 0; - } - - dev_info(&dev->pdev->dev, - "%s: allocated counter index %d for port %d\n", - __func__, ibdev->counters[i].counter_index, i+1); - } else { - ibdev->counters[i].counter_index = MLX4_SINK_COUNTER_INDEX; - ibdev->counters[i].status = -ENOSPC; - } - } - - spin_lock_init(&ibdev->sm_lock); - mutex_init(&ibdev->cap_mask_mutex); - - if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED && - !mlx4_is_mfunc(dev)) { - ibdev->steer_qpn_count = MLX4_IB_UC_MAX_NUM_QPS; - err = mlx4_qp_reserve_range(dev, ibdev->steer_qpn_count, - MLX4_IB_UC_STEER_QPN_ALIGN, &ibdev->steer_qpn_base, 0); - if (err) - goto err_counter; - - ibdev->ib_uc_qpns_bitmap = - kmalloc(BITS_TO_LONGS(ibdev->steer_qpn_count) * - sizeof(long), - GFP_KERNEL); - if (!ibdev->ib_uc_qpns_bitmap) { - dev_err(&dev->pdev->dev, "bit map alloc failed\n"); - goto err_steer_qp_release; - } - - bitmap_zero(ibdev->ib_uc_qpns_bitmap, ibdev->steer_qpn_count); - - err = mlx4_FLOW_STEERING_IB_UC_QP_RANGE(dev, ibdev->steer_qpn_base, - ibdev->steer_qpn_base + ibdev->steer_qpn_count - 1); - if (err) - goto err_steer_free_bitmap; - } - - if (ib_register_device(&ibdev->ib_dev, NULL)) - goto err_steer_free_bitmap; - - if (mlx4_ib_mad_init(ibdev)) - goto err_reg; - - if (mlx4_ib_init_sriov(ibdev)) - goto err_mad; - - if (dev->caps.flags & MLX4_DEV_CAP_FLAG_IBOE) { - if (!iboe->nb.notifier_call) { - iboe->nb.notifier_call = mlx4_ib_netdev_event; - err = register_netdevice_notifier(&iboe->nb); - if (err) { - iboe->nb.notifier_call = NULL; - goto err_notify; - } - } - if (!iboe->nb_inet.notifier_call) { - iboe->nb_inet.notifier_call = mlx4_ib_inet_event; - err = register_inetaddr_notifier(&iboe->nb_inet); - if (err) { - iboe->nb_inet.notifier_call = NULL; - goto err_notify; - } - } - mlx4_ib_scan_netdevs(ibdev, NULL, 0); - } - for (j = 0; j < ARRAY_SIZE(mlx4_class_attributes); ++j) { - if (device_create_file(&ibdev->ib_dev.dev, - mlx4_class_attributes[j])) - goto err_notify; - } - if (sysfs_create_group(&ibdev->ib_dev.dev.kobj, &diag_counters_group)) - goto err_notify; - - ibdev->ib_active = true; - - if (mlx4_is_mfunc(ibdev->dev)) - init_pkeys(ibdev); - - /* create paravirt contexts for any VFs which are active */ - if (mlx4_is_master(ibdev->dev)) { - for (j = 0; j < MLX4_MFUNC_MAX; j++) { - if (j == mlx4_master_func_num(ibdev->dev)) - continue; - if (mlx4_is_slave_active(ibdev->dev, j)) - do_slave_init(ibdev, j, 1); - } - } - return ibdev; - -err_notify: - for (j = 0; j < ARRAY_SIZE(mlx4_class_attributes); ++j) { - device_remove_file(&ibdev->ib_dev.dev, - mlx4_class_attributes[j]); - } - - if (ibdev->iboe.nb.notifier_call) { - if (unregister_netdevice_notifier(&ibdev->iboe.nb)) - pr_warn("failure unregistering notifier\n"); - ibdev->iboe.nb.notifier_call = NULL; - } - if (ibdev->iboe.nb_inet.notifier_call) { - if (unregister_inetaddr_notifier(&ibdev->iboe.nb_inet)) - pr_warn("failure unregistering notifier\n"); - ibdev->iboe.nb_inet.notifier_call = NULL; - } - flush_workqueue(wq); - - mlx4_ib_close_sriov(ibdev); - -err_mad: - mlx4_ib_mad_cleanup(ibdev); - -err_reg: - ib_unregister_device(&ibdev->ib_dev); - -err_steer_free_bitmap: - kfree(ibdev->ib_uc_qpns_bitmap); - -err_steer_qp_release: - if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) - mlx4_qp_release_range(dev, ibdev->steer_qpn_base, - ibdev->steer_qpn_count); -err_counter: - for (; i; --i) { - if (mlx4_ib_port_link_layer(&ibdev->ib_dev, i) == - IB_LINK_LAYER_ETHERNET) { - mlx4_counter_free(ibdev->dev, - i, - ibdev->counters[i - 1].counter_index); - } - } - -err_map: - iounmap(ibdev->priv_uar.map); - mlx4_ib_free_eqs(dev, ibdev); - -err_uar: - mlx4_uar_free(dev, &ibdev->priv_uar); - -err_pd: - mlx4_pd_free(dev, ibdev->priv_pdn); - -err_dealloc: - ib_dealloc_device(&ibdev->ib_dev); - - return NULL; -} - -int mlx4_ib_steer_qp_alloc(struct mlx4_ib_dev *dev, int count, int *qpn) -{ - int offset; - - WARN_ON(!dev->ib_uc_qpns_bitmap); - - offset = bitmap_find_free_region(dev->ib_uc_qpns_bitmap, - dev->steer_qpn_count, - get_count_order(count)); - if (offset < 0) - return offset; - - *qpn = dev->steer_qpn_base + offset; - return 0; -} - -void mlx4_ib_steer_qp_free(struct mlx4_ib_dev *dev, u32 qpn, int count) -{ - if (!qpn || - dev->dev->caps.steering_mode != MLX4_STEERING_MODE_DEVICE_MANAGED) - return; - - BUG_ON(qpn < dev->steer_qpn_base); - - bitmap_release_region(dev->ib_uc_qpns_bitmap, - qpn - dev->steer_qpn_base, get_count_order(count)); -} - -int mlx4_ib_steer_qp_reg(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp, - int is_attach) -{ - int err; - size_t flow_size; - struct ib_flow_attr *flow = NULL; - struct ib_flow_spec_ib *ib_spec; - - if (is_attach) { - flow_size = sizeof(struct ib_flow_attr) + - sizeof(struct ib_flow_spec_ib); - flow = kzalloc(flow_size, GFP_KERNEL); - if (!flow) - return -ENOMEM; - flow->port = mqp->port; - flow->num_of_specs = 1; - flow->size = flow_size; - ib_spec = (struct ib_flow_spec_ib *)(flow + 1); - ib_spec->type = IB_FLOW_SPEC_IB; - ib_spec->size = sizeof(struct ib_flow_spec_ib); - ib_spec->val.l3_type_qpn = mqp->ibqp.qp_num; - ib_spec->mask.l3_type_qpn = MLX4_IB_FLOW_QPN_MASK; - - err = __mlx4_ib_create_flow(&mqp->ibqp, flow, - IB_FLOW_DOMAIN_NIC, - MLX4_FS_REGULAR, - &mqp->reg_id); - } else { - err = __mlx4_ib_destroy_flow(mdev->dev, mqp->reg_id); - } - kfree(flow); - return err; -} - -static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr) -{ - struct mlx4_ib_dev *ibdev = ibdev_ptr; - int p, j; - int dev_idx, ret; - - if (ibdev->iboe.nb_inet.notifier_call) { - if (unregister_inetaddr_notifier(&ibdev->iboe.nb_inet)) - pr_warn("failure unregistering notifier\n"); - ibdev->iboe.nb_inet.notifier_call = NULL; - } - - mlx4_ib_close_sriov(ibdev); - sysfs_remove_group(&ibdev->ib_dev.dev.kobj, &diag_counters_group); - mlx4_ib_mad_cleanup(ibdev); - - for (j = 0; j < ARRAY_SIZE(mlx4_class_attributes); ++j) { - device_remove_file(&ibdev->ib_dev.dev, - mlx4_class_attributes[j]); - } - - - dev_idx = -1; - if (dr_active && !(ibdev->dev->flags & MLX4_FLAG_DEV_NUM_STR)) { - ret = sscanf(ibdev->ib_dev.name, "mlx4_%d", &dev_idx); - if (ret != 1) - dev_idx = -1; - } - ib_unregister_device(&ibdev->ib_dev); - if (dev_idx >= 0) { - spin_lock(&dev_num_str_lock); - bitmap_release_region(dev_num_str_bitmap, dev_idx, 0); - spin_unlock(&dev_num_str_lock); - } - - if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) { - mlx4_qp_release_range(dev, ibdev->steer_qpn_base, - ibdev->steer_qpn_count); - kfree(ibdev->ib_uc_qpns_bitmap); - } - - if (ibdev->iboe.nb.notifier_call) { - if (unregister_netdevice_notifier(&ibdev->iboe.nb)) - pr_warn("failure unregistering notifier\n"); - ibdev->iboe.nb.notifier_call = NULL; - } - iounmap(ibdev->priv_uar.map); - - for (p = 0; p < ibdev->num_ports; ++p) { - if (mlx4_ib_port_link_layer(&ibdev->ib_dev, p + 1) == - IB_LINK_LAYER_ETHERNET) { - mlx4_counter_free(ibdev->dev, - p + 1, - ibdev->counters[p].counter_index); - } - } - - mlx4_foreach_port(p, dev, MLX4_PORT_TYPE_IB) - mlx4_CLOSE_PORT(dev, p); - - mlx4_ib_free_eqs(dev, ibdev); - - mlx4_uar_free(dev, &ibdev->priv_uar); - mlx4_pd_free(dev, ibdev->priv_pdn); - ib_dealloc_device(&ibdev->ib_dev); -} - -static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init) -{ - struct mlx4_ib_demux_work **dm = NULL; - struct mlx4_dev *dev = ibdev->dev; - int i; - unsigned long flags; - - if (!mlx4_is_master(dev)) - return; - - dm = kcalloc(dev->caps.num_ports, sizeof *dm, GFP_ATOMIC); - if (!dm) { - pr_err("failed to allocate memory for tunneling qp update\n"); - goto out; - } - - for (i = 0; i < dev->caps.num_ports; i++) { - dm[i] = kmalloc(sizeof (struct mlx4_ib_demux_work), GFP_ATOMIC); - if (!dm[i]) { - pr_err("failed to allocate memory for tunneling qp update work struct\n"); - for (i = 0; i < dev->caps.num_ports; i++) { - if (dm[i]) - kfree(dm[i]); - } - goto out; - } - } - /* initialize or tear down tunnel QPs for the slave */ - for (i = 0; i < dev->caps.num_ports; i++) { - INIT_WORK(&dm[i]->work, mlx4_ib_tunnels_update_work); - dm[i]->port = i + 1; - dm[i]->slave = slave; - dm[i]->do_init = do_init; - dm[i]->dev = ibdev; - spin_lock_irqsave(&ibdev->sriov.going_down_lock, flags); - if (!ibdev->sriov.is_going_down) - queue_work(ibdev->sriov.demux[i].ud_wq, &dm[i]->work); - spin_unlock_irqrestore(&ibdev->sriov.going_down_lock, flags); - } -out: - if (dm) - kfree(dm); - return; -} - -static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr, - enum mlx4_dev_event event, unsigned long param) -{ - struct ib_event ibev; - struct mlx4_ib_dev *ibdev = to_mdev((struct ib_device *) ibdev_ptr); - struct mlx4_eqe *eqe = NULL; - struct ib_event_work *ew; - int p = 0; - - if (event == MLX4_DEV_EVENT_PORT_MGMT_CHANGE) - eqe = (struct mlx4_eqe *)param; - else - p = (int) param; - - switch (event) { - case MLX4_DEV_EVENT_PORT_UP: - if (p > ibdev->num_ports) - return; - if (mlx4_is_master(dev) && - rdma_port_get_link_layer(&ibdev->ib_dev, p) == - IB_LINK_LAYER_INFINIBAND) { - mlx4_ib_invalidate_all_guid_record(ibdev, p); - } - mlx4_ib_info((struct ib_device *) ibdev_ptr, - "Port %d logical link is up\n", p); - ibev.event = IB_EVENT_PORT_ACTIVE; - break; - - case MLX4_DEV_EVENT_PORT_DOWN: - if (p > ibdev->num_ports) - return; - mlx4_ib_info((struct ib_device *) ibdev_ptr, - "Port %d logical link is down\n", p); - ibev.event = IB_EVENT_PORT_ERR; - break; - - case MLX4_DEV_EVENT_CATASTROPHIC_ERROR: - ibdev->ib_active = false; - ibev.event = IB_EVENT_DEVICE_FATAL; - break; - - case MLX4_DEV_EVENT_PORT_MGMT_CHANGE: - ew = kmalloc(sizeof *ew, GFP_ATOMIC); - if (!ew) { - pr_err("failed to allocate memory for events work\n"); - break; - } - - INIT_WORK(&ew->work, handle_port_mgmt_change_event); - memcpy(&ew->ib_eqe, eqe, sizeof *eqe); - ew->ib_dev = ibdev; - /* need to queue only for port owner, which uses GEN_EQE */ - if (mlx4_is_master(dev)) - queue_work(wq, &ew->work); - else - handle_port_mgmt_change_event(&ew->work); - return; - - case MLX4_DEV_EVENT_SLAVE_INIT: - /* here, p is the slave id */ - do_slave_init(ibdev, p, 1); - return; - - case MLX4_DEV_EVENT_SLAVE_SHUTDOWN: - /* here, p is the slave id */ - do_slave_init(ibdev, p, 0); - return; - - default: - return; - } - - ibev.device = ibdev_ptr; - ibev.element.port_num = (u8) p; - - ib_dispatch_event(&ibev); -} - -static struct mlx4_interface mlx4_ib_interface = { - .add = mlx4_ib_add, - .remove = mlx4_ib_remove, - .event = mlx4_ib_event, - .protocol = MLX4_PROT_IB_IPV6 -}; - -static int __init mlx4_ib_init(void) -{ - int err; - - wq = create_singlethread_workqueue("mlx4_ib"); - if (!wq) - return -ENOMEM; - - err = mlx4_ib_mcg_init(); - if (err) - goto clean_proc; - - init_dev_assign(); - - err = mlx4_register_interface(&mlx4_ib_interface); - if (err) - goto clean_mcg; - - return 0; - -clean_mcg: - mlx4_ib_mcg_destroy(); - -clean_proc: - destroy_workqueue(wq); - return err; -} - -static void __exit mlx4_ib_cleanup(void) -{ - mlx4_unregister_interface(&mlx4_ib_interface); - mlx4_ib_mcg_destroy(); - destroy_workqueue(wq); - - kfree(dev_num_str_bitmap); -} - -module_init_order(mlx4_ib_init, SI_ORDER_MIDDLE); -module_exit(mlx4_ib_cleanup); - -static int -mlx4ib_evhand(module_t mod, int event, void *arg) -{ - return (0); -} - -static moduledata_t mlx4ib_mod = { - .name = "mlx4ib", - .evhand = mlx4ib_evhand, -}; - -DECLARE_MODULE(mlx4ib, mlx4ib_mod, SI_SUB_LAST, SI_ORDER_ANY); -MODULE_DEPEND(mlx4ib, mlx4, 1, 1, 1); -MODULE_DEPEND(mlx4ib, ibcore, 1, 1, 1); -MODULE_DEPEND(mlx4ib, linuxkpi, 1, 1, 1); Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/main.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mlx4_exp.c =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mlx4_exp.c (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mlx4_exp.c (nonexistent) @@ -1,116 +0,0 @@ -/* - * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include "mlx4_ib.h" -#include "mlx4_exp.h" -#include - -int mlx4_ib_exp_query_device(struct ib_device *ibdev, - struct ib_exp_device_attr *props) -{ - struct ib_device_attr *base = &props->base; - struct mlx4_ib_dev *dev = to_mdev(ibdev); - int ret = mlx4_ib_query_device(ibdev, &props->base); - - props->exp_comp_mask = IB_EXP_DEVICE_ATTR_INLINE_RECV_SZ; - props->inline_recv_sz = dev->dev->caps.max_rq_sg * sizeof(struct mlx4_wqe_data_seg); - props->device_cap_flags2 = 0; - - /* move RSS device cap from device_cap to device_cap_flags2 */ - if (base->device_cap_flags & IB_DEVICE_QPG) { - props->device_cap_flags2 |= IB_EXP_DEVICE_QPG; - if (base->device_cap_flags & IB_DEVICE_UD_RSS) - props->device_cap_flags2 |= IB_EXP_DEVICE_UD_RSS; - } - base->device_cap_flags &= ~(IB_DEVICE_QPG | - IB_DEVICE_UD_RSS | - IB_DEVICE_UD_TSS); - - if (base->max_rss_tbl_sz > 0) { - props->max_rss_tbl_sz = base->max_rss_tbl_sz; - props->exp_comp_mask |= IB_EXP_DEVICE_ATTR_RSS_TBL_SZ; - } else { - props->max_rss_tbl_sz = 0; - props->exp_comp_mask &= ~IB_EXP_DEVICE_ATTR_RSS_TBL_SZ; - } - - if (props->device_cap_flags2) - props->exp_comp_mask |= IB_EXP_DEVICE_ATTR_CAP_FLAGS2; - - return ret; -} - -/* - * Experimental functions - */ -struct ib_qp *mlx4_ib_exp_create_qp(struct ib_pd *pd, - struct ib_exp_qp_init_attr *init_attr, - struct ib_udata *udata) -{ - int rwqe_size; - struct ib_qp *qp; - struct mlx4_ib_qp *mqp; - int use_inlr; - struct mlx4_ib_dev *dev; - - if (init_attr->max_inl_recv && !udata) - return ERR_PTR(-EINVAL); - - use_inlr = mlx4_ib_qp_has_rq((struct ib_qp_init_attr *)init_attr) && - init_attr->max_inl_recv && pd; - if (use_inlr) { - rwqe_size = roundup_pow_of_two(max(1U, init_attr->cap.max_recv_sge)) * - sizeof(struct mlx4_wqe_data_seg); - if (rwqe_size < init_attr->max_inl_recv) { - dev = to_mdev(pd->device); - init_attr->max_inl_recv = min(init_attr->max_inl_recv, - (u32)(dev->dev->caps.max_rq_sg * - sizeof(struct mlx4_wqe_data_seg))); - init_attr->cap.max_recv_sge = roundup_pow_of_two(init_attr->max_inl_recv) / - sizeof(struct mlx4_wqe_data_seg); - } - } else { - init_attr->max_inl_recv = 0; - } - qp = mlx4_ib_create_qp(pd, (struct ib_qp_init_attr *)init_attr, udata); - if (IS_ERR(qp)) - return qp; - - if (use_inlr) { - mqp = to_mqp(qp); - mqp->max_inlr_data = 1 << mqp->rq.wqe_shift; - init_attr->max_inl_recv = mqp->max_inlr_data; - } - - return qp; -} Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mlx4_exp.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/Kconfig =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/Kconfig (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/Kconfig (nonexistent) @@ -1,10 +0,0 @@ -config MLX4_INFINIBAND - tristate "Mellanox ConnectX HCA support" - depends on NETDEVICES && ETHERNET && PCI - select NET_VENDOR_MELLANOX - select MLX4_CORE - ---help--- - This driver provides low-level InfiniBand support for - Mellanox ConnectX PCI Express host channel adapters (HCAs). - This is required to use InfiniBand protocols such as - IP-over-IB or SRP with these devices. Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/Kconfig ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/ah.c =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/ah.c (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/ah.c (nonexistent) @@ -1,198 +0,0 @@ -/* - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include -#include - -#include -#include - -#include "mlx4_ib.h" - -int mlx4_ib_resolve_grh(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah_attr, - u8 *mac, int *is_mcast, u8 port) -{ - struct in6_addr in6; - - *is_mcast = 0; - - memcpy(&in6, ah_attr->grh.dgid.raw, sizeof in6); - if (rdma_link_local_addr(&in6)) - rdma_get_ll_mac(&in6, mac); - else if (rdma_is_multicast_addr(&in6)) { - rdma_get_mcast_mac(&in6, mac); - *is_mcast = 1; - } else - return -EINVAL; - - return 0; -} - -static struct ib_ah *create_ib_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr, - struct mlx4_ib_ah *ah) -{ - struct mlx4_dev *dev = to_mdev(pd->device)->dev; - - ah->av.ib.port_pd = cpu_to_be32(to_mpd(pd)->pdn | (ah_attr->port_num << 24)); - ah->av.ib.g_slid = ah_attr->src_path_bits; - if (ah_attr->ah_flags & IB_AH_GRH) { - ah->av.ib.g_slid |= 0x80; - ah->av.ib.gid_index = ah_attr->grh.sgid_index; - ah->av.ib.hop_limit = ah_attr->grh.hop_limit; - ah->av.ib.sl_tclass_flowlabel |= - cpu_to_be32((ah_attr->grh.traffic_class << 20) | - ah_attr->grh.flow_label); - memcpy(ah->av.ib.dgid, ah_attr->grh.dgid.raw, 16); - } - - ah->av.ib.dlid = cpu_to_be16(ah_attr->dlid); - if (ah_attr->static_rate) { - ah->av.ib.stat_rate = ah_attr->static_rate + MLX4_STAT_RATE_OFFSET; - while (ah->av.ib.stat_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET && - !(1 << ah->av.ib.stat_rate & dev->caps.stat_rate_support)) - --ah->av.ib.stat_rate; - } - ah->av.ib.sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 28); - - return &ah->ibah; -} - -static struct ib_ah *create_iboe_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr, - struct mlx4_ib_ah *ah) -{ - struct mlx4_ib_dev *ibdev = to_mdev(pd->device); - struct mlx4_dev *dev = ibdev->dev; - int is_mcast = 0; - struct in6_addr in6; - u16 vlan_tag; - - memcpy(&in6, ah_attr->grh.dgid.raw, sizeof(in6)); - if (rdma_is_multicast_addr(&in6)) { - is_mcast = 1; - resolve_mcast_mac(&in6, ah->av.eth.mac); - } else { - memcpy(ah->av.eth.mac, ah_attr->dmac, 6); - } - vlan_tag = ah_attr->vlan_id; - if (vlan_tag < 0x1000) - vlan_tag |= (ah_attr->sl & 7) << 13; - ah->av.eth.port_pd = cpu_to_be32(to_mpd(pd)->pdn | (ah_attr->port_num << 24)); - ah->av.eth.gid_index = ah_attr->grh.sgid_index; - ah->av.eth.vlan = cpu_to_be16(vlan_tag); - if (ah_attr->static_rate) { - ah->av.eth.stat_rate = ah_attr->static_rate + MLX4_STAT_RATE_OFFSET; - while (ah->av.eth.stat_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET && - !(1 << ah->av.eth.stat_rate & dev->caps.stat_rate_support)) - --ah->av.eth.stat_rate; - } - - /* - * HW requires multicast LID so we just choose one. - */ - if (is_mcast) - ah->av.ib.dlid = cpu_to_be16(0xc000); - - memcpy(ah->av.eth.dgid, ah_attr->grh.dgid.raw, 16); - ah->av.eth.sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 29); - - return &ah->ibah; -} - -struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr) -{ - struct mlx4_ib_ah *ah; - struct ib_ah *ret; - - ah = kzalloc(sizeof *ah, GFP_ATOMIC); - if (!ah) - return ERR_PTR(-ENOMEM); - - if (rdma_port_get_link_layer(pd->device, ah_attr->port_num) == IB_LINK_LAYER_ETHERNET) { - if (!(ah_attr->ah_flags & IB_AH_GRH)) { - ret = ERR_PTR(-EINVAL); - } else { - /* - * TBD: need to handle the case when we get - * called in an atomic context and there we - * might sleep. We don't expect this - * currently since we're working with link - * local addresses which we can translate - * without going to sleep. - */ - ret = create_iboe_ah(pd, ah_attr, ah); - } - - if (IS_ERR(ret)) - kfree(ah); - - return ret; - } else - return create_ib_ah(pd, ah_attr, ah); /* never fails */ -} - -int mlx4_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr) -{ - struct mlx4_ib_ah *ah = to_mah(ibah); - enum rdma_link_layer ll; - - memset(ah_attr, 0, sizeof *ah_attr); - ah_attr->sl = be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28; - ah_attr->port_num = be32_to_cpu(ah->av.ib.port_pd) >> 24; - ll = rdma_port_get_link_layer(ibah->device, ah_attr->port_num); - ah_attr->dlid = ll == IB_LINK_LAYER_INFINIBAND ? be16_to_cpu(ah->av.ib.dlid) : 0; - if (ah->av.ib.stat_rate) - ah_attr->static_rate = ah->av.ib.stat_rate - MLX4_STAT_RATE_OFFSET; - ah_attr->src_path_bits = ah->av.ib.g_slid & 0x7F; - - if (mlx4_ib_ah_grh_present(ah)) { - ah_attr->ah_flags = IB_AH_GRH; - - ah_attr->grh.traffic_class = - be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20; - ah_attr->grh.flow_label = - be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) & 0xfffff; - ah_attr->grh.hop_limit = ah->av.ib.hop_limit; - ah_attr->grh.sgid_index = ah->av.ib.gid_index; - memcpy(ah_attr->grh.dgid.raw, ah->av.ib.dgid, 16); - } - - return 0; -} - -int mlx4_ib_destroy_ah(struct ib_ah *ah) -{ - kfree(to_mah(ah)); - return 0; -} Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/ah.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mlx4_exp.h =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mlx4_exp.h (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mlx4_exp.h (nonexistent) @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2006, 2007 Cisco Systems. All rights reserved. - * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MLX4_EXP_H -#define MLX4_EXP_H - -#include -#include "mlx4_ib.h" - -struct ib_qp *mlx4_ib_exp_create_qp(struct ib_pd *pd, - struct ib_exp_qp_init_attr *init_attr, - struct ib_udata *udata); -int mlx4_ib_exp_query_device(struct ib_device *ibdev, - struct ib_exp_device_attr *props); - -#endif /* MLX4_EXP_H */ Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/mlx4_exp.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/cq.c =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/cq.c (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/cq.c (nonexistent) @@ -1,1009 +0,0 @@ -/* - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include - -#include "mlx4_ib.h" -#include "user.h" - -/* Which firmware version adds support for Resize CQ */ -#define MLX4_FW_VER_RESIZE_CQ mlx4_fw_ver(2, 5, 0) -#define MLX4_FW_VER_IGNORE_OVERRUN_CQ mlx4_fw_ver(2, 7, 8200) - -static void mlx4_ib_cq_comp(struct mlx4_cq *cq) -{ - struct ib_cq *ibcq = &to_mibcq(cq)->ibcq; - ibcq->comp_handler(ibcq, ibcq->cq_context); -} - -static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type) -{ - struct ib_event event; - struct ib_cq *ibcq; - - if (type != MLX4_EVENT_TYPE_CQ_ERROR) { - pr_warn("Unexpected event type %d " - "on CQ %06x\n", type, cq->cqn); - return; - } - - ibcq = &to_mibcq(cq)->ibcq; - if (ibcq->event_handler) { - event.device = ibcq->device; - event.event = IB_EVENT_CQ_ERR; - event.element.cq = ibcq; - ibcq->event_handler(&event, ibcq->cq_context); - } -} - -static void *get_cqe_from_buf(struct mlx4_ib_cq_buf *buf, int n) -{ - return mlx4_buf_offset(&buf->buf, n * buf->entry_size); -} - -static void *get_cqe(struct mlx4_ib_cq *cq, int n) -{ - return get_cqe_from_buf(&cq->buf, n); -} - -static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n) -{ - struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe); - struct mlx4_cqe *tcqe = ((cq->buf.entry_size == 64) ? (cqe + 1) : cqe); - - return (!!(tcqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^ - !!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe; -} - -static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq) -{ - return get_sw_cqe(cq, cq->mcq.cons_index); -} - -int mlx4_ib_modify_cq(struct ib_cq *cq, - struct ib_cq_attr *cq_attr, - int cq_attr_mask) -{ - int err = 0; - struct mlx4_ib_cq *mcq = to_mcq(cq); - struct mlx4_ib_dev *dev = to_mdev(cq->device); - - if (cq_attr_mask & IB_CQ_CAP_FLAGS) { - if (cq_attr->cq_cap_flags & IB_CQ_TIMESTAMP) - return -ENOTSUPP; - - if (cq_attr->cq_cap_flags & IB_CQ_IGNORE_OVERRUN) { - if (dev->dev->caps.cq_flags & MLX4_DEV_CAP_CQ_FLAG_IO) - err = mlx4_cq_ignore_overrun(dev->dev, &mcq->mcq); - else - err = -ENOSYS; - } - } - - if (!err) - if (cq_attr_mask & IB_CQ_MODERATION) - err = mlx4_cq_modify(dev->dev, &mcq->mcq, - cq_attr->moderation.cq_count, - cq_attr->moderation.cq_period); - - return err; -} - -static int mlx4_ib_alloc_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *buf, int nent) -{ - int err; - - err = mlx4_buf_alloc(dev->dev, nent * dev->dev->caps.cqe_size, - PAGE_SIZE * 2, &buf->buf); - - if (err) - goto out; - - buf->entry_size = dev->dev->caps.cqe_size; - err = mlx4_mtt_init(dev->dev, buf->buf.npages, buf->buf.page_shift, - &buf->mtt); - if (err) - goto err_buf; - - err = mlx4_buf_write_mtt(dev->dev, &buf->mtt, &buf->buf); - if (err) - goto err_mtt; - - return 0; - -err_mtt: - mlx4_mtt_cleanup(dev->dev, &buf->mtt); - -err_buf: - mlx4_buf_free(dev->dev, nent * buf->entry_size, &buf->buf); - -out: - return err; -} - -static void mlx4_ib_free_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *buf, int cqe) -{ - mlx4_buf_free(dev->dev, (cqe + 1) * buf->entry_size, &buf->buf); -} - -static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev, struct ib_ucontext *context, - struct mlx4_ib_cq_buf *buf, struct ib_umem **umem, - u64 buf_addr, int cqe) -{ - int err; - int cqe_size = dev->dev->caps.cqe_size; - int shift; - int n; - - *umem = ib_umem_get(context, buf_addr, cqe * cqe_size, - IB_ACCESS_LOCAL_WRITE, 1); - if (IS_ERR(*umem)) - return PTR_ERR(*umem); - - n = ib_umem_page_count(*umem); - shift = mlx4_ib_umem_calc_optimal_mtt_size(*umem, 0, &n); - err = mlx4_mtt_init(dev->dev, n, shift, &buf->mtt); - - if (err) - goto err_buf; - - err = mlx4_ib_umem_write_mtt(dev, &buf->mtt, *umem); - if (err) - goto err_mtt; - - return 0; - -err_mtt: - mlx4_mtt_cleanup(dev->dev, &buf->mtt); - -err_buf: - ib_umem_release(*umem); - - return err; -} - -/* we don't support system timestamping */ -#define CQ_CREATE_FLAGS_SUPPORTED IB_CQ_TIMESTAMP - -struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, - struct ib_cq_init_attr *attr, - struct ib_ucontext *context, - struct ib_udata *udata) -{ - struct mlx4_ib_dev *dev = to_mdev(ibdev); - struct mlx4_ib_cq *cq; - struct mlx4_uar *uar; - int err; - int entries = attr->cqe; - int vector = attr->comp_vector; - - if (entries < 1 || entries > dev->dev->caps.max_cqes) - return ERR_PTR(-EINVAL); - - if (attr->flags & ~CQ_CREATE_FLAGS_SUPPORTED) - return ERR_PTR(-EINVAL); - - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) - return ERR_PTR(-ENOMEM); - - entries = roundup_pow_of_two(entries + 1); - cq->ibcq.cqe = entries - 1; - mutex_init(&cq->resize_mutex); - spin_lock_init(&cq->lock); - cq->resize_buf = NULL; - cq->resize_umem = NULL; - cq->create_flags = attr->flags; - - if (context) { - struct mlx4_ib_create_cq ucmd; - - if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { - err = -EFAULT; - goto err_cq; - } - - err = mlx4_ib_get_cq_umem(dev, context, &cq->buf, &cq->umem, - ucmd.buf_addr, entries); - if (err) - goto err_cq; - - err = mlx4_ib_db_map_user(to_mucontext(context), ucmd.db_addr, - &cq->db); - if (err) - goto err_mtt; - - uar = &to_mucontext(context)->uar; - } else { - err = mlx4_db_alloc(dev->dev, &cq->db, 1); - if (err) - goto err_cq; - - cq->mcq.set_ci_db = cq->db.db; - cq->mcq.arm_db = cq->db.db + 1; - *cq->mcq.set_ci_db = 0; - *cq->mcq.arm_db = 0; - - err = mlx4_ib_alloc_cq_buf(dev, &cq->buf, entries); - if (err) - goto err_db; - - uar = &dev->priv_uar; - } - - if (dev->eq_table) - vector = dev->eq_table[vector % ibdev->num_comp_vectors]; - - err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar, - cq->db.dma, &cq->mcq, vector, 0, - !!(cq->create_flags & IB_CQ_TIMESTAMP)); - if (err) - goto err_dbmap; - - cq->mcq.comp = mlx4_ib_cq_comp; - cq->mcq.event = mlx4_ib_cq_event; - - if (context) - if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) { - err = -EFAULT; - goto err_dbmap; - } - - return &cq->ibcq; - -err_dbmap: - if (context) - mlx4_ib_db_unmap_user(to_mucontext(context), &cq->db); - -err_mtt: - mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt); - - if (context) - ib_umem_release(cq->umem); - else - mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); - -err_db: - if (!context) - mlx4_db_free(dev->dev, &cq->db); - -err_cq: - kfree(cq); - - return ERR_PTR(err); -} - -static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq, - int entries) -{ - int err; - - if (cq->resize_buf) - return -EBUSY; - - cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC); - if (!cq->resize_buf) - return -ENOMEM; - - err = mlx4_ib_alloc_cq_buf(dev, &cq->resize_buf->buf, entries); - if (err) { - kfree(cq->resize_buf); - cq->resize_buf = NULL; - return err; - } - - cq->resize_buf->cqe = entries - 1; - - return 0; -} - -static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq, - int entries, struct ib_udata *udata) -{ - struct mlx4_ib_resize_cq ucmd; - int err; - - if (cq->resize_umem) - return -EBUSY; - - if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) - return -EFAULT; - - cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC); - if (!cq->resize_buf) - return -ENOMEM; - - err = mlx4_ib_get_cq_umem(dev, cq->umem->context, &cq->resize_buf->buf, - &cq->resize_umem, ucmd.buf_addr, entries); - if (err) { - kfree(cq->resize_buf); - cq->resize_buf = NULL; - return err; - } - - cq->resize_buf->cqe = entries - 1; - - return 0; -} - -static int mlx4_ib_get_outstanding_cqes(struct mlx4_ib_cq *cq) -{ - u32 i; - - i = cq->mcq.cons_index; - while (get_sw_cqe(cq, i)) - ++i; - - return i - cq->mcq.cons_index; -} - -static int mlx4_ib_cq_resize_copy_cqes(struct mlx4_ib_cq *cq) -{ - struct mlx4_cqe *cqe, *new_cqe; - int i; - int cqe_size = cq->buf.entry_size; - int cqe_inc = cqe_size == 64 ? 1 : 0; - struct mlx4_cqe *start_cqe; - - i = cq->mcq.cons_index; - cqe = get_cqe(cq, i & cq->ibcq.cqe); - start_cqe = cqe; - cqe += cqe_inc; - - while ((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) != MLX4_CQE_OPCODE_RESIZE) { - new_cqe = get_cqe_from_buf(&cq->resize_buf->buf, - (i + 1) & cq->resize_buf->cqe); - memcpy(new_cqe, get_cqe(cq, i & cq->ibcq.cqe), cqe_size); - new_cqe += cqe_inc; - - new_cqe->owner_sr_opcode = (cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK) | - (((i + 1) & (cq->resize_buf->cqe + 1)) ? MLX4_CQE_OWNER_MASK : 0); - cqe = get_cqe(cq, ++i & cq->ibcq.cqe); - if (cqe == start_cqe) { - pr_warn("resize CQ failed to get resize CQE, CQN 0x%x\n", cq->mcq.cqn); - return -ENOMEM; - } - cqe += cqe_inc; - - } - ++cq->mcq.cons_index; - return 0; -} - -int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) -{ - struct mlx4_ib_dev *dev = to_mdev(ibcq->device); - struct mlx4_ib_cq *cq = to_mcq(ibcq); - struct mlx4_mtt mtt; - int outst_cqe; - int err; - - if (dev->dev->caps.fw_ver < MLX4_FW_VER_RESIZE_CQ) - return -ENOSYS; - - mutex_lock(&cq->resize_mutex); - if (entries < 1 || entries > dev->dev->caps.max_cqes) { - err = -EINVAL; - goto out; - } - - entries = roundup_pow_of_two(entries + 1); - if (entries == ibcq->cqe + 1) { - err = 0; - goto out; - } - - if (entries > dev->dev->caps.max_cqes + 1) { - err = -EINVAL; - goto out; - } - - if (ibcq->uobject) { - err = mlx4_alloc_resize_umem(dev, cq, entries, udata); - if (err) - goto out; - } else { - /* Can't be smaller than the number of outstanding CQEs */ - outst_cqe = mlx4_ib_get_outstanding_cqes(cq); - if (entries < outst_cqe + 1) { - err = 0; - goto out; - } - - err = mlx4_alloc_resize_buf(dev, cq, entries); - if (err) - goto out; - } - - mtt = cq->buf.mtt; - - err = mlx4_cq_resize(dev->dev, &cq->mcq, entries, &cq->resize_buf->buf.mtt); - if (err) - goto err_buf; - - mlx4_mtt_cleanup(dev->dev, &mtt); - if (ibcq->uobject) { - cq->buf = cq->resize_buf->buf; - cq->ibcq.cqe = cq->resize_buf->cqe; - ib_umem_release(cq->umem); - cq->umem = cq->resize_umem; - - kfree(cq->resize_buf); - cq->resize_buf = NULL; - cq->resize_umem = NULL; - } else { - struct mlx4_ib_cq_buf tmp_buf; - int tmp_cqe = 0; - - spin_lock_irq(&cq->lock); - if (cq->resize_buf) { - err = mlx4_ib_cq_resize_copy_cqes(cq); - tmp_buf = cq->buf; - tmp_cqe = cq->ibcq.cqe; - cq->buf = cq->resize_buf->buf; - cq->ibcq.cqe = cq->resize_buf->cqe; - - kfree(cq->resize_buf); - cq->resize_buf = NULL; - } - spin_unlock_irq(&cq->lock); - - if (tmp_cqe) - mlx4_ib_free_cq_buf(dev, &tmp_buf, tmp_cqe); - } - - goto out; - -err_buf: - mlx4_mtt_cleanup(dev->dev, &cq->resize_buf->buf.mtt); - if (!ibcq->uobject) - mlx4_ib_free_cq_buf(dev, &cq->resize_buf->buf, - cq->resize_buf->cqe); - - kfree(cq->resize_buf); - cq->resize_buf = NULL; - - if (cq->resize_umem) { - ib_umem_release(cq->resize_umem); - cq->resize_umem = NULL; - } - -out: - mutex_unlock(&cq->resize_mutex); - - return err; -} - -int mlx4_ib_ignore_overrun_cq(struct ib_cq *ibcq) -{ - struct mlx4_ib_dev *dev = to_mdev(ibcq->device); - struct mlx4_ib_cq *cq = to_mcq(ibcq); - - if (dev->dev->caps.fw_ver < MLX4_FW_VER_IGNORE_OVERRUN_CQ) - return -ENOSYS; - - return mlx4_cq_ignore_overrun(dev->dev, &cq->mcq); -} - -int mlx4_ib_destroy_cq(struct ib_cq *cq) -{ - struct mlx4_ib_dev *dev = to_mdev(cq->device); - struct mlx4_ib_cq *mcq = to_mcq(cq); - - mlx4_cq_free(dev->dev, &mcq->mcq); - mlx4_mtt_cleanup(dev->dev, &mcq->buf.mtt); - - if (cq->uobject) { - mlx4_ib_db_unmap_user(to_mucontext(cq->uobject->context), &mcq->db); - ib_umem_release(mcq->umem); - } else { - mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe); - mlx4_db_free(dev->dev, &mcq->db); - } - - kfree(mcq); - - return 0; -} - -static void dump_cqe(void *cqe) -{ - __be32 *buf = cqe; - - pr_debug("CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n", - be32_to_cpu(buf[0]), be32_to_cpu(buf[1]), be32_to_cpu(buf[2]), - be32_to_cpu(buf[3]), be32_to_cpu(buf[4]), be32_to_cpu(buf[5]), - be32_to_cpu(buf[6]), be32_to_cpu(buf[7])); -} - -static void mlx4_ib_handle_error_cqe(struct mlx4_err_cqe *cqe, - struct ib_wc *wc) -{ - if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR) { - pr_debug("local QP operation err " - "(QPN %06x, WQE index %x, vendor syndrome %02x, " - "opcode = %02x)\n", - be32_to_cpu(cqe->my_qpn), be16_to_cpu(cqe->wqe_index), - cqe->vendor_err_syndrome, - cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK); - dump_cqe(cqe); - } - - switch (cqe->syndrome) { - case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR: - wc->status = IB_WC_LOC_LEN_ERR; - break; - case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR: - wc->status = IB_WC_LOC_QP_OP_ERR; - break; - case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR: - wc->status = IB_WC_LOC_PROT_ERR; - break; - case MLX4_CQE_SYNDROME_WR_FLUSH_ERR: - wc->status = IB_WC_WR_FLUSH_ERR; - break; - case MLX4_CQE_SYNDROME_MW_BIND_ERR: - wc->status = IB_WC_MW_BIND_ERR; - break; - case MLX4_CQE_SYNDROME_BAD_RESP_ERR: - wc->status = IB_WC_BAD_RESP_ERR; - break; - case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR: - wc->status = IB_WC_LOC_ACCESS_ERR; - break; - case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR: - wc->status = IB_WC_REM_INV_REQ_ERR; - break; - case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR: - wc->status = IB_WC_REM_ACCESS_ERR; - break; - case MLX4_CQE_SYNDROME_REMOTE_OP_ERR: - wc->status = IB_WC_REM_OP_ERR; - break; - case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR: - wc->status = IB_WC_RETRY_EXC_ERR; - break; - case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR: - wc->status = IB_WC_RNR_RETRY_EXC_ERR; - break; - case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR: - wc->status = IB_WC_REM_ABORT_ERR; - break; - default: - wc->status = IB_WC_GENERAL_ERR; - break; - } - - wc->vendor_err = cqe->vendor_err_syndrome; -} - -static int mlx4_ib_ipoib_csum_ok(__be16 status, __be16 checksum) -{ - return ((status & cpu_to_be16(MLX4_CQE_STATUS_IPV4 | - MLX4_CQE_STATUS_IPV4F | - MLX4_CQE_STATUS_IPV4OPT | - MLX4_CQE_STATUS_IPV6 | - MLX4_CQE_STATUS_IPOK)) == - cpu_to_be16(MLX4_CQE_STATUS_IPV4 | - MLX4_CQE_STATUS_IPOK)) && - (status & cpu_to_be16(MLX4_CQE_STATUS_UDP | - MLX4_CQE_STATUS_TCP)) && - checksum == cpu_to_be16(0xffff); -} - -static int use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct ib_wc *wc, - unsigned tail, struct mlx4_cqe *cqe, int is_eth) -{ - struct mlx4_ib_proxy_sqp_hdr *hdr; - - ib_dma_sync_single_for_cpu(qp->ibqp.device, - qp->sqp_proxy_rcv[tail].map, - sizeof (struct mlx4_ib_proxy_sqp_hdr), - DMA_FROM_DEVICE); - hdr = (struct mlx4_ib_proxy_sqp_hdr *) (qp->sqp_proxy_rcv[tail].addr); - wc->pkey_index = be16_to_cpu(hdr->tun.pkey_index); - wc->src_qp = be32_to_cpu(hdr->tun.flags_src_qp) & 0xFFFFFF; - wc->wc_flags |= (hdr->tun.g_ml_path & 0x80) ? (IB_WC_GRH) : 0; - wc->dlid_path_bits = 0; - - if (is_eth) { - wc->vlan_id = be16_to_cpu(hdr->tun.sl_vid); - memcpy(&(wc->smac[0]), (char *)&hdr->tun.mac_31_0, 4); - memcpy(&(wc->smac[4]), (char *)&hdr->tun.slid_mac_47_32, 2); - } else { - wc->slid = be16_to_cpu(hdr->tun.slid_mac_47_32); - wc->sl = (u8) (be16_to_cpu(hdr->tun.sl_vid) >> 12); - } - - return 0; -} - -static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq, - struct mlx4_ib_qp **cur_qp, - struct ib_wc *wc) -{ - struct mlx4_cqe *cqe; - struct mlx4_qp *mqp; - struct mlx4_ib_wq *wq; - struct mlx4_ib_srq *srq; - struct mlx4_srq *msrq = NULL; - int is_send; - int is_error; - u32 g_mlpath_rqpn; - u16 wqe_ctr; - unsigned tail = 0; - int timestamp_en = !!(cq->create_flags & IB_CQ_TIMESTAMP); - - -repoll: - cqe = next_cqe_sw(cq); - if (!cqe) - return -EAGAIN; - - if (cq->buf.entry_size == 64) - cqe++; - - ++cq->mcq.cons_index; - - /* - * Make sure we read CQ entry contents after we've checked the - * ownership bit. - */ - rmb(); - - is_send = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK; - is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == - MLX4_CQE_OPCODE_ERROR; - - if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_OPCODE_NOP && - is_send)) { - pr_warn("Completion for NOP opcode detected!\n"); - return -EINVAL; - } - - /* Resize CQ in progress */ - if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_CQE_OPCODE_RESIZE)) { - if (cq->resize_buf) { - struct mlx4_ib_dev *dev = to_mdev(cq->ibcq.device); - - mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); - cq->buf = cq->resize_buf->buf; - cq->ibcq.cqe = cq->resize_buf->cqe; - - kfree(cq->resize_buf); - cq->resize_buf = NULL; - } - - goto repoll; - } - - if (!*cur_qp || - (be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK) != (*cur_qp)->mqp.qpn) { - /* - * We do not have to take the QP table lock here, - * because CQs will be locked while QPs are removed - * from the table. - */ - mqp = __mlx4_qp_lookup(to_mdev(cq->ibcq.device)->dev, - be32_to_cpu(cqe->vlan_my_qpn)); - if (unlikely(!mqp)) { - pr_warn("CQ %06x with entry for unknown QPN %06x\n", - cq->mcq.cqn, be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK); - return -EINVAL; - } - - *cur_qp = to_mibqp(mqp); - } - - wc->qp = &(*cur_qp)->ibqp; - - if (wc->qp->qp_type == IB_QPT_XRC_TGT) { - u32 srq_num; - g_mlpath_rqpn = be32_to_cpu(cqe->g_mlpath_rqpn); - srq_num = g_mlpath_rqpn & 0xffffff; - /* SRQ is also in the radix tree */ - msrq = mlx4_srq_lookup(to_mdev(cq->ibcq.device)->dev, - srq_num); - if (unlikely(!msrq)) { - pr_warn("CQ %06x with entry for unknown SRQN %06x\n", - cq->mcq.cqn, srq_num); - return -EINVAL; - } - } - - if (is_send) { - wq = &(*cur_qp)->sq; - if (!(*cur_qp)->sq_signal_bits) { - wqe_ctr = be16_to_cpu(cqe->wqe_index); - wq->tail += (u16) (wqe_ctr - (u16) wq->tail); - } - wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; - ++wq->tail; - } else if ((*cur_qp)->ibqp.srq) { - srq = to_msrq((*cur_qp)->ibqp.srq); - wqe_ctr = be16_to_cpu(cqe->wqe_index); - wc->wr_id = srq->wrid[wqe_ctr]; - mlx4_ib_free_srq_wqe(srq, wqe_ctr); - } else if (msrq) { - srq = to_mibsrq(msrq); - wqe_ctr = be16_to_cpu(cqe->wqe_index); - wc->wr_id = srq->wrid[wqe_ctr]; - mlx4_ib_free_srq_wqe(srq, wqe_ctr); - } else { - wq = &(*cur_qp)->rq; - tail = wq->tail & (wq->wqe_cnt - 1); - wc->wr_id = wq->wrid[tail]; - ++wq->tail; - } - - if (unlikely(is_error)) { - mlx4_ib_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc); - return 0; - } - - wc->status = IB_WC_SUCCESS; - - if (is_send) { - wc->wc_flags = 0; - switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) { - case MLX4_OPCODE_RDMA_WRITE_IMM: - wc->wc_flags |= IB_WC_WITH_IMM; - /* fall through */ - case MLX4_OPCODE_RDMA_WRITE: - wc->opcode = IB_WC_RDMA_WRITE; - break; - case MLX4_OPCODE_SEND_IMM: - wc->wc_flags |= IB_WC_WITH_IMM; - case MLX4_OPCODE_SEND: - case MLX4_OPCODE_SEND_INVAL: - wc->opcode = IB_WC_SEND; - break; - case MLX4_OPCODE_RDMA_READ: - wc->opcode = IB_WC_RDMA_READ; - wc->byte_len = be32_to_cpu(cqe->byte_cnt); - break; - case MLX4_OPCODE_ATOMIC_CS: - wc->opcode = IB_WC_COMP_SWAP; - wc->byte_len = 8; - break; - case MLX4_OPCODE_ATOMIC_FA: - wc->opcode = IB_WC_FETCH_ADD; - wc->byte_len = 8; - break; - case MLX4_OPCODE_MASKED_ATOMIC_CS: - wc->opcode = IB_WC_MASKED_COMP_SWAP; - wc->byte_len = 8; - break; - case MLX4_OPCODE_MASKED_ATOMIC_FA: - wc->opcode = IB_WC_MASKED_FETCH_ADD; - wc->byte_len = 8; - break; - case MLX4_OPCODE_BIND_MW: - wc->opcode = IB_WC_BIND_MW; - break; - case MLX4_OPCODE_LSO: - wc->opcode = IB_WC_LSO; - break; - case MLX4_OPCODE_FMR: - wc->opcode = IB_WC_FAST_REG_MR; - break; - case MLX4_OPCODE_LOCAL_INVAL: - wc->opcode = IB_WC_LOCAL_INV; - break; - } - } else { - wc->byte_len = be32_to_cpu(cqe->byte_cnt); - - switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) { - case MLX4_RECV_OPCODE_RDMA_WRITE_IMM: - wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; - wc->wc_flags = IB_WC_WITH_IMM; - wc->ex.imm_data = cqe->immed_rss_invalid; - break; - case MLX4_RECV_OPCODE_SEND_INVAL: - wc->opcode = IB_WC_RECV; - wc->wc_flags = IB_WC_WITH_INVALIDATE; - wc->ex.invalidate_rkey = be32_to_cpu(cqe->immed_rss_invalid); - break; - case MLX4_RECV_OPCODE_SEND: - wc->opcode = IB_WC_RECV; - wc->wc_flags = 0; - break; - case MLX4_RECV_OPCODE_SEND_IMM: - wc->opcode = IB_WC_RECV; - wc->wc_flags = IB_WC_WITH_IMM; - wc->ex.imm_data = cqe->immed_rss_invalid; - break; - } - - if (mlx4_is_mfunc(to_mdev(cq->ibcq.device)->dev)) { - if ((*cur_qp)->mlx4_ib_qp_type & - (MLX4_IB_QPT_PROXY_SMI_OWNER | - MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) - return use_tunnel_data - (*cur_qp, cq, wc, tail, cqe, - rdma_port_get_link_layer - (wc->qp->device, - (*cur_qp)->port) == - IB_LINK_LAYER_ETHERNET); - } - - if (timestamp_en) { - /* currently, only CQ_CREATE_WITH_TIMESTAMPING_RAW is - * supported. CQ_CREATE_WITH_TIMESTAMPING_SYS isn't - * supported */ - if (cq->create_flags & IB_CQ_TIMESTAMP_TO_SYS_TIME) { - wc->ts.timestamp = 0; - } else { - wc->ts.timestamp = - ((u64)(be32_to_cpu(cqe->timestamp_16_47) - + !cqe->timestamp_0_15) << 16) - | be16_to_cpu(cqe->timestamp_0_15); - wc->wc_flags |= IB_WC_WITH_TIMESTAMP; - } - } else { - wc->wc_flags |= IB_WC_WITH_SLID; - wc->slid = be16_to_cpu(cqe->rlid); - } - g_mlpath_rqpn = be32_to_cpu(cqe->g_mlpath_rqpn); - wc->src_qp = g_mlpath_rqpn & 0xffffff; - wc->dlid_path_bits = (g_mlpath_rqpn >> 24) & 0x7f; - wc->wc_flags |= g_mlpath_rqpn & 0x80000000 ? IB_WC_GRH : 0; - wc->pkey_index = be32_to_cpu(cqe->immed_rss_invalid) & 0x7f; - wc->wc_flags |= mlx4_ib_ipoib_csum_ok(cqe->status, - cqe->checksum) ? IB_WC_IP_CSUM_OK : 0; - if (!timestamp_en) { - if (rdma_port_get_link_layer(wc->qp->device, - (*cur_qp)->port) == - IB_LINK_LAYER_ETHERNET) - wc->sl = be16_to_cpu(cqe->sl_vid) >> 13; - else - wc->sl = be16_to_cpu(cqe->sl_vid) >> 12; - wc->wc_flags |= IB_WC_WITH_SL; - } - if ((be32_to_cpu(cqe->vlan_my_qpn) & - MLX4_CQE_VLAN_PRESENT_MASK) && !timestamp_en) { - wc->vlan_id = be16_to_cpu(cqe->sl_vid) & - MLX4_CQE_VID_MASK; - wc->wc_flags |= IB_WC_WITH_VLAN; - } else { - wc->vlan_id = 0xffff; - } - if (!timestamp_en) { - memcpy(wc->smac, cqe->smac, 6); - wc->wc_flags |= IB_WC_WITH_SMAC; - } - } - - return 0; -} - -int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) -{ - struct mlx4_ib_cq *cq = to_mcq(ibcq); - struct mlx4_ib_qp *cur_qp = NULL; - unsigned long flags; - int npolled; - int err = 0; - - spin_lock_irqsave(&cq->lock, flags); - - for (npolled = 0; npolled < num_entries; ++npolled) { - err = mlx4_ib_poll_one(cq, &cur_qp, wc + npolled); - if (err) - break; - } - - mlx4_cq_set_ci(&cq->mcq); - - spin_unlock_irqrestore(&cq->lock, flags); - - if (err == 0 || err == -EAGAIN) - return npolled; - else - return err; -} - -int mlx4_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) -{ - mlx4_cq_arm(&to_mcq(ibcq)->mcq, - (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? - MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT, - to_mdev(ibcq->device)->priv_uar.map, - MLX4_GET_DOORBELL_LOCK(&to_mdev(ibcq->device)->uar_lock)); - - return 0; -} - -void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) -{ - u32 prod_index; - int nfreed = 0; - struct mlx4_cqe *cqe, *dest; - u8 owner_bit; - int cqe_inc = cq->buf.entry_size == 64 ? 1 : 0; - - /* - * First we need to find the current producer index, so we - * know where to start cleaning from. It doesn't matter if HW - * adds new entries after this loop -- the QP we're worried - * about is already in RESET, so the new entries won't come - * from our QP and therefore don't need to be checked. - */ - for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index) - if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe) - break; - - /* - * Now sweep backwards through the CQ, removing CQ entries - * that match our QP by copying older entries on top of them. - */ - while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) { - cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); - cqe += cqe_inc; - - if ((be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK) == qpn) { - if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)) - mlx4_ib_free_srq_wqe(srq, be16_to_cpu(cqe->wqe_index)); - ++nfreed; - } else if (nfreed) { - dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe); - dest += cqe_inc; - - owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK; - memcpy(dest, cqe, sizeof *cqe); - dest->owner_sr_opcode = owner_bit | - (dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK); - } - } - - if (nfreed) { - cq->mcq.cons_index += nfreed; - /* - * Make sure update of buffer contents is done before - * updating consumer index. - */ - wmb(); - mlx4_cq_set_ci(&cq->mcq); - } -} - -void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) -{ - spin_lock_irq(&cq->lock); - __mlx4_ib_cq_clean(cq, qpn, srq); - spin_unlock_irq(&cq->lock); -} Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/cq.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/cm.c =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/cm.c (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/cm.c (nonexistent) @@ -1,484 +0,0 @@ -/* - * Copyright (c) 2012 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include - -#include -#include -#include -#include - -#include "mlx4_ib.h" - -#define CM_CLEANUP_CACHE_TIMEOUT (5 * HZ) - -struct id_map_entry { - struct rb_node node; - - u32 sl_cm_id; - u32 pv_cm_id; - int slave_id; - int scheduled_delete; - struct mlx4_ib_dev *dev; - - struct list_head list; - struct delayed_work timeout; -}; - -struct cm_generic_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; -}; - -struct cm_sidr_generic_msg { - struct ib_mad_hdr hdr; - __be32 request_id; -}; - -struct cm_req_msg { - unsigned char unused[0x60]; - union ib_gid primary_path_sgid; -}; - - -static void set_local_comm_id(struct ib_mad *mad, u32 cm_id) -{ - if (mad->mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID) { - struct cm_sidr_generic_msg *msg = - (struct cm_sidr_generic_msg *)mad; - msg->request_id = cpu_to_be32(cm_id); - } else if (mad->mad_hdr.attr_id == CM_SIDR_REP_ATTR_ID) { - pr_err("trying to set local_comm_id in SIDR_REP\n"); - return; - } else { - struct cm_generic_msg *msg = (struct cm_generic_msg *)mad; - msg->local_comm_id = cpu_to_be32(cm_id); - } -} - -static u32 get_local_comm_id(struct ib_mad *mad) -{ - if (mad->mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID) { - struct cm_sidr_generic_msg *msg = - (struct cm_sidr_generic_msg *)mad; - return be32_to_cpu(msg->request_id); - } else if (mad->mad_hdr.attr_id == CM_SIDR_REP_ATTR_ID) { - pr_err("trying to set local_comm_id in SIDR_REP\n"); - return -1; - } else { - struct cm_generic_msg *msg = (struct cm_generic_msg *)mad; - return be32_to_cpu(msg->local_comm_id); - } -} - -static void set_remote_comm_id(struct ib_mad *mad, u32 cm_id) -{ - if (mad->mad_hdr.attr_id == CM_SIDR_REP_ATTR_ID) { - struct cm_sidr_generic_msg *msg = - (struct cm_sidr_generic_msg *)mad; - msg->request_id = cpu_to_be32(cm_id); - } else if (mad->mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID) { - pr_err("trying to set remote_comm_id in SIDR_REQ\n"); - return; - } else { - struct cm_generic_msg *msg = (struct cm_generic_msg *)mad; - msg->remote_comm_id = cpu_to_be32(cm_id); - } -} - -static u32 get_remote_comm_id(struct ib_mad *mad) -{ - if (mad->mad_hdr.attr_id == CM_SIDR_REP_ATTR_ID) { - struct cm_sidr_generic_msg *msg = - (struct cm_sidr_generic_msg *)mad; - return be32_to_cpu(msg->request_id); - } else if (mad->mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID) { - pr_err("trying to set remote_comm_id in SIDR_REQ\n"); - return -1; - } else { - struct cm_generic_msg *msg = (struct cm_generic_msg *)mad; - return be32_to_cpu(msg->remote_comm_id); - } -} - -static union ib_gid gid_from_req_msg(struct ib_device *ibdev, struct ib_mad *mad) -{ - struct cm_req_msg *msg = (struct cm_req_msg *)mad; - - return msg->primary_path_sgid; -} - -/* Lock should be taken before called */ -static struct id_map_entry * -id_map_find_by_sl_id(struct ib_device *ibdev, u32 slave_id, u32 sl_cm_id) -{ - struct rb_root *sl_id_map = &to_mdev(ibdev)->sriov.sl_id_map; - struct rb_node *node = sl_id_map->rb_node; - - while (node) { - struct id_map_entry *id_map_entry = - rb_entry(node, struct id_map_entry, node); - - if (id_map_entry->sl_cm_id > sl_cm_id) - node = node->rb_left; - else if (id_map_entry->sl_cm_id < sl_cm_id) - node = node->rb_right; - else if (id_map_entry->slave_id > slave_id) - node = node->rb_left; - else if (id_map_entry->slave_id < slave_id) - node = node->rb_right; - else - return id_map_entry; - } - return NULL; -} - -static void id_map_ent_timeout(struct work_struct *work) -{ - struct delayed_work *delay = to_delayed_work(work); - struct id_map_entry *ent = container_of(delay, struct id_map_entry, timeout); - struct id_map_entry *db_ent, *found_ent; - struct mlx4_ib_dev *dev = ent->dev; - struct mlx4_ib_sriov *sriov = &dev->sriov; - struct rb_root *sl_id_map = &sriov->sl_id_map; - int pv_id = (int) ent->pv_cm_id; - - spin_lock(&sriov->id_map_lock); - db_ent = (struct id_map_entry *)idr_find(&sriov->pv_id_table, pv_id); - if (!db_ent) - goto out; - found_ent = id_map_find_by_sl_id(&dev->ib_dev, ent->slave_id, ent->sl_cm_id); - if (found_ent && found_ent == ent) - rb_erase(&found_ent->node, sl_id_map); - idr_remove(&sriov->pv_id_table, pv_id); - -out: - list_del(&ent->list); - spin_unlock(&sriov->id_map_lock); - kfree(ent); -} - -static void id_map_find_del(struct ib_device *ibdev, int pv_cm_id) -{ - struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; - struct rb_root *sl_id_map = &sriov->sl_id_map; - struct id_map_entry *ent, *found_ent; - - spin_lock(&sriov->id_map_lock); - ent = (struct id_map_entry *)idr_find(&sriov->pv_id_table, pv_cm_id); - if (!ent) - goto out; - found_ent = id_map_find_by_sl_id(ibdev, ent->slave_id, ent->sl_cm_id); - if (found_ent && found_ent == ent) - rb_erase(&found_ent->node, sl_id_map); - idr_remove(&sriov->pv_id_table, pv_cm_id); -out: - spin_unlock(&sriov->id_map_lock); -} - -static void sl_id_map_add(struct ib_device *ibdev, struct id_map_entry *new) -{ - struct rb_root *sl_id_map = &to_mdev(ibdev)->sriov.sl_id_map; - struct rb_node **link = &sl_id_map->rb_node, *parent = NULL; - struct id_map_entry *ent; - int slave_id = new->slave_id; - int sl_cm_id = new->sl_cm_id; - - ent = id_map_find_by_sl_id(ibdev, slave_id, sl_cm_id); - if (ent) { - pr_debug("overriding existing sl_id_map entry (cm_id = %x)\n", - sl_cm_id); - - rb_replace_node(&ent->node, &new->node, sl_id_map); - return; - } - - /* Go to the bottom of the tree */ - while (*link) { - parent = *link; - ent = rb_entry(parent, struct id_map_entry, node); - - if (ent->sl_cm_id > sl_cm_id || (ent->sl_cm_id == sl_cm_id && ent->slave_id > slave_id)) - link = &(*link)->rb_left; - else - link = &(*link)->rb_right; - } - - rb_link_node(&new->node, parent, link); - rb_insert_color(&new->node, sl_id_map); -} - -static struct id_map_entry * -id_map_alloc(struct ib_device *ibdev, int slave_id, u32 sl_cm_id) -{ - int ret, id; - static int next_id; - struct id_map_entry *ent; - struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; - - ent = kmalloc(sizeof (struct id_map_entry), GFP_KERNEL); - if (!ent) { - mlx4_ib_warn(ibdev, "Couldn't allocate id cache entry - out of memory\n"); - return ERR_PTR(-ENOMEM); - } - - ent->sl_cm_id = sl_cm_id; - ent->slave_id = slave_id; - ent->scheduled_delete = 0; - ent->dev = to_mdev(ibdev); - INIT_DELAYED_WORK(&ent->timeout, id_map_ent_timeout); - - do { - spin_lock(&to_mdev(ibdev)->sriov.id_map_lock); - ret = idr_get_new_above(&sriov->pv_id_table, ent, - next_id, &id); - if (!ret) { - next_id = ((unsigned) id + 1) & MAX_IDR_MASK; - ent->pv_cm_id = (u32)id; - sl_id_map_add(ibdev, ent); - } - - spin_unlock(&sriov->id_map_lock); - } while (ret == -EAGAIN && idr_pre_get(&sriov->pv_id_table, GFP_KERNEL)); - /*the function idr_get_new_above can return -ENOSPC, so don't insert in that case.*/ - if (!ret) { - spin_lock(&sriov->id_map_lock); - list_add_tail(&ent->list, &sriov->cm_list); - spin_unlock(&sriov->id_map_lock); - return ent; - } - /*error flow*/ - kfree(ent); - mlx4_ib_warn(ibdev, "No more space in the idr (err:0x%x)\n", ret); - return ERR_PTR(-ENOMEM); -} - -static struct id_map_entry * -id_map_get(struct ib_device *ibdev, int *pv_cm_id, int sl_cm_id, int slave_id) -{ - struct id_map_entry *ent; - struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; - - spin_lock(&sriov->id_map_lock); - if (*pv_cm_id == -1) { - ent = id_map_find_by_sl_id(ibdev, sl_cm_id, slave_id); - if (ent) - *pv_cm_id = (int) ent->pv_cm_id; - } else - ent = (struct id_map_entry *)idr_find(&sriov->pv_id_table, *pv_cm_id); - spin_unlock(&sriov->id_map_lock); - - return ent; -} - -static void schedule_delayed(struct ib_device *ibdev, struct id_map_entry *id) -{ - struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; - unsigned long flags; - - spin_lock(&sriov->id_map_lock); - spin_lock_irqsave(&sriov->going_down_lock, flags); - /*make sure that there is no schedule inside the scheduled work.*/ - if (!sriov->is_going_down) { - id->scheduled_delete = 1; - schedule_delayed_work(&id->timeout, CM_CLEANUP_CACHE_TIMEOUT); - } - spin_unlock_irqrestore(&sriov->going_down_lock, flags); - spin_unlock(&sriov->id_map_lock); -} - -int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id, - struct ib_mad *mad) -{ - struct id_map_entry *id; - u32 sl_cm_id; - int pv_cm_id = -1; - - if (mad->mad_hdr.attr_id == CM_REQ_ATTR_ID || - mad->mad_hdr.attr_id == CM_REP_ATTR_ID || - mad->mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID || - mad->mad_hdr.attr_id == CM_SIDR_REP_ATTR_ID) { - sl_cm_id = get_local_comm_id(mad); - id = id_map_alloc(ibdev, slave_id, sl_cm_id); - if (IS_ERR(id)) { - mlx4_ib_warn(ibdev, "%s: id{slave: %d, sl_cm_id: 0x%x} Failed to id_map_alloc\n", - __func__, slave_id, sl_cm_id); - return PTR_ERR(id); - } - } else if (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID || - mad->mad_hdr.attr_id == CM_SIDR_REP_ATTR_ID) { - return 0; - } else { - sl_cm_id = get_local_comm_id(mad); - id = id_map_get(ibdev, &pv_cm_id, slave_id, sl_cm_id); - } - - if (!id) { - pr_debug("id{slave: %d, sl_cm_id: 0x%x} is NULL!\n", - slave_id, sl_cm_id); - return -EINVAL; - } - - set_local_comm_id(mad, id->pv_cm_id); - - if (mad->mad_hdr.attr_id == CM_DREQ_ATTR_ID) - schedule_delayed(ibdev, id); - else if (mad->mad_hdr.attr_id == CM_DREP_ATTR_ID) - id_map_find_del(ibdev, pv_cm_id); - - return 0; -} - -int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave, - struct ib_mad *mad, int is_eth) -{ - u32 pv_cm_id; - struct id_map_entry *id; - - if (mad->mad_hdr.attr_id == CM_REQ_ATTR_ID || - mad->mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID) { - union ib_gid gid; - - if (is_eth) - return 0; - - gid = gid_from_req_msg(ibdev, mad); - *slave = mlx4_ib_find_real_gid(ibdev, port, gid.global.interface_id); - if (*slave < 0) { - mlx4_ib_warn(ibdev, "failed matching slave_id by gid (0x%llx)\n", - (unsigned long long)gid.global.interface_id); - return -ENOENT; - } - return 0; - } - - pv_cm_id = get_remote_comm_id(mad); - id = id_map_get(ibdev, (int *)&pv_cm_id, -1, -1); - - if (!id) { - pr_debug("Couldn't find an entry for pv_cm_id 0x%x\n", pv_cm_id); - return -ENOENT; - } - - if (!is_eth) - *slave = id->slave_id; - set_remote_comm_id(mad, id->sl_cm_id); - - if (mad->mad_hdr.attr_id == CM_DREQ_ATTR_ID) - schedule_delayed(ibdev, id); - else if (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID || - mad->mad_hdr.attr_id == CM_DREP_ATTR_ID) { - id_map_find_del(ibdev, (int) pv_cm_id); - } - - return 0; -} - -void mlx4_ib_cm_paravirt_init(struct mlx4_ib_dev *dev) -{ - spin_lock_init(&dev->sriov.id_map_lock); - INIT_LIST_HEAD(&dev->sriov.cm_list); - dev->sriov.sl_id_map = RB_ROOT; - idr_init(&dev->sriov.pv_id_table); - idr_pre_get(&dev->sriov.pv_id_table, GFP_KERNEL); -} - -/* slave = -1 ==> all slaves */ -/* TBD -- call paravirt clean for single slave. Need for slave RESET event */ -void mlx4_ib_cm_paravirt_clean(struct mlx4_ib_dev *dev, int slave) -{ - struct mlx4_ib_sriov *sriov = &dev->sriov; - struct rb_root *sl_id_map = &sriov->sl_id_map; - struct list_head lh; - struct rb_node *nd; - int need_flush = 1; - struct id_map_entry *map, *tmp_map; - /* cancel all delayed work queue entries */ - INIT_LIST_HEAD(&lh); - spin_lock(&sriov->id_map_lock); - list_for_each_entry_safe(map, tmp_map, &dev->sriov.cm_list, list) { - if (slave < 0 || slave == map->slave_id) { - if (map->scheduled_delete) - need_flush &= !!cancel_delayed_work(&map->timeout); - } - } - - spin_unlock(&sriov->id_map_lock); - - if (!need_flush) - flush_scheduled_work(); /* make sure all timers were flushed */ - - /* now, remove all leftover entries from databases*/ - spin_lock(&sriov->id_map_lock); - if (slave < 0) { - while (rb_first(sl_id_map)) { - struct id_map_entry *ent = - rb_entry(rb_first(sl_id_map), - struct id_map_entry, node); - - rb_erase(&ent->node, sl_id_map); - idr_remove(&sriov->pv_id_table, (int) ent->pv_cm_id); - } - list_splice_init(&dev->sriov.cm_list, &lh); - } else { - /* first, move nodes belonging to slave to db remove list */ - nd = rb_first(sl_id_map); - while (nd) { - struct id_map_entry *ent = - rb_entry(nd, struct id_map_entry, node); - nd = rb_next(nd); - if (ent->slave_id == slave) - list_move_tail(&ent->list, &lh); - } - /* remove those nodes from databases */ - list_for_each_entry_safe(map, tmp_map, &lh, list) { - rb_erase(&map->node, sl_id_map); - idr_remove(&sriov->pv_id_table, (int) map->pv_cm_id); - } - - /* add remaining nodes from cm_list */ - list_for_each_entry_safe(map, tmp_map, &dev->sriov.cm_list, list) { - if (slave == map->slave_id) - list_move_tail(&map->list, &lh); - } - } - - spin_unlock(&sriov->id_map_lock); - - /* free any map entries left behind due to cancel_delayed_work above */ - list_for_each_entry_safe(map, tmp_map, &lh, list) { - list_del(&map->list); - kfree(map); - } -} Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/cm.c ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/user.h =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4/user.h (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4/user.h (nonexistent) @@ -1,107 +0,0 @@ -/* - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MLX4_IB_USER_H -#define MLX4_IB_USER_H - -#include - -/* - * Increment this value if any changes that break userspace ABI - * compatibility are made. - */ - -#define MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION 3 -#define MLX4_IB_UVERBS_ABI_VERSION 4 - -/* - * Make sure that all structs defined in this file remain laid out so - * that they pack the same way on 32-bit and 64-bit architectures (to - * avoid incompatibility between 32-bit userspace and 64-bit kernels). - * In particular do not use pointer types -- pass pointers in __u64 - * instead. - */ - -struct mlx4_ib_alloc_ucontext_resp_v3 { - __u32 qp_tab_size; - __u16 bf_reg_size; - __u16 bf_regs_per_page; -}; - -struct mlx4_ib_alloc_ucontext_resp { - __u32 dev_caps; - __u32 qp_tab_size; - __u16 bf_reg_size; - __u16 bf_regs_per_page; - __u32 cqe_size; -}; - -struct mlx4_ib_alloc_pd_resp { - __u32 pdn; - __u32 reserved; -}; - -struct mlx4_ib_create_cq { - __u64 buf_addr; - __u64 db_addr; -}; - -struct mlx4_ib_create_cq_resp { - __u32 cqn; - __u32 reserved; -}; - -struct mlx4_ib_resize_cq { - __u64 buf_addr; -}; - -struct mlx4_ib_create_srq { - __u64 buf_addr; - __u64 db_addr; -}; - -struct mlx4_ib_create_srq_resp { - __u32 srqn; - __u32 reserved; -}; - -struct mlx4_ib_create_qp { - __u64 buf_addr; - __u64 db_addr; - __u8 log_sq_bb_count; - __u8 log_sq_stride; - __u8 sq_no_prefetch; - __u8 reserved[5]; -}; - -#endif /* MLX4_IB_USER_H */ Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4/user.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/drivers/infiniband/hw/mlx4 =================================================================== --- stable/11/sys/ofed/drivers/infiniband/hw/mlx4 (revision 329158) +++ stable/11/sys/ofed/drivers/infiniband/hw/mlx4 (nonexistent) Property changes on: stable/11/sys/ofed/drivers/infiniband/hw/mlx4 ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/include/linux/mlx4/srq.h =================================================================== --- stable/11/sys/ofed/include/linux/mlx4/srq.h (revision 329158) +++ stable/11/sys/ofed/include/linux/mlx4/srq.h (nonexistent) @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MLX4_SRQ_H -#define MLX4_SRQ_H - -struct mlx4_wqe_srq_next_seg { - u16 reserved1; - __be16 next_wqe_index; - u32 reserved2[3]; -}; - -struct mlx4_srq *mlx4_srq_lookup(struct mlx4_dev *dev, u32 srqn); - -#endif /* MLX4_SRQ_H */ Property changes on: stable/11/sys/ofed/include/linux/mlx4/srq.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/include/linux/mlx4/driver.h =================================================================== --- stable/11/sys/ofed/include/linux/mlx4/driver.h (revision 329158) +++ stable/11/sys/ofed/include/linux/mlx4/driver.h (nonexistent) @@ -1,136 +0,0 @@ -/* - * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MLX4_DRIVER_H -#define MLX4_DRIVER_H - -#include - -struct mlx4_dev; - -#define MLX4_MAC_MASK 0xffffffffffffULL -#define MLX4_BE_SHORT_MASK cpu_to_be16(0xffff) -#define MLX4_BE_WORD_MASK cpu_to_be32(0xffffffff) - -enum mlx4_dev_event { - MLX4_DEV_EVENT_CATASTROPHIC_ERROR, - MLX4_DEV_EVENT_PORT_UP, - MLX4_DEV_EVENT_PORT_DOWN, - MLX4_DEV_EVENT_PORT_REINIT, - MLX4_DEV_EVENT_PORT_MGMT_CHANGE, - MLX4_DEV_EVENT_SLAVE_INIT, - MLX4_DEV_EVENT_SLAVE_SHUTDOWN, -}; - -struct mlx4_interface { - void * (*add) (struct mlx4_dev *dev); - void (*remove)(struct mlx4_dev *dev, void *context); - void (*event) (struct mlx4_dev *dev, void *context, - enum mlx4_dev_event event, unsigned long param); - void * (*get_dev)(struct mlx4_dev *dev, void *context, u8 port); - struct list_head list; - enum mlx4_protocol protocol; -}; - -enum { - MLX4_MAX_DEVICES = 32, - MLX4_DEVS_TBL_SIZE = MLX4_MAX_DEVICES + 1, - MLX4_DBDF2VAL_STR_SIZE = 512, - MLX4_STR_NAME_SIZE = 64, - MLX4_MAX_BDF_VALS = 2, - MLX4_ENDOF_TBL = -1LL -}; - -struct mlx4_dbdf2val { - u64 dbdf; - int val[MLX4_MAX_BDF_VALS]; -}; - -struct mlx4_range { - int min; - int max; -}; - -/* - * mlx4_dbdf2val_lst struct holds all the data needed to convert - * dbdf-to-value-list string into dbdf-to-value table. - * dbdf-to-value-list string is a comma separated list of dbdf-to-value strings. - * the format of dbdf-to-value string is: "[mmmm:]bb:dd.f-v1[;v2]" - * mmmm - Domain number (optional) - * bb - Bus number - * dd - device number - * f - Function number - * v1 - First value related to the domain-bus-device-function. - * v2 - Second value related to the domain-bus-device-function (optional). - * bb, dd - Two hexadecimal digits without preceding 0x. - * mmmm - Four hexadecimal digits without preceding 0x. - * f - One hexadecimal without preceding 0x. - * v1,v2 - Number with normal convention (e.g 100, 0xd3). - * dbdf-to-value-list string format: - * "[mmmm:]bb:dd.f-v1[;v2],[mmmm:]bb:dd.f-v1[;v2],..." - * - */ -struct mlx4_dbdf2val_lst { - char name[MLX4_STR_NAME_SIZE]; /* String name */ - char str[MLX4_DBDF2VAL_STR_SIZE]; /* dbdf2val list str */ - struct mlx4_dbdf2val tbl[MLX4_DEVS_TBL_SIZE];/* dbdf to value table */ - int num_vals; /* # of vals per dbdf */ - int def_val[MLX4_MAX_BDF_VALS]; /* Default values */ - struct mlx4_range range; /* Valid values range */ -}; - -int mlx4_fill_dbdf2val_tbl(struct mlx4_dbdf2val_lst *dbdf2val_lst); -int mlx4_get_val(struct mlx4_dbdf2val *tbl, struct pci_dev *pdev, int idx, - int *val); - -int mlx4_register_interface(struct mlx4_interface *intf); -void mlx4_unregister_interface(struct mlx4_interface *intf); - -void *mlx4_get_protocol_dev(struct mlx4_dev *dev, enum mlx4_protocol proto, - int port); - -#ifndef ETH_ALEN -#define ETH_ALEN 6 -#endif -static inline u64 mlx4_mac_to_u64(const u8 *addr) -{ - u64 mac = 0; - int i; - - for (i = 0; i < ETH_ALEN; i++) { - mac <<= 8; - mac |= addr[i]; - } - return mac; -} - -#endif /* MLX4_DRIVER_H */ Property changes on: stable/11/sys/ofed/include/linux/mlx4/driver.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/include/linux/mlx4/qp.h =================================================================== --- stable/11/sys/ofed/include/linux/mlx4/qp.h (revision 329158) +++ stable/11/sys/ofed/include/linux/mlx4/qp.h (nonexistent) @@ -1,455 +0,0 @@ -/* - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MLX4_QP_H -#define MLX4_QP_H - -#include - -#include - -#define MLX4_INVALID_LKEY 0x100 - -#define DS_SIZE_ALIGNMENT 16 - -#define SET_BYTE_COUNT(byte_count) cpu_to_be32(byte_count) -#define SET_LSO_MSS(mss_hdr_size) cpu_to_be32(mss_hdr_size) -#define DS_BYTE_COUNT_MASK cpu_to_be32(0x7fffffff) - -enum ib_m_qp_attr_mask { - IB_M_EXT_CLASS_1 = 1 << 28, - IB_M_EXT_CLASS_2 = 1 << 29, - IB_M_EXT_CLASS_3 = 1 << 30, - - IB_M_QP_MOD_VEND_MASK = (IB_M_EXT_CLASS_1 | IB_M_EXT_CLASS_2 | - IB_M_EXT_CLASS_3) -}; - -enum mlx4_qp_optpar { - MLX4_QP_OPTPAR_ALT_ADDR_PATH = 1 << 0, - MLX4_QP_OPTPAR_RRE = 1 << 1, - MLX4_QP_OPTPAR_RAE = 1 << 2, - MLX4_QP_OPTPAR_RWE = 1 << 3, - MLX4_QP_OPTPAR_PKEY_INDEX = 1 << 4, - MLX4_QP_OPTPAR_Q_KEY = 1 << 5, - MLX4_QP_OPTPAR_RNR_TIMEOUT = 1 << 6, - MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH = 1 << 7, - MLX4_QP_OPTPAR_SRA_MAX = 1 << 8, - MLX4_QP_OPTPAR_RRA_MAX = 1 << 9, - MLX4_QP_OPTPAR_PM_STATE = 1 << 10, - MLX4_QP_OPTPAR_RETRY_COUNT = 1 << 12, - MLX4_QP_OPTPAR_RNR_RETRY = 1 << 13, - MLX4_QP_OPTPAR_ACK_TIMEOUT = 1 << 14, - MLX4_QP_OPTPAR_SCHED_QUEUE = 1 << 16, - MLX4_QP_OPTPAR_COUNTER_INDEX = 1 << 20 -}; - -enum mlx4_qp_state { - MLX4_QP_STATE_RST = 0, - MLX4_QP_STATE_INIT = 1, - MLX4_QP_STATE_RTR = 2, - MLX4_QP_STATE_RTS = 3, - MLX4_QP_STATE_SQER = 4, - MLX4_QP_STATE_SQD = 5, - MLX4_QP_STATE_ERR = 6, - MLX4_QP_STATE_SQ_DRAINING = 7, - MLX4_QP_NUM_STATE -}; - -enum { - MLX4_QP_ST_RC = 0x0, - MLX4_QP_ST_UC = 0x1, - MLX4_QP_ST_RD = 0x2, - MLX4_QP_ST_UD = 0x3, - MLX4_QP_ST_XRC = 0x6, - MLX4_QP_ST_MLX = 0x7 -}; - -enum { - MLX4_QP_PM_MIGRATED = 0x3, - MLX4_QP_PM_ARMED = 0x0, - MLX4_QP_PM_REARM = 0x1 -}; - -enum { - /* params1 */ - MLX4_QP_BIT_SRE = 1 << 15, - MLX4_QP_BIT_SWE = 1 << 14, - MLX4_QP_BIT_SAE = 1 << 13, - /* params2 */ - MLX4_QP_BIT_RRE = 1 << 15, - MLX4_QP_BIT_RWE = 1 << 14, - MLX4_QP_BIT_RAE = 1 << 13, - MLX4_QP_BIT_RIC = 1 << 4, - MLX4_QP_BIT_COLL_SYNC_RQ = 1 << 2, - MLX4_QP_BIT_COLL_SYNC_SQ = 1 << 1, - MLX4_QP_BIT_COLL_MASTER = 1 << 0 -}; - -enum { - MLX4_RSS_HASH_XOR = 0, - MLX4_RSS_HASH_TOP = 1, - - MLX4_RSS_UDP_IPV6 = 1 << 0, - MLX4_RSS_UDP_IPV4 = 1 << 1, - MLX4_RSS_TCP_IPV6 = 1 << 2, - MLX4_RSS_IPV6 = 1 << 3, - MLX4_RSS_TCP_IPV4 = 1 << 4, - MLX4_RSS_IPV4 = 1 << 5, - - /* offset of mlx4_rss_context within mlx4_qp_context.pri_path */ - MLX4_RSS_OFFSET_IN_QPC_PRI_PATH = 0x24, - /* offset of being RSS indirection QP within mlx4_qp_context.flags */ - MLX4_RSS_QPC_FLAG_OFFSET = 13, -}; - -struct mlx4_rss_context { - __be32 base_qpn; - __be32 default_qpn; - u16 reserved; - u8 hash_fn; - u8 flags; - __be32 rss_key[10]; - __be32 base_qpn_udp; -}; - -struct mlx4_qp_path { - u8 fl; - u8 vlan_control; - u8 disable_pkey_check; - u8 pkey_index; - u8 counter_index; - u8 grh_mylmc; - __be16 rlid; - u8 ackto; - u8 mgid_index; - u8 static_rate; - u8 hop_limit; - __be32 tclass_flowlabel; - u8 rgid[16]; - u8 sched_queue; - u8 vlan_index; - u8 feup; - u8 fvl_rx; - u8 reserved4[2]; - u8 dmac[6]; -}; - -enum { /* fl */ - MLX4_FL_CV = 1 << 6, - MLX4_FL_ETH_HIDE_CQE_VLAN = 1 << 2, - MLX4_FL_ETH_SRC_CHECK_MC_LB = 1 << 1, - MLX4_FL_ETH_SRC_CHECK_UC_LB = 1 << 0, -}; -enum { /* vlan_control */ - MLX4_VLAN_CTRL_ETH_SRC_CHECK_IF_COUNTER = 1 << 7, - MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED = 1 << 6, - MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED = 1 << 2, - MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED = 1 << 1,/* 802.1p priorty tag*/ - MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED = 1 << 0 -}; - -enum { /* feup */ - MLX4_FEUP_FORCE_ETH_UP = 1 << 6, /* force Eth UP */ - MLX4_FSM_FORCE_ETH_SRC_MAC = 1 << 5, /* force Source MAC */ - MLX4_FVL_FORCE_ETH_VLAN = 1 << 3 /* force Eth vlan */ -}; - -enum { /* fvl_rx */ - MLX4_FVL_RX_FORCE_ETH_VLAN = 1 << 0 /* enforce Eth rx vlan */ -}; - -struct mlx4_qp_context { - __be32 flags; - __be32 pd; - u8 mtu_msgmax; - u8 rq_size_stride; - u8 sq_size_stride; - u8 rlkey; - __be32 usr_page; - __be32 local_qpn; - __be32 remote_qpn; - struct mlx4_qp_path pri_path; - struct mlx4_qp_path alt_path; - __be32 params1; - u32 reserved1; - __be32 next_send_psn; - __be32 cqn_send; - u32 reserved2[2]; - __be32 last_acked_psn; - __be32 ssn; - __be32 params2; - __be32 rnr_nextrecvpsn; - __be32 xrcd; - __be32 cqn_recv; - __be64 db_rec_addr; - __be32 qkey; - __be32 srqn; - __be32 msn; - __be16 rq_wqe_counter; - __be16 sq_wqe_counter; - u32 reserved3[2]; - __be32 param3; - __be32 nummmcpeers_basemkey; - u8 log_page_size; - u8 reserved4[2]; - u8 mtt_base_addr_h; - __be32 mtt_base_addr_l; - u32 reserved5[10]; -}; - -struct mlx4_update_qp_context { - __be64 qp_mask; - __be64 primary_addr_path_mask; - __be64 secondary_addr_path_mask; - u64 reserved1; - struct mlx4_qp_context qp_context; - u64 reserved2[58]; -}; - -enum { - MLX4_UPD_QP_MASK_PM_STATE = 32, - MLX4_UPD_QP_MASK_VSD = 33, -}; - -enum { - MLX4_UPD_QP_PATH_MASK_PKEY_INDEX = 0 + 32, - MLX4_UPD_QP_PATH_MASK_FSM = 1 + 32, - MLX4_UPD_QP_PATH_MASK_MAC_INDEX = 2 + 32, - MLX4_UPD_QP_PATH_MASK_FVL = 3 + 32, - MLX4_UPD_QP_PATH_MASK_CV = 4 + 32, - MLX4_UPD_QP_PATH_MASK_VLAN_INDEX = 5 + 32, - MLX4_UPD_QP_PATH_MASK_ETH_HIDE_CQE_VLAN = 6 + 32, - MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_UNTAGGED = 7 + 32, - MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_1P = 8 + 32, - MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_TAGGED = 9 + 32, - MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_UNTAGGED = 10 + 32, - MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_1P = 11 + 32, - MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_TAGGED = 12 + 32, - MLX4_UPD_QP_PATH_MASK_FEUP = 13 + 32, - MLX4_UPD_QP_PATH_MASK_SCHED_QUEUE = 14 + 32, - MLX4_UPD_QP_PATH_MASK_IF_COUNTER_INDEX = 15 + 32, - MLX4_UPD_QP_PATH_MASK_FVL_RX = 16 + 32, - MLX4_UPD_QP_PATH_MASK_ETH_SRC_CHECK_UC_LB = 18 + 32, - MLX4_UPD_QP_PATH_MASK_ETH_SRC_CHECK_MC_LB = 19 + 32, -}; - -enum { /* param3 */ - MLX4_STRIP_VLAN = 1 << 30 -}; - - -/* Which firmware version adds support for NEC (NoErrorCompletion) bit */ -#define MLX4_FW_VER_WQE_CTRL_NEC mlx4_fw_ver(2, 2, 232) - -enum { - MLX4_WQE_CTRL_OWN = 1 << 31, - MLX4_WQE_CTRL_NEC = 1 << 29, - MLX4_WQE_CTRL_RR = 1 << 6, - MLX4_WQE_CTRL_FENCE = 1 << 6, - MLX4_WQE_CTRL_CQ_UPDATE = 3 << 2, - MLX4_WQE_CTRL_SOLICITED = 1 << 1, - MLX4_WQE_CTRL_IP_CSUM = 1 << 4, - MLX4_WQE_CTRL_TCP_UDP_CSUM = 1 << 5, - MLX4_WQE_CTRL_INS_VLAN = 1 << 6, - MLX4_WQE_CTRL_STRONG_ORDER = 1 << 7, - MLX4_WQE_CTRL_FORCE_LOOPBACK = 1 << 0, -}; - -struct mlx4_wqe_ctrl_seg { - __be32 owner_opcode; - __be16 vlan_tag; - u8 ins_vlan; - u8 fence_size; - /* - * High 24 bits are SRC remote buffer; low 8 bits are flags: - * [7] SO (strong ordering) - * [5] TCP/UDP checksum - * [4] IP checksum - * [3:2] C (generate completion queue entry) - * [1] SE (solicited event) - * [0] FL (force loopback) - */ - union { - __be32 srcrb_flags; - __be16 srcrb_flags16[2]; - }; - /* - * imm is immediate data for send/RDMA write w/ immediate; - * also invalidation key for send with invalidate; input - * modifier for WQEs on CCQs. - */ - __be32 imm; -}; - -enum { - MLX4_WQE_MLX_VL15 = 1 << 17, - MLX4_WQE_MLX_SLR = 1 << 16 -}; - -struct mlx4_wqe_mlx_seg { - u8 owner; - u8 reserved1[2]; - u8 opcode; - __be16 sched_prio; - u8 reserved2; - u8 size; - /* - * [17] VL15 - * [16] SLR - * [15:12] static rate - * [11:8] SL - * [4] ICRC - * [3:2] C - * [0] FL (force loopback) - */ - __be32 flags; - __be16 rlid; - u16 reserved3; -}; - -struct mlx4_wqe_datagram_seg { - __be32 av[8]; - __be32 dqpn; - __be32 qkey; - __be16 vlan; - u8 mac[6]; -}; - -struct mlx4_wqe_lso_seg { - __be32 mss_hdr_size; - __be32 header[0]; -}; - -enum mlx4_wqe_bind_seg_flags2 { - MLX4_WQE_BIND_TYPE_2 = (1<<31), - MLX4_WQE_BIND_ZERO_BASED = (1<<30), -}; - -struct mlx4_wqe_bind_seg { - __be32 flags1; - __be32 flags2; - __be32 new_rkey; - __be32 lkey; - __be64 addr; - __be64 length; -}; - -enum { - MLX4_WQE_FMR_PERM_LOCAL_READ = 1 << 27, - MLX4_WQE_FMR_PERM_LOCAL_WRITE = 1 << 28, - MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_READ = 1 << 29, - MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_WRITE = 1 << 30, - MLX4_WQE_FMR_AND_BIND_PERM_ATOMIC = 1 << 31 -}; - -struct mlx4_wqe_fmr_seg { - __be32 flags; - __be32 mem_key; - __be64 buf_list; - __be64 start_addr; - __be64 reg_len; - __be32 offset; - __be32 page_size; - u32 reserved[2]; -}; - -struct mlx4_wqe_fmr_ext_seg { - u8 flags; - u8 reserved; - __be16 app_mask; - __be16 wire_app_tag; - __be16 mem_app_tag; - __be32 wire_ref_tag_base; - __be32 mem_ref_tag_base; -}; - -struct mlx4_wqe_local_inval_seg { - u64 reserved1; - __be32 mem_key; - u32 reserved2; - u64 reserved3[2]; -}; - -struct mlx4_wqe_raddr_seg { - __be64 raddr; - __be32 rkey; - u32 reserved; -}; - -struct mlx4_wqe_atomic_seg { - __be64 swap_add; - __be64 compare; -}; - -struct mlx4_wqe_masked_atomic_seg { - __be64 swap_add; - __be64 compare; - __be64 swap_add_mask; - __be64 compare_mask; -}; - -struct mlx4_wqe_data_seg { - __be32 byte_count; - __be32 lkey; - __be64 addr; -}; - -enum { - MLX4_INLINE_ALIGN = 64, - MLX4_INLINE_SEG = 1 << 31, -}; - -struct mlx4_wqe_inline_seg { - __be32 byte_count; -}; - -int mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt, - enum mlx4_qp_state cur_state, enum mlx4_qp_state new_state, - struct mlx4_qp_context *context, enum mlx4_qp_optpar optpar, - int sqd_event, struct mlx4_qp *qp); - -int mlx4_qp_query(struct mlx4_dev *dev, struct mlx4_qp *qp, - struct mlx4_qp_context *context); - -int mlx4_qp_to_ready(struct mlx4_dev *dev, struct mlx4_mtt *mtt, - struct mlx4_qp_context *context, - struct mlx4_qp *qp, enum mlx4_qp_state *qp_state); - -static inline struct mlx4_qp *__mlx4_qp_lookup(struct mlx4_dev *dev, u32 qpn) -{ - return radix_tree_lookup(&dev->qp_table_tree, qpn & (dev->caps.num_qps - 1)); -} - -void mlx4_qp_remove(struct mlx4_dev *dev, struct mlx4_qp *qp); - -#endif /* MLX4_QP_H */ Property changes on: stable/11/sys/ofed/include/linux/mlx4/qp.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/include/linux/mlx4/device.h =================================================================== --- stable/11/sys/ofed/include/linux/mlx4/device.h (revision 329158) +++ stable/11/sys/ofed/include/linux/mlx4/device.h (nonexistent) @@ -1,1346 +0,0 @@ -/* - * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MLX4_DEVICE_H -#define MLX4_DEVICE_H - -#include -#include -#include -#include -#include -#include -#include - -#include - -#define MAX_MSIX_P_PORT 17 -#define MAX_MSIX 64 -#define MSIX_LEGACY_SZ 4 -#define MIN_MSIX_P_PORT 5 - -#define MLX4_ROCE_MAX_GIDS 128 -#define MLX4_ROCE_PF_GIDS 16 - -#define MLX4_NUM_UP 8 -#define MLX4_NUM_TC 8 -#define MLX4_MAX_100M_UNITS_VAL 255 /* - * work around: can't set values - * greater then this value when - * using 100 Mbps units. - */ -#define MLX4_RATELIMIT_100M_UNITS 3 /* 100 Mbps */ -#define MLX4_RATELIMIT_1G_UNITS 4 /* 1 Gbps */ -#define MLX4_RATELIMIT_DEFAULT 0x00ff - -#define CORE_CLOCK_MASK 0xffffffffffffULL - -enum { - MLX4_FLAG_MSI_X = 1 << 0, - MLX4_FLAG_OLD_PORT_CMDS = 1 << 1, - MLX4_FLAG_MASTER = 1 << 2, - MLX4_FLAG_SLAVE = 1 << 3, - MLX4_FLAG_SRIOV = 1 << 4, - MLX4_FLAG_DEV_NUM_STR = 1 << 5, - MLX4_FLAG_OLD_REG_MAC = 1 << 6, -}; - -enum { - MLX4_PORT_CAP_IS_SM = 1 << 1, - MLX4_PORT_CAP_DEV_MGMT_SUP = 1 << 19, -}; - -enum { - MLX4_MAX_PORTS = 2, - MLX4_MAX_PORT_PKEYS = 128 -}; - -/* base qkey for use in sriov tunnel-qp/proxy-qp communication. - * These qkeys must not be allowed for general use. This is a 64k range, - * and to test for violation, we use the mask (protect against future chg). - */ -#define MLX4_RESERVED_QKEY_BASE (0xFFFF0000) -#define MLX4_RESERVED_QKEY_MASK (0xFFFF0000) - -enum { - MLX4_BOARD_ID_LEN = 64, - MLX4_VSD_LEN = 208 -}; - -enum { - MLX4_MAX_NUM_PF = 16, - MLX4_MAX_NUM_VF = 64, - MLX4_MFUNC_MAX = 80, - MLX4_MAX_EQ_NUM = 1024, - MLX4_MFUNC_EQ_NUM = 4, - MLX4_MFUNC_MAX_EQES = 8, - MLX4_MFUNC_EQE_MASK = (MLX4_MFUNC_MAX_EQES - 1) -}; - -/* Driver supports 3 different device methods to manage traffic steering: - * -device managed - High level API for ib and eth flow steering. FW is - * managing flow steering tables. - * - B0 steering mode - Common low level API for ib and (if supported) eth. - * - A0 steering mode - Limited low level API for eth. In case of IB, - * B0 mode is in use. - */ -enum { - MLX4_STEERING_MODE_A0, - MLX4_STEERING_MODE_B0, - MLX4_STEERING_MODE_DEVICE_MANAGED -}; - -static inline const char *mlx4_steering_mode_str(int steering_mode) -{ - switch (steering_mode) { - case MLX4_STEERING_MODE_A0: - return "A0 steering"; - - case MLX4_STEERING_MODE_B0: - return "B0 steering"; - - case MLX4_STEERING_MODE_DEVICE_MANAGED: - return "Device managed flow steering"; - - default: - return "Unrecognize steering mode"; - } -} - -enum { - MLX4_DEV_CAP_FLAG_RC = 1LL << 0, - MLX4_DEV_CAP_FLAG_UC = 1LL << 1, - MLX4_DEV_CAP_FLAG_UD = 1LL << 2, - MLX4_DEV_CAP_FLAG_XRC = 1LL << 3, - MLX4_DEV_CAP_FLAG_SRQ = 1LL << 6, - MLX4_DEV_CAP_FLAG_IPOIB_CSUM = 1LL << 7, - MLX4_DEV_CAP_FLAG_BAD_PKEY_CNTR = 1LL << 8, - MLX4_DEV_CAP_FLAG_BAD_QKEY_CNTR = 1LL << 9, - MLX4_DEV_CAP_FLAG_DPDP = 1LL << 12, - MLX4_DEV_CAP_FLAG_BLH = 1LL << 15, - MLX4_DEV_CAP_FLAG_MEM_WINDOW = 1LL << 16, - MLX4_DEV_CAP_FLAG_APM = 1LL << 17, - MLX4_DEV_CAP_FLAG_ATOMIC = 1LL << 18, - MLX4_DEV_CAP_FLAG_RAW_MCAST = 1LL << 19, - MLX4_DEV_CAP_FLAG_UD_AV_PORT = 1LL << 20, - MLX4_DEV_CAP_FLAG_UD_MCAST = 1LL << 21, - MLX4_DEV_CAP_FLAG_IBOE = 1LL << 30, - MLX4_DEV_CAP_FLAG_UC_LOOPBACK = 1LL << 32, - MLX4_DEV_CAP_FLAG_FCS_KEEP = 1LL << 34, - MLX4_DEV_CAP_FLAG_WOL_PORT1 = 1LL << 37, - MLX4_DEV_CAP_FLAG_WOL_PORT2 = 1LL << 38, - MLX4_DEV_CAP_FLAG_UDP_RSS = 1LL << 40, - MLX4_DEV_CAP_FLAG_VEP_UC_STEER = 1LL << 41, - MLX4_DEV_CAP_FLAG_VEP_MC_STEER = 1LL << 42, - MLX4_DEV_CAP_FLAG_CROSS_CHANNEL = 1LL << 44, - MLX4_DEV_CAP_FLAG_COUNTERS = 1LL << 48, - MLX4_DEV_CAP_FLAG_COUNTERS_EXT = 1LL << 49, - MLX4_DEV_CAP_FLAG_SET_PORT_ETH_SCHED = 1LL << 53, - MLX4_DEV_CAP_FLAG_SENSE_SUPPORT = 1LL << 55, - MLX4_DEV_CAP_FLAG_FAST_DROP = 1LL << 57, - MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV = 1LL << 59, - MLX4_DEV_CAP_FLAG_64B_EQE = 1LL << 61, - MLX4_DEV_CAP_FLAG_64B_CQE = 1LL << 62 -}; - -enum { - MLX4_DEV_CAP_FLAG2_RSS = 1LL << 0, - MLX4_DEV_CAP_FLAG2_RSS_TOP = 1LL << 1, - MLX4_DEV_CAP_FLAG2_RSS_XOR = 1LL << 2, - MLX4_DEV_CAP_FLAG2_FS_EN = 1LL << 3, - MLX4_DEV_CAP_FLAG2_FSM = 1LL << 4, - MLX4_DEV_CAP_FLAG2_VLAN_CONTROL = 1LL << 5, - MLX4_DEV_CAP_FLAG2_UPDATE_QP = 1LL << 6, - MLX4_DEV_CAP_FLAG2_LB_SRC_CHK = 1LL << 7, - MLX4_DEV_CAP_FLAG2_DMFS_IPOIB = 1LL << 8, - MLX4_DEV_CAP_FLAG2_ETS_CFG = 1LL << 9, - MLX4_DEV_CAP_FLAG2_ETH_BACKPL_AN_REP = 1LL << 10, - MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN = 1LL << 11, - MLX4_DEV_CAP_FLAG2_RECOVERABLE_ERROR_EVENT = 1LL << 12, - MLX4_DEV_CAP_FLAG2_TS = 1LL << 13, - MLX4_DEV_CAP_FLAG2_DRIVER_VERSION_TO_FW = 1LL << 14, - MLX4_DEV_CAP_FLAG2_REASSIGN_MAC_EN = 1LL << 15, - MLX4_DEV_CAP_FLAG2_VXLAN_OFFLOADS = 1LL << 16, - MLX4_DEV_CAP_FLAG2_FS_EN_NCSI = 1LL << 17, - MLX4_DEV_CAP_FLAG2_80_VFS = 1LL << 18, - MLX4_DEV_CAP_FLAG2_DMFS_TAG_MODE = 1LL << 19, - MLX4_DEV_CAP_FLAG2_ROCEV2 = 1LL << 20, - MLX4_DEV_CAP_FLAG2_ETH_PROT_CTRL = 1LL << 21, - MLX4_DEV_CAP_FLAG2_CQE_STRIDE = 1LL << 22, - MLX4_DEV_CAP_FLAG2_EQE_STRIDE = 1LL << 23, - MLX4_DEV_CAP_FLAG2_UPDATE_QP_SRC_CHECK_LB = 1LL << 24, - MLX4_DEV_CAP_FLAG2_RX_CSUM_MODE = 1LL << 25, - MLX4_DEV_CAP_FLAG2_SYS_EQS = 1LL << 26, -}; - -/* bit enums for an 8-bit flags field indicating special use - * QPs which require special handling in qp_reserve_range. - * Currently, this only includes QPs used by the ETH interface, - * where we expect to use blueflame. These QPs must not have - * bits 6 and 7 set in their qp number. - * - * This enum may use only bits 0..7. - */ -enum { - MLX4_RESERVE_BF_QP = 1 << 7, -}; - -enum { - MLX4_DEV_CAP_CQ_FLAG_IO = 1 << 0 -}; - -enum { - MLX4_QUERY_FUNC_FLAGS_BF_RES_QP = 1LL << 0 -}; - -/* bit enums for an 8-bit flags field indicating special use - * QPs which require special handling in qp_reserve_range. - * Currently, this only includes QPs used by the ETH interface, - * where we expect to use blueflame. These QPs must not have - * bits 6 and 7 set in their qp number. - * - * This enum may use only bits 0..7. - */ -enum { - MLX4_RESERVE_ETH_BF_QP = 1 << 7, -}; - - -enum { - MLX4_DEV_CAP_64B_EQE_ENABLED = 1LL << 0, - MLX4_DEV_CAP_64B_CQE_ENABLED = 1LL << 1 -}; - -enum { - MLX4_USER_DEV_CAP_64B_CQE = 1L << 0 -}; - -enum { - MLX4_FUNC_CAP_64B_EQE_CQE = 1L << 0 -}; - - -#define MLX4_ATTR_EXTENDED_PORT_INFO cpu_to_be16(0xff90) - -enum { - MLX4_BMME_FLAG_WIN_TYPE_2B = 1 << 1, - MLX4_BMME_FLAG_LOCAL_INV = 1 << 6, - MLX4_BMME_FLAG_REMOTE_INV = 1 << 7, - MLX4_BMME_FLAG_TYPE_2_WIN = 1 << 9, - MLX4_BMME_FLAG_RESERVED_LKEY = 1 << 10, - MLX4_BMME_FLAG_FAST_REG_WR = 1 << 11, -}; - -enum mlx4_event { - MLX4_EVENT_TYPE_COMP = 0x00, - MLX4_EVENT_TYPE_PATH_MIG = 0x01, - MLX4_EVENT_TYPE_COMM_EST = 0x02, - MLX4_EVENT_TYPE_SQ_DRAINED = 0x03, - MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE = 0x13, - MLX4_EVENT_TYPE_SRQ_LIMIT = 0x14, - MLX4_EVENT_TYPE_CQ_ERROR = 0x04, - MLX4_EVENT_TYPE_WQ_CATAS_ERROR = 0x05, - MLX4_EVENT_TYPE_EEC_CATAS_ERROR = 0x06, - MLX4_EVENT_TYPE_PATH_MIG_FAILED = 0x07, - MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10, - MLX4_EVENT_TYPE_WQ_ACCESS_ERROR = 0x11, - MLX4_EVENT_TYPE_SRQ_CATAS_ERROR = 0x12, - MLX4_EVENT_TYPE_LOCAL_CATAS_ERROR = 0x08, - MLX4_EVENT_TYPE_PORT_CHANGE = 0x09, - MLX4_EVENT_TYPE_EQ_OVERFLOW = 0x0f, - MLX4_EVENT_TYPE_ECC_DETECT = 0x0e, - MLX4_EVENT_TYPE_CMD = 0x0a, - MLX4_EVENT_TYPE_VEP_UPDATE = 0x19, - MLX4_EVENT_TYPE_COMM_CHANNEL = 0x18, - MLX4_EVENT_TYPE_OP_REQUIRED = 0x1a, - MLX4_EVENT_TYPE_FATAL_WARNING = 0x1b, - MLX4_EVENT_TYPE_FLR_EVENT = 0x1c, - MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT = 0x1d, - MLX4_EVENT_TYPE_RECOVERABLE_ERROR_EVENT = 0x3e, - MLX4_EVENT_TYPE_NONE = 0xff, -}; - -enum { - MLX4_PORT_CHANGE_SUBTYPE_DOWN = 1, - MLX4_PORT_CHANGE_SUBTYPE_ACTIVE = 4 -}; - -enum { - MLX4_RECOVERABLE_ERROR_EVENT_SUBTYPE_BAD_CABLE = 1, - MLX4_RECOVERABLE_ERROR_EVENT_SUBTYPE_UNSUPPORTED_CABLE = 2, -}; - -enum { - MLX4_FATAL_WARNING_SUBTYPE_WARMING = 0, -}; - -enum slave_port_state { - SLAVE_PORT_DOWN = 0, - SLAVE_PENDING_UP, - SLAVE_PORT_UP, -}; - -enum slave_port_gen_event { - SLAVE_PORT_GEN_EVENT_DOWN = 0, - SLAVE_PORT_GEN_EVENT_UP, - SLAVE_PORT_GEN_EVENT_NONE, -}; - -enum slave_port_state_event { - MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN, - MLX4_PORT_STATE_DEV_EVENT_PORT_UP, - MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID, - MLX4_PORT_STATE_IB_EVENT_GID_INVALID, -}; - -enum { - MLX4_PERM_LOCAL_READ = 1 << 10, - MLX4_PERM_LOCAL_WRITE = 1 << 11, - MLX4_PERM_REMOTE_READ = 1 << 12, - MLX4_PERM_REMOTE_WRITE = 1 << 13, - MLX4_PERM_ATOMIC = 1 << 14, - MLX4_PERM_BIND_MW = 1 << 15, -}; - -enum { - MLX4_OPCODE_NOP = 0x00, - MLX4_OPCODE_SEND_INVAL = 0x01, - MLX4_OPCODE_RDMA_WRITE = 0x08, - MLX4_OPCODE_RDMA_WRITE_IMM = 0x09, - MLX4_OPCODE_SEND = 0x0a, - MLX4_OPCODE_SEND_IMM = 0x0b, - MLX4_OPCODE_LSO = 0x0e, - MLX4_OPCODE_RDMA_READ = 0x10, - MLX4_OPCODE_ATOMIC_CS = 0x11, - MLX4_OPCODE_ATOMIC_FA = 0x12, - MLX4_OPCODE_MASKED_ATOMIC_CS = 0x14, - MLX4_OPCODE_MASKED_ATOMIC_FA = 0x15, - MLX4_OPCODE_BIND_MW = 0x18, - MLX4_OPCODE_FMR = 0x19, - MLX4_OPCODE_LOCAL_INVAL = 0x1b, - MLX4_OPCODE_CONFIG_CMD = 0x1f, - - MLX4_RECV_OPCODE_RDMA_WRITE_IMM = 0x00, - MLX4_RECV_OPCODE_SEND = 0x01, - MLX4_RECV_OPCODE_SEND_IMM = 0x02, - MLX4_RECV_OPCODE_SEND_INVAL = 0x03, - - MLX4_CQE_OPCODE_ERROR = 0x1e, - MLX4_CQE_OPCODE_RESIZE = 0x16, -}; - -enum { - MLX4_STAT_RATE_OFFSET = 5 -}; - -enum mlx4_protocol { - MLX4_PROT_IB_IPV6 = 0, - MLX4_PROT_ETH, - MLX4_PROT_IB_IPV4, - MLX4_PROT_FCOE -}; - -enum { - MLX4_MTT_FLAG_PRESENT = 1 -}; - -enum { - MLX4_MAX_MTT_SHIFT = 31 -}; - -enum mlx4_qp_region { - MLX4_QP_REGION_FW = 0, - MLX4_QP_REGION_ETH_ADDR, - MLX4_QP_REGION_FC_ADDR, - MLX4_QP_REGION_FC_EXCH, - MLX4_NUM_QP_REGION -}; - -enum mlx4_port_type { - MLX4_PORT_TYPE_NONE = 0, - MLX4_PORT_TYPE_IB = 1, - MLX4_PORT_TYPE_ETH = 2, - MLX4_PORT_TYPE_AUTO = 3, - MLX4_PORT_TYPE_NA = 4 -}; - -enum mlx4_special_vlan_idx { - MLX4_NO_VLAN_IDX = 0, - MLX4_VLAN_MISS_IDX, - MLX4_VLAN_REGULAR -}; - -enum mlx4_steer_type { - MLX4_MC_STEER = 0, - MLX4_UC_STEER, - MLX4_NUM_STEERS -}; - -enum { - MLX4_NUM_FEXCH = 64 * 1024, -}; - -enum { - MLX4_MAX_FAST_REG_PAGES = 511, -}; - -enum { - MLX4_DEV_PMC_SUBTYPE_GUID_INFO = 0x14, - MLX4_DEV_PMC_SUBTYPE_PORT_INFO = 0x15, - MLX4_DEV_PMC_SUBTYPE_PKEY_TABLE = 0x16, -}; - -/* Port mgmt change event handling */ -enum { - MLX4_EQ_PORT_INFO_MSTR_SM_LID_CHANGE_MASK = 1 << 0, - MLX4_EQ_PORT_INFO_GID_PFX_CHANGE_MASK = 1 << 1, - MLX4_EQ_PORT_INFO_LID_CHANGE_MASK = 1 << 2, - MLX4_EQ_PORT_INFO_CLIENT_REREG_MASK = 1 << 3, - MLX4_EQ_PORT_INFO_MSTR_SM_SL_CHANGE_MASK = 1 << 4, -}; - -#define MSTR_SM_CHANGE_MASK (MLX4_EQ_PORT_INFO_MSTR_SM_SL_CHANGE_MASK | \ - MLX4_EQ_PORT_INFO_MSTR_SM_LID_CHANGE_MASK) - -enum mlx4_module_id { - MLX4_MODULE_ID_SFP = 0x3, - MLX4_MODULE_ID_QSFP = 0xC, - MLX4_MODULE_ID_QSFP_PLUS = 0xD, - MLX4_MODULE_ID_QSFP28 = 0x11, -}; - -static inline u64 mlx4_fw_ver(u64 major, u64 minor, u64 subminor) -{ - return (major << 32) | (minor << 16) | subminor; -} - -struct mlx4_phys_caps { - u32 gid_phys_table_len[MLX4_MAX_PORTS + 1]; - u32 pkey_phys_table_len[MLX4_MAX_PORTS + 1]; - u32 num_phys_eqs; - u32 base_sqpn; - u32 base_proxy_sqpn; - u32 base_tunnel_sqpn; -}; - -struct mlx4_caps { - u64 fw_ver; - u32 function; - int num_ports; - int vl_cap[MLX4_MAX_PORTS + 1]; - int ib_mtu_cap[MLX4_MAX_PORTS + 1]; - __be32 ib_port_def_cap[MLX4_MAX_PORTS + 1]; - u64 def_mac[MLX4_MAX_PORTS + 1]; - int eth_mtu_cap[MLX4_MAX_PORTS + 1]; - int gid_table_len[MLX4_MAX_PORTS + 1]; - int pkey_table_len[MLX4_MAX_PORTS + 1]; - int trans_type[MLX4_MAX_PORTS + 1]; - int vendor_oui[MLX4_MAX_PORTS + 1]; - int wavelength[MLX4_MAX_PORTS + 1]; - u64 trans_code[MLX4_MAX_PORTS + 1]; - int local_ca_ack_delay; - int num_uars; - u32 uar_page_size; - int bf_reg_size; - int bf_regs_per_page; - int max_sq_sg; - int max_rq_sg; - int num_qps; - int max_wqes; - int max_sq_desc_sz; - int max_rq_desc_sz; - int max_qp_init_rdma; - int max_qp_dest_rdma; - u32 *qp0_proxy; - u32 *qp1_proxy; - u32 *qp0_tunnel; - u32 *qp1_tunnel; - int num_srqs; - int max_srq_wqes; - int max_srq_sge; - int reserved_srqs; - int num_cqs; - int max_cqes; - int reserved_cqs; - int num_sys_eqs; - int num_eqs; - int reserved_eqs; - int num_comp_vectors; - int comp_pool; - int num_mpts; - int max_fmr_maps; - u64 num_mtts; - int fmr_reserved_mtts; - int reserved_mtts; - int reserved_mrws; - int reserved_uars; - int num_mgms; - int num_amgms; - int reserved_mcgs; - int num_qp_per_mgm; - int steering_mode; - int num_pds; - int reserved_pds; - int max_xrcds; - int reserved_xrcds; - int mtt_entry_sz; - u32 max_msg_sz; - u32 page_size_cap; - u64 flags; - u64 flags2; - u32 bmme_flags; - u32 reserved_lkey; - u16 stat_rate_support; - u8 cq_timestamp; - u8 port_width_cap[MLX4_MAX_PORTS + 1]; - int max_gso_sz; - int max_rss_tbl_sz; - int reserved_qps_cnt[MLX4_NUM_QP_REGION]; - int reserved_qps; - int reserved_qps_base[MLX4_NUM_QP_REGION]; - int log_num_macs; - int log_num_vlans; - enum mlx4_port_type port_type[MLX4_MAX_PORTS + 1]; - u8 supported_type[MLX4_MAX_PORTS + 1]; - u8 suggested_type[MLX4_MAX_PORTS + 1]; - u8 default_sense[MLX4_MAX_PORTS + 1]; - u32 port_mask[MLX4_MAX_PORTS + 1]; - enum mlx4_port_type possible_type[MLX4_MAX_PORTS + 1]; - u32 max_counters; - u8 port_ib_mtu[MLX4_MAX_PORTS + 1]; - u16 sqp_demux; - u32 sync_qp; - u32 cq_flags; - u32 eqe_size; - u32 cqe_size; - u8 eqe_factor; - u32 userspace_caps; /* userspace must be aware to */ - u32 function_caps; /* functions must be aware to */ - u8 fast_drop; - u16 hca_core_clock; - u32 max_basic_counters; - u32 max_extended_counters; - u8 def_counter_index[MLX4_MAX_PORTS + 1]; - u8 alloc_res_qp_mask; -}; - -struct mlx4_buf_list { - void *buf; - dma_addr_t map; -}; - -struct mlx4_buf { - struct mlx4_buf_list direct; - struct mlx4_buf_list *page_list; - int nbufs; - int npages; - int page_shift; -}; - -struct mlx4_mtt { - u32 offset; - int order; - int page_shift; -}; - -enum { - MLX4_DB_PER_PAGE = PAGE_SIZE / 4 -}; - -struct mlx4_db_pgdir { - struct list_head list; - DECLARE_BITMAP(order0, MLX4_DB_PER_PAGE); - DECLARE_BITMAP(order1, MLX4_DB_PER_PAGE / 2); - unsigned long *bits[2]; - __be32 *db_page; - dma_addr_t db_dma; -}; - -struct mlx4_ib_user_db_page; - -struct mlx4_db { - __be32 *db; - union { - struct mlx4_db_pgdir *pgdir; - struct mlx4_ib_user_db_page *user_page; - } u; - dma_addr_t dma; - int index; - int order; -}; - -struct mlx4_hwq_resources { - struct mlx4_db db; - struct mlx4_mtt mtt; - struct mlx4_buf buf; -}; - -struct mlx4_mr { - struct mlx4_mtt mtt; - u64 iova; - u64 size; - u32 key; - u32 pd; - u32 access; - int enabled; -}; - -enum mlx4_mw_type { - MLX4_MW_TYPE_1 = 1, - MLX4_MW_TYPE_2 = 2, -}; - -struct mlx4_mw { - u32 key; - u32 pd; - enum mlx4_mw_type type; - int enabled; -}; - -struct mlx4_fmr { - struct mlx4_mr mr; - struct mlx4_mpt_entry *mpt; - __be64 *mtts; - dma_addr_t dma_handle; - int max_pages; - int max_maps; - int maps; - u8 page_shift; -}; - -struct mlx4_uar { - unsigned long pfn; - int index; - struct list_head bf_list; - unsigned free_bf_bmap; - void __iomem *map; - void __iomem *bf_map; -}; - -struct mlx4_bf { - unsigned long offset; - int buf_size; - struct mlx4_uar *uar; - void __iomem *reg; -}; - -struct mlx4_cq { - void (*comp) (struct mlx4_cq *); - void (*event) (struct mlx4_cq *, enum mlx4_event); - - struct mlx4_uar *uar; - - u32 cons_index; - - __be32 *set_ci_db; - __be32 *arm_db; - int arm_sn; - - int cqn; - unsigned vector; - - atomic_t refcount; - struct completion free; - int eqn; - u16 irq; -}; - -struct mlx4_qp { - void (*event) (struct mlx4_qp *, enum mlx4_event); - - int qpn; - - atomic_t refcount; - struct completion free; -}; - -struct mlx4_srq { - void (*event) (struct mlx4_srq *, enum mlx4_event); - - int srqn; - int max; - int max_gs; - int wqe_shift; - - atomic_t refcount; - struct completion free; -}; - -struct mlx4_av { - __be32 port_pd; - u8 reserved1; - u8 g_slid; - __be16 dlid; - u8 reserved2; - u8 gid_index; - u8 stat_rate; - u8 hop_limit; - __be32 sl_tclass_flowlabel; - u8 dgid[16]; -}; - -struct mlx4_eth_av { - __be32 port_pd; - u8 reserved1; - u8 smac_idx; - u16 reserved2; - u8 reserved3; - u8 gid_index; - u8 stat_rate; - u8 hop_limit; - __be32 sl_tclass_flowlabel; - u8 dgid[16]; - u8 s_mac[6]; - u8 reserved4[2]; - __be16 vlan; - u8 mac[6]; -}; - -union mlx4_ext_av { - struct mlx4_av ib; - struct mlx4_eth_av eth; -}; - -struct mlx4_if_stat_control { - u8 reserved1[3]; - /* Extended counters enabled */ - u8 cnt_mode; - /* Number of interfaces */ - __be32 num_of_if; - __be32 reserved[2]; -}; - -struct mlx4_if_stat_basic { - struct mlx4_if_stat_control control; - struct { - __be64 IfRxFrames; - __be64 IfRxOctets; - __be64 IfTxFrames; - __be64 IfTxOctets; - } counters[]; -}; -#define MLX4_IF_STAT_BSC_SZ(ports)(sizeof(struct mlx4_if_stat_extended) +\ - sizeof(((struct mlx4_if_stat_extended *)0)->\ - counters[0]) * ports) - -struct mlx4_if_stat_extended { - struct mlx4_if_stat_control control; - struct { - __be64 IfRxUnicastFrames; - __be64 IfRxUnicastOctets; - __be64 IfRxMulticastFrames; - __be64 IfRxMulticastOctets; - __be64 IfRxBroadcastFrames; - __be64 IfRxBroadcastOctets; - __be64 IfRxNoBufferFrames; - __be64 IfRxNoBufferOctets; - __be64 IfRxErrorFrames; - __be64 IfRxErrorOctets; - __be32 reserved[39]; - __be64 IfTxUnicastFrames; - __be64 IfTxUnicastOctets; - __be64 IfTxMulticastFrames; - __be64 IfTxMulticastOctets; - __be64 IfTxBroadcastFrames; - __be64 IfTxBroadcastOctets; - __be64 IfTxDroppedFrames; - __be64 IfTxDroppedOctets; - __be64 IfTxRequestedFramesSent; - __be64 IfTxGeneratedFramesSent; - __be64 IfTxTsoOctets; - } __packed counters[]; -}; -#define MLX4_IF_STAT_EXT_SZ(ports) (sizeof(struct mlx4_if_stat_extended) +\ - sizeof(((struct mlx4_if_stat_extended *)\ - 0)->counters[0]) * ports) - -union mlx4_counter { - struct mlx4_if_stat_control control; - struct mlx4_if_stat_basic basic; - struct mlx4_if_stat_extended ext; -}; -#define MLX4_IF_STAT_SZ(ports) MLX4_IF_STAT_EXT_SZ(ports) - -struct mlx4_quotas { - int qp; - int cq; - int srq; - int mpt; - int mtt; - int counter; - int xrcd; -}; - -struct mlx4_dev { - struct pci_dev *pdev; - unsigned long flags; - unsigned long num_slaves; - struct mlx4_caps caps; - struct mlx4_phys_caps phys_caps; - struct mlx4_quotas quotas; - struct radix_tree_root qp_table_tree; - u8 rev_id; - char board_id[MLX4_BOARD_ID_LEN]; - u16 vsd_vendor_id; - char vsd[MLX4_VSD_LEN]; - int num_vfs; - int numa_node; - int oper_log_mgm_entry_size; - u64 regid_promisc_array[MLX4_MAX_PORTS + 1]; - u64 regid_allmulti_array[MLX4_MAX_PORTS + 1]; -}; - -struct mlx4_clock_params { - u64 offset; - u8 bar; - u8 size; -}; - -struct mlx4_eqe { - u8 reserved1; - u8 type; - u8 reserved2; - u8 subtype; - union { - u32 raw[6]; - struct { - __be32 cqn; - } __packed comp; - struct { - u16 reserved1; - __be16 token; - u32 reserved2; - u8 reserved3[3]; - u8 status; - __be64 out_param; - } __packed cmd; - struct { - __be32 qpn; - } __packed qp; - struct { - __be32 srqn; - } __packed srq; - struct { - __be32 cqn; - u32 reserved1; - u8 reserved2[3]; - u8 syndrome; - } __packed cq_err; - struct { - u32 reserved1[2]; - __be32 port; - } __packed port_change; - struct { - #define COMM_CHANNEL_BIT_ARRAY_SIZE 4 - u32 reserved; - u32 bit_vec[COMM_CHANNEL_BIT_ARRAY_SIZE]; - } __packed comm_channel_arm; - struct { - u8 port; - u8 reserved[3]; - __be64 mac; - } __packed mac_update; - struct { - __be32 slave_id; - } __packed flr_event; - struct { - __be16 current_temperature; - __be16 warning_threshold; - } __packed warming; - struct { - u8 reserved[3]; - u8 port; - union { - struct { - __be16 mstr_sm_lid; - __be16 port_lid; - __be32 changed_attr; - u8 reserved[3]; - u8 mstr_sm_sl; - __be64 gid_prefix; - } __packed port_info; - struct { - __be32 block_ptr; - __be32 tbl_entries_mask; - } __packed tbl_change_info; - } params; - } __packed port_mgmt_change; - struct { - u8 reserved[3]; - u8 port; - u32 reserved1[5]; - } __packed bad_cable; - } event; - u8 slave_id; - u8 reserved3[2]; - u8 owner; -} __packed; - -struct mlx4_init_port_param { - int set_guid0; - int set_node_guid; - int set_si_guid; - u16 mtu; - int port_width_cap; - u16 vl_cap; - u16 max_gid; - u16 max_pkey; - u64 guid0; - u64 node_guid; - u64 si_guid; -}; - -#define MAD_IFC_DATA_SZ 192 -/* MAD IFC Mailbox */ -struct mlx4_mad_ifc { - u8 base_version; - u8 mgmt_class; - u8 class_version; - u8 method; - __be16 status; - __be16 class_specific; - __be64 tid; - __be16 attr_id; - __be16 resv; - __be32 attr_mod; - __be64 mkey; - __be16 dr_slid; - __be16 dr_dlid; - u8 reserved[28]; - u8 data[MAD_IFC_DATA_SZ]; -} __packed; - -#define mlx4_foreach_port(port, dev, type) \ - for ((port) = 1; (port) <= (dev)->caps.num_ports; (port)++) \ - if ((type) == (dev)->caps.port_mask[(port)]) - -#define mlx4_foreach_non_ib_transport_port(port, dev) \ - for ((port) = 1; (port) <= (dev)->caps.num_ports; (port)++) \ - if (((dev)->caps.port_mask[port] != MLX4_PORT_TYPE_IB)) - -#define mlx4_foreach_ib_transport_port(port, dev) \ - for ((port) = 1; (port) <= (dev)->caps.num_ports; (port)++) \ - if (((dev)->caps.port_mask[port] == MLX4_PORT_TYPE_IB) || \ - ((dev)->caps.flags & MLX4_DEV_CAP_FLAG_IBOE)) - -#define MLX4_INVALID_SLAVE_ID 0xFF - -#define MLX4_SINK_COUNTER_INDEX 0xff - -void handle_port_mgmt_change_event(struct work_struct *work); - -static inline int mlx4_master_func_num(struct mlx4_dev *dev) -{ - return dev->caps.function; -} - -static inline int mlx4_is_master(struct mlx4_dev *dev) -{ - return dev->flags & MLX4_FLAG_MASTER; -} - -static inline int mlx4_num_reserved_sqps(struct mlx4_dev *dev) -{ - return dev->phys_caps.base_sqpn + 8 + - 16 * MLX4_MFUNC_MAX * !!mlx4_is_master(dev); -} - -static inline int mlx4_is_qp_reserved(struct mlx4_dev *dev, u32 qpn) -{ - return (qpn < dev->phys_caps.base_sqpn + 8 + - 16 * MLX4_MFUNC_MAX * !!mlx4_is_master(dev)); -} - -static inline int mlx4_is_guest_proxy(struct mlx4_dev *dev, int slave, u32 qpn) -{ - int guest_proxy_base = dev->phys_caps.base_proxy_sqpn + slave * 8; - - if (qpn >= guest_proxy_base && qpn < guest_proxy_base + 8) - return 1; - - return 0; -} - -static inline int mlx4_is_mfunc(struct mlx4_dev *dev) -{ - return dev->flags & (MLX4_FLAG_SLAVE | MLX4_FLAG_MASTER); -} - -static inline int mlx4_is_slave(struct mlx4_dev *dev) -{ - return dev->flags & MLX4_FLAG_SLAVE; -} - -int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, - struct mlx4_buf *buf); -void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf); -static inline void *mlx4_buf_offset(struct mlx4_buf *buf, int offset) -{ - if (BITS_PER_LONG == 64 || buf->nbufs == 1) - return (u8 *)buf->direct.buf + offset; - else - return (u8 *)buf->page_list[offset >> PAGE_SHIFT].buf + - (offset & (PAGE_SIZE - 1)); -} - -int mlx4_pd_alloc(struct mlx4_dev *dev, u32 *pdn); -void mlx4_pd_free(struct mlx4_dev *dev, u32 pdn); -int mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn); -void mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn); - -int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar); -void mlx4_uar_free(struct mlx4_dev *dev, struct mlx4_uar *uar); -int mlx4_bf_alloc(struct mlx4_dev *dev, struct mlx4_bf *bf, int node); -void mlx4_bf_free(struct mlx4_dev *dev, struct mlx4_bf *bf); - -int mlx4_mtt_init(struct mlx4_dev *dev, int npages, int page_shift, - struct mlx4_mtt *mtt); -void mlx4_mtt_cleanup(struct mlx4_dev *dev, struct mlx4_mtt *mtt); -u64 mlx4_mtt_addr(struct mlx4_dev *dev, struct mlx4_mtt *mtt); - -int mlx4_mr_alloc(struct mlx4_dev *dev, u32 pd, u64 iova, u64 size, u32 access, - int npages, int page_shift, struct mlx4_mr *mr); -int mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr); -int mlx4_mr_enable(struct mlx4_dev *dev, struct mlx4_mr *mr); -int mlx4_mw_alloc(struct mlx4_dev *dev, u32 pd, enum mlx4_mw_type type, - struct mlx4_mw *mw); -void mlx4_mw_free(struct mlx4_dev *dev, struct mlx4_mw *mw); -int mlx4_mw_enable(struct mlx4_dev *dev, struct mlx4_mw *mw); -int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, - int start_index, int npages, u64 *page_list); -int mlx4_buf_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, - struct mlx4_buf *buf); - -int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order); -void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db); - -int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, - int size, int max_direct); -void mlx4_free_hwq_res(struct mlx4_dev *mdev, struct mlx4_hwq_resources *wqres, - int size); - -int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt, - struct mlx4_uar *uar, u64 db_rec, struct mlx4_cq *cq, - unsigned vector, int collapsed, int timestamp_en); -void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq); - -int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, - int *base, u8 flags); -void mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt); - -int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp); -void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp); - -int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, u32 cqn, u16 xrcdn, - struct mlx4_mtt *mtt, u64 db_rec, struct mlx4_srq *srq); -void mlx4_srq_free(struct mlx4_dev *dev, struct mlx4_srq *srq); -int mlx4_srq_arm(struct mlx4_dev *dev, struct mlx4_srq *srq, int limit_watermark); -int mlx4_srq_query(struct mlx4_dev *dev, struct mlx4_srq *srq, int *limit_watermark); - -int mlx4_INIT_PORT(struct mlx4_dev *dev, int port); -int mlx4_CLOSE_PORT(struct mlx4_dev *dev, int port); - -int mlx4_unicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], - int block_mcast_loopback, enum mlx4_protocol prot); -int mlx4_unicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], - enum mlx4_protocol prot); -int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], - u8 port, int block_mcast_loopback, - enum mlx4_protocol protocol, u64 *reg_id); -int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], - enum mlx4_protocol protocol, u64 reg_id); - -enum { - MLX4_DOMAIN_UVERBS = 0x1000, - MLX4_DOMAIN_ETHTOOL = 0x2000, - MLX4_DOMAIN_RFS = 0x3000, - MLX4_DOMAIN_NIC = 0x5000, -}; - -enum mlx4_net_trans_rule_id { - MLX4_NET_TRANS_RULE_ID_ETH = 0, - MLX4_NET_TRANS_RULE_ID_IB, - MLX4_NET_TRANS_RULE_ID_IPV6, - MLX4_NET_TRANS_RULE_ID_IPV4, - MLX4_NET_TRANS_RULE_ID_TCP, - MLX4_NET_TRANS_RULE_ID_UDP, - MLX4_NET_TRANS_RULE_NUM, /* should be last */ - MLX4_NET_TRANS_RULE_DUMMY = -1, /* force enum to be signed */ -}; - -extern const u16 __sw_id_hw[]; - -static inline int map_hw_to_sw_id(u16 header_id) -{ - - int i; - for (i = 0; i < MLX4_NET_TRANS_RULE_NUM; i++) { - if (header_id == __sw_id_hw[i]) - return i; - } - return -EINVAL; -} - -enum mlx4_net_trans_promisc_mode { - MLX4_FS_REGULAR = 1, - MLX4_FS_ALL_DEFAULT, - MLX4_FS_MC_DEFAULT, - MLX4_FS_UC_SNIFFER, - MLX4_FS_MC_SNIFFER, - MLX4_FS_MODE_NUM, /* should be last */ - MLX4_FS_MODE_DUMMY = -1, /* force enum to be signed */ -}; - -struct mlx4_spec_eth { - u8 dst_mac[6]; - u8 dst_mac_msk[6]; - u8 src_mac[6]; - u8 src_mac_msk[6]; - u8 ether_type_enable; - __be16 ether_type; - __be16 vlan_id_msk; - __be16 vlan_id; -}; - -struct mlx4_spec_tcp_udp { - __be16 dst_port; - __be16 dst_port_msk; - __be16 src_port; - __be16 src_port_msk; -}; - -struct mlx4_spec_ipv4 { - __be32 dst_ip; - __be32 dst_ip_msk; - __be32 src_ip; - __be32 src_ip_msk; -}; - -struct mlx4_spec_ib { - __be32 l3_qpn; - __be32 qpn_msk; - u8 dst_gid[16]; - u8 dst_gid_msk[16]; -}; - -struct mlx4_spec_list { - struct list_head list; - enum mlx4_net_trans_rule_id id; - union { - struct mlx4_spec_eth eth; - struct mlx4_spec_ib ib; - struct mlx4_spec_ipv4 ipv4; - struct mlx4_spec_tcp_udp tcp_udp; - }; -}; - -enum mlx4_net_trans_hw_rule_queue { - MLX4_NET_TRANS_Q_FIFO, - MLX4_NET_TRANS_Q_LIFO, -}; - -struct mlx4_net_trans_rule { - struct list_head list; - enum mlx4_net_trans_hw_rule_queue queue_mode; - bool exclusive; - bool allow_loopback; - enum mlx4_net_trans_promisc_mode promisc_mode; - u8 port; - u16 priority; - u32 qpn; -}; - -struct mlx4_net_trans_rule_hw_ctrl { - __be16 prio; - u8 type; - u8 flags; - u8 rsvd1; - u8 funcid; - u8 vep; - u8 port; - __be32 qpn; - __be32 rsvd2; -}; - -struct mlx4_net_trans_rule_hw_ib { - u8 size; - u8 rsvd1; - __be16 id; - u32 rsvd2; - __be32 l3_qpn; - __be32 qpn_mask; - u8 dst_gid[16]; - u8 dst_gid_msk[16]; -} __packed; - -struct mlx4_net_trans_rule_hw_eth { - u8 size; - u8 rsvd; - __be16 id; - u8 rsvd1[6]; - u8 dst_mac[6]; - u16 rsvd2; - u8 dst_mac_msk[6]; - u16 rsvd3; - u8 src_mac[6]; - u16 rsvd4; - u8 src_mac_msk[6]; - u8 rsvd5; - u8 ether_type_enable; - __be16 ether_type; - __be16 vlan_tag_msk; - __be16 vlan_tag; -} __packed; - -struct mlx4_net_trans_rule_hw_tcp_udp { - u8 size; - u8 rsvd; - __be16 id; - __be16 rsvd1[3]; - __be16 dst_port; - __be16 rsvd2; - __be16 dst_port_msk; - __be16 rsvd3; - __be16 src_port; - __be16 rsvd4; - __be16 src_port_msk; -} __packed; - -struct mlx4_net_trans_rule_hw_ipv4 { - u8 size; - u8 rsvd; - __be16 id; - __be32 rsvd1; - __be32 dst_ip; - __be32 dst_ip_msk; - __be32 src_ip; - __be32 src_ip_msk; -} __packed; - -struct _rule_hw { - union { - struct { - u8 size; - u8 rsvd; - __be16 id; - }; - struct mlx4_net_trans_rule_hw_eth eth; - struct mlx4_net_trans_rule_hw_ib ib; - struct mlx4_net_trans_rule_hw_ipv4 ipv4; - struct mlx4_net_trans_rule_hw_tcp_udp tcp_udp; - }; -}; - -int mlx4_flow_steer_promisc_add(struct mlx4_dev *dev, u8 port, u32 qpn, - enum mlx4_net_trans_promisc_mode mode); -int mlx4_flow_steer_promisc_remove(struct mlx4_dev *dev, u8 port, - enum mlx4_net_trans_promisc_mode mode); -int mlx4_multicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port); -int mlx4_multicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port); -int mlx4_unicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port); -int mlx4_unicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port); - -int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac); -void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, u64 mac); -int mlx4_get_base_qpn(struct mlx4_dev *dev, u8 port); -int __mlx4_replace_mac(struct mlx4_dev *dev, u8 port, int qpn, u64 new_mac); -int mlx4_SET_PORT_general(struct mlx4_dev *dev, u8 port, int mtu, - u8 pptx, u8 pfctx, u8 pprx, u8 pfcrx); -int mlx4_SET_PORT_qpn_calc(struct mlx4_dev *dev, u8 port, u32 base_qpn, - u8 promisc); -int mlx4_SET_PORT_PRIO2TC(struct mlx4_dev *dev, u8 port, u8 *prio2tc); -int mlx4_SET_PORT_SCHEDULER(struct mlx4_dev *dev, u8 port, u8 *tc_tx_bw, - u8 *pg, u16 *ratelimit); -int mlx4_find_cached_vlan(struct mlx4_dev *dev, u8 port, u16 vid, int *idx); -int mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, int *index); -void mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, u16 vlan); - -int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list, - int npages, u64 iova, u32 *lkey, u32 *rkey); -int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages, - int max_maps, u8 page_shift, struct mlx4_fmr *fmr); -int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr); -void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr, - u32 *lkey, u32 *rkey); -int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr); -int mlx4_SYNC_TPT(struct mlx4_dev *dev); -int mlx4_query_diag_counters(struct mlx4_dev *mlx4_dev, int array_length, - u8 op_modifier, u32 in_offset[], - u32 counter_out[]); - -int mlx4_test_interrupts(struct mlx4_dev *dev); -int mlx4_assign_eq(struct mlx4_dev *dev, char* name, int * vector); -void mlx4_release_eq(struct mlx4_dev *dev, int vec); - -int mlx4_wol_read(struct mlx4_dev *dev, u64 *config, int port); -int mlx4_wol_write(struct mlx4_dev *dev, u64 config, int port); - -int mlx4_counter_alloc(struct mlx4_dev *dev, u8 port, u32 *idx); -void mlx4_counter_free(struct mlx4_dev *dev, u8 port, u32 idx); - -int mlx4_flow_attach(struct mlx4_dev *dev, - struct mlx4_net_trans_rule *rule, u64 *reg_id); -int mlx4_flow_detach(struct mlx4_dev *dev, u64 reg_id); -int map_sw_to_hw_steering_mode(struct mlx4_dev *dev, - enum mlx4_net_trans_promisc_mode flow_type); -int map_sw_to_hw_steering_id(struct mlx4_dev *dev, - enum mlx4_net_trans_rule_id id); -int hw_rule_sz(struct mlx4_dev *dev, enum mlx4_net_trans_rule_id id); - -void mlx4_sync_pkey_table(struct mlx4_dev *dev, int slave, int port, - int i, int val); - -int mlx4_get_parav_qkey(struct mlx4_dev *dev, u32 qpn, u32 *qkey); - -int mlx4_is_slave_active(struct mlx4_dev *dev, int slave); -int mlx4_gen_pkey_eqe(struct mlx4_dev *dev, int slave, u8 port); -int mlx4_gen_guid_change_eqe(struct mlx4_dev *dev, int slave, u8 port); -int mlx4_gen_slaves_port_mgt_ev(struct mlx4_dev *dev, u8 port, int attr, u16 lid, u8 sl); -int mlx4_gen_port_state_change_eqe(struct mlx4_dev *dev, int slave, u8 port, u8 port_subtype_change); -enum slave_port_state mlx4_get_slave_port_state(struct mlx4_dev *dev, int slave, u8 port); -int set_and_calc_slave_port_state(struct mlx4_dev *dev, int slave, u8 port, int event, enum slave_port_gen_event *gen_event); - -void mlx4_put_slave_node_guid(struct mlx4_dev *dev, int slave, __be64 guid); -__be64 mlx4_get_slave_node_guid(struct mlx4_dev *dev, int slave); -int mlx4_get_slave_from_roce_gid(struct mlx4_dev *dev, int port, u8 *gid, int *slave_id); -int mlx4_get_roce_gid_from_slave(struct mlx4_dev *dev, int port, int slave_id, u8 *gid); - -int mlx4_FLOW_STEERING_IB_UC_QP_RANGE(struct mlx4_dev *dev, u32 min_range_qpn, u32 max_range_qpn); - -s64 mlx4_read_clock(struct mlx4_dev *dev); -int mlx4_get_internal_clock_params(struct mlx4_dev *dev, - struct mlx4_clock_params *params); - -int mlx4_get_module_info(struct mlx4_dev *dev, u8 port, - u16 offset, u16 size, u8 *data); - -#endif /* MLX4_DEVICE_H */ Property changes on: stable/11/sys/ofed/include/linux/mlx4/device.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/include/linux/mlx4/doorbell.h =================================================================== --- stable/11/sys/ofed/include/linux/mlx4/doorbell.h (revision 329158) +++ stable/11/sys/ofed/include/linux/mlx4/doorbell.h (nonexistent) @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2004 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MLX4_DOORBELL_H -#define MLX4_DOORBELL_H - -#include -#include - -#define MLX4_SEND_DOORBELL 0x14 -#define MLX4_CQ_DOORBELL 0x20 - -#if BITS_PER_LONG == 64 -/* - * Assume that we can just write a 64-bit doorbell atomically. s390 - * actually doesn't have writeq() but S/390 systems don't even have - * PCI so we won't worry about it. - */ - -#define MLX4_DECLARE_DOORBELL_LOCK(name) -#define MLX4_INIT_DOORBELL_LOCK(ptr) do { } while (0) -#define MLX4_GET_DOORBELL_LOCK(ptr) (NULL) - -static inline void mlx4_write64(__be32 val[2], void __iomem *dest, - spinlock_t *doorbell_lock) -{ - __raw_writeq(*(u64 *) val, dest); -} - -#else - -/* - * Just fall back to a spinlock to protect the doorbell if - * BITS_PER_LONG is 32 -- there's no portable way to do atomic 64-bit - * MMIO writes. - */ - -#define MLX4_DECLARE_DOORBELL_LOCK(name) spinlock_t name; -#define MLX4_INIT_DOORBELL_LOCK(ptr) spin_lock_init(ptr) -#define MLX4_GET_DOORBELL_LOCK(ptr) (ptr) - -static inline void mlx4_write64(__be32 val[2], void __iomem *dest, - spinlock_t *doorbell_lock) -{ - unsigned long flags; - - spin_lock_irqsave(doorbell_lock, flags); - __raw_writel((__force u32) val[0], dest); - __raw_writel((__force u32) val[1], (u8 *)dest + 4); - spin_unlock_irqrestore(doorbell_lock, flags); -} - -#endif - -#endif /* MLX4_DOORBELL_H */ Property changes on: stable/11/sys/ofed/include/linux/mlx4/doorbell.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/include/linux/mlx4/cq.h =================================================================== --- stable/11/sys/ofed/include/linux/mlx4/cq.h (revision 329158) +++ stable/11/sys/ofed/include/linux/mlx4/cq.h (nonexistent) @@ -1,181 +0,0 @@ -/* - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MLX4_CQ_H -#define MLX4_CQ_H - -#include - -#include -#include - -struct mlx4_cqe { - __be32 vlan_my_qpn; - __be32 immed_rss_invalid; - __be32 g_mlpath_rqpn; - union { - struct { - union { - struct { - __be16 sl_vid; - __be16 rlid; - }; - __be32 timestamp_16_47; - }; - __be16 status; - u8 ipv6_ext_mask; - u8 badfcs_enc; - }; - struct { - __be16 reserved1; - u8 smac[6]; - }; - }; - __be32 byte_cnt; - __be16 wqe_index; - __be16 checksum; - u8 reserved2[1]; - __be16 timestamp_0_15; - u8 owner_sr_opcode; -} __packed; - -struct mlx4_err_cqe { - __be32 my_qpn; - u32 reserved1[5]; - __be16 wqe_index; - u8 vendor_err_syndrome; - u8 syndrome; - u8 reserved2[3]; - u8 owner_sr_opcode; -}; - -struct mlx4_ts_cqe { - __be32 vlan_my_qpn; - __be32 immed_rss_invalid; - __be32 g_mlpath_rqpn; - __be32 timestamp_hi; - __be16 status; - u8 ipv6_ext_mask; - u8 badfcs_enc; - __be32 byte_cnt; - __be16 wqe_index; - __be16 checksum; - u8 reserved; - __be16 timestamp_lo; - u8 owner_sr_opcode; -} __packed; - -enum { - MLX4_CQE_VLAN_PRESENT_MASK = 1 << 29, - MLX4_CQE_QPN_MASK = 0xffffff, - MLX4_CQE_VID_MASK = 0xfff, -}; - -enum { - MLX4_CQE_OWNER_MASK = 0x80, - MLX4_CQE_IS_SEND_MASK = 0x40, - MLX4_CQE_OPCODE_MASK = 0x1f -}; - -enum { - MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR = 0x01, - MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR = 0x02, - MLX4_CQE_SYNDROME_LOCAL_PROT_ERR = 0x04, - MLX4_CQE_SYNDROME_WR_FLUSH_ERR = 0x05, - MLX4_CQE_SYNDROME_MW_BIND_ERR = 0x06, - MLX4_CQE_SYNDROME_BAD_RESP_ERR = 0x10, - MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR = 0x11, - MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR = 0x12, - MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR = 0x13, - MLX4_CQE_SYNDROME_REMOTE_OP_ERR = 0x14, - MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR = 0x15, - MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR = 0x16, - MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR = 0x22, -}; - -enum { - MLX4_CQE_STATUS_IPV4 = 1 << 6, - MLX4_CQE_STATUS_IPV4F = 1 << 7, - MLX4_CQE_STATUS_IPV6 = 1 << 8, - MLX4_CQE_STATUS_IPV4OPT = 1 << 9, - MLX4_CQE_STATUS_TCP = 1 << 10, - MLX4_CQE_STATUS_UDP = 1 << 11, - MLX4_CQE_STATUS_IPOK = 1 << 12, -}; - -enum { - MLX4_CQE_LLC = 1, - MLX4_CQE_SNAP = 1 << 1, - MLX4_CQE_BAD_FCS = 1 << 4, -}; - -static inline void mlx4_cq_arm(struct mlx4_cq *cq, u32 cmd, - u8 __iomem *uar_page, - spinlock_t *doorbell_lock) -{ - __be32 doorbell[2]; - u32 sn; - u32 ci; - - sn = cq->arm_sn & 3; - ci = cq->cons_index & 0xffffff; - - *cq->arm_db = cpu_to_be32(sn << 28 | cmd | ci); - - /* - * Make sure that the doorbell record in host memory is - * written before ringing the doorbell via PCI MMIO. - */ - wmb(); - - doorbell[0] = cpu_to_be32(sn << 28 | cmd | cq->cqn); - doorbell[1] = cpu_to_be32(ci); - - mlx4_write64(doorbell, uar_page + MLX4_CQ_DOORBELL, doorbell_lock); -} - -static inline void mlx4_cq_set_ci(struct mlx4_cq *cq) -{ - *cq->set_ci_db = cpu_to_be32(cq->cons_index & 0xffffff); -} - -enum { - MLX4_CQ_DB_REQ_NOT_SOL = 1 << 24, - MLX4_CQ_DB_REQ_NOT = 2 << 24 -}; - -int mlx4_cq_modify(struct mlx4_dev *dev, struct mlx4_cq *cq, - u16 count, u16 period); -int mlx4_cq_resize(struct mlx4_dev *dev, struct mlx4_cq *cq, - int entries, struct mlx4_mtt *mtt); -int mlx4_cq_ignore_overrun(struct mlx4_dev *dev, struct mlx4_cq *cq); -#endif /* MLX4_CQ_H */ Property changes on: stable/11/sys/ofed/include/linux/mlx4/cq.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/include/linux/mlx4/cmd.h =================================================================== --- stable/11/sys/ofed/include/linux/mlx4/cmd.h (revision 329158) +++ stable/11/sys/ofed/include/linux/mlx4/cmd.h (nonexistent) @@ -1,258 +0,0 @@ -/* - * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MLX4_CMD_H -#define MLX4_CMD_H - -#include -#include - -enum { - /* initialization and general commands */ - MLX4_CMD_SYS_EN = 0x1, - MLX4_CMD_SYS_DIS = 0x2, - MLX4_CMD_MAP_FA = 0xfff, - MLX4_CMD_UNMAP_FA = 0xffe, - MLX4_CMD_RUN_FW = 0xff6, - MLX4_CMD_MOD_STAT_CFG = 0x34, - MLX4_CMD_QUERY_DEV_CAP = 0x3, - MLX4_CMD_QUERY_FW = 0x4, - MLX4_CMD_ENABLE_LAM = 0xff8, - MLX4_CMD_DISABLE_LAM = 0xff7, - MLX4_CMD_QUERY_DDR = 0x5, - MLX4_CMD_QUERY_ADAPTER = 0x6, - MLX4_CMD_INIT_HCA = 0x7, - MLX4_CMD_CLOSE_HCA = 0x8, - MLX4_CMD_INIT_PORT = 0x9, - MLX4_CMD_CLOSE_PORT = 0xa, - MLX4_CMD_QUERY_HCA = 0xb, - MLX4_CMD_QUERY_PORT = 0x43, - MLX4_CMD_SENSE_PORT = 0x4d, - MLX4_CMD_HW_HEALTH_CHECK = 0x50, - MLX4_CMD_SET_PORT = 0xc, - MLX4_CMD_SET_NODE = 0x5a, - MLX4_CMD_QUERY_FUNC = 0x56, - MLX4_CMD_ACCESS_DDR = 0x2e, - MLX4_CMD_MAP_ICM = 0xffa, - MLX4_CMD_UNMAP_ICM = 0xff9, - MLX4_CMD_MAP_ICM_AUX = 0xffc, - MLX4_CMD_UNMAP_ICM_AUX = 0xffb, - MLX4_CMD_SET_ICM_SIZE = 0xffd, - /*master notify fw on finish for slave's flr*/ - MLX4_CMD_INFORM_FLR_DONE = 0x5b, - MLX4_CMD_GET_OP_REQ = 0x59, - - /* TPT commands */ - MLX4_CMD_SW2HW_MPT = 0xd, - MLX4_CMD_QUERY_MPT = 0xe, - MLX4_CMD_HW2SW_MPT = 0xf, - MLX4_CMD_READ_MTT = 0x10, - MLX4_CMD_WRITE_MTT = 0x11, - MLX4_CMD_SYNC_TPT = 0x2f, - - /* EQ commands */ - MLX4_CMD_MAP_EQ = 0x12, - MLX4_CMD_SW2HW_EQ = 0x13, - MLX4_CMD_HW2SW_EQ = 0x14, - MLX4_CMD_QUERY_EQ = 0x15, - - /* CQ commands */ - MLX4_CMD_SW2HW_CQ = 0x16, - MLX4_CMD_HW2SW_CQ = 0x17, - MLX4_CMD_QUERY_CQ = 0x18, - MLX4_CMD_MODIFY_CQ = 0x2c, - - /* SRQ commands */ - MLX4_CMD_SW2HW_SRQ = 0x35, - MLX4_CMD_HW2SW_SRQ = 0x36, - MLX4_CMD_QUERY_SRQ = 0x37, - MLX4_CMD_ARM_SRQ = 0x40, - - /* QP/EE commands */ - MLX4_CMD_RST2INIT_QP = 0x19, - MLX4_CMD_INIT2RTR_QP = 0x1a, - MLX4_CMD_RTR2RTS_QP = 0x1b, - MLX4_CMD_RTS2RTS_QP = 0x1c, - MLX4_CMD_SQERR2RTS_QP = 0x1d, - MLX4_CMD_2ERR_QP = 0x1e, - MLX4_CMD_RTS2SQD_QP = 0x1f, - MLX4_CMD_SQD2SQD_QP = 0x38, - MLX4_CMD_SQD2RTS_QP = 0x20, - MLX4_CMD_2RST_QP = 0x21, - MLX4_CMD_QUERY_QP = 0x22, - MLX4_CMD_INIT2INIT_QP = 0x2d, - MLX4_CMD_SUSPEND_QP = 0x32, - MLX4_CMD_UNSUSPEND_QP = 0x33, - MLX4_CMD_UPDATE_QP = 0x61, - /* special QP and management commands */ - MLX4_CMD_CONF_SPECIAL_QP = 0x23, - MLX4_CMD_MAD_IFC = 0x24, - - /* multicast commands */ - MLX4_CMD_READ_MCG = 0x25, - MLX4_CMD_WRITE_MCG = 0x26, - MLX4_CMD_MGID_HASH = 0x27, - - /* miscellaneous commands */ - MLX4_CMD_DIAG_RPRT = 0x30, - MLX4_CMD_NOP = 0x31, - MLX4_CMD_ACCESS_MEM = 0x2e, - MLX4_CMD_SET_VEP = 0x52, - - /* Ethernet specific commands */ - MLX4_CMD_SET_VLAN_FLTR = 0x47, - MLX4_CMD_SET_MCAST_FLTR = 0x48, - MLX4_CMD_DUMP_ETH_STATS = 0x49, - - /* Communication channel commands */ - MLX4_CMD_ARM_COMM_CHANNEL = 0x57, - MLX4_CMD_GEN_EQE = 0x58, - - /* virtual commands */ - MLX4_CMD_ALLOC_RES = 0xf00, - MLX4_CMD_FREE_RES = 0xf01, - MLX4_CMD_MCAST_ATTACH = 0xf05, - MLX4_CMD_UCAST_ATTACH = 0xf06, - MLX4_CMD_PROMISC = 0xf08, - MLX4_CMD_QUERY_FUNC_CAP = 0xf0a, - MLX4_CMD_QP_ATTACH = 0xf0b, - - /* debug commands */ - MLX4_CMD_QUERY_DEBUG_MSG = 0x2a, - MLX4_CMD_SET_DEBUG_MSG = 0x2b, - - /* statistics commands */ - MLX4_CMD_QUERY_IF_STAT = 0X54, - MLX4_CMD_SET_IF_STAT = 0X55, - - /* register/delete flow steering network rules */ - MLX4_QP_FLOW_STEERING_ATTACH = 0x65, - MLX4_QP_FLOW_STEERING_DETACH = 0x66, - MLX4_FLOW_STEERING_IB_UC_QP_RANGE = 0x64, -}; - -enum { - MLX4_CMD_TIME_CLASS_A = 60000, - MLX4_CMD_TIME_CLASS_B = 60000, - MLX4_CMD_TIME_CLASS_C = 60000, -}; - -enum { - MLX4_MAILBOX_SIZE = 4096, - MLX4_ACCESS_MEM_ALIGN = 256, -}; - -enum { - /* set port opcode modifiers */ - MLX4_SET_PORT_GENERAL = 0x0, - MLX4_SET_PORT_RQP_CALC = 0x1, - MLX4_SET_PORT_MAC_TABLE = 0x2, - MLX4_SET_PORT_VLAN_TABLE = 0x3, - MLX4_SET_PORT_PRIO_MAP = 0x4, - MLX4_SET_PORT_GID_TABLE = 0x5, - MLX4_SET_PORT_PRIO2TC = 0x8, - MLX4_SET_PORT_SCHEDULER = 0x9 -}; - -enum { - MLX4_CMD_WRAPPED, - MLX4_CMD_NATIVE -}; - -struct mlx4_dev; - -struct mlx4_cmd_mailbox { - void *buf; - dma_addr_t dma; -}; - -int __mlx4_cmd(struct mlx4_dev *dev, u64 in_param, u64 *out_param, - int out_is_imm, u32 in_modifier, u8 op_modifier, - u16 op, unsigned long timeout, int native); - -/* Invoke a command with no output parameter */ -static inline int mlx4_cmd(struct mlx4_dev *dev, u64 in_param, u32 in_modifier, - u8 op_modifier, u16 op, unsigned long timeout, - int native) -{ - return __mlx4_cmd(dev, in_param, NULL, 0, in_modifier, - op_modifier, op, timeout, native); -} - -/* Invoke a command with an output mailbox */ -static inline int mlx4_cmd_box(struct mlx4_dev *dev, u64 in_param, u64 out_param, - u32 in_modifier, u8 op_modifier, u16 op, - unsigned long timeout, int native) -{ - return __mlx4_cmd(dev, in_param, &out_param, 0, in_modifier, - op_modifier, op, timeout, native); -} - -/* - * Invoke a command with an immediate output parameter (and copy the - * output into the caller's out_param pointer after the command - * executes). - */ -static inline int mlx4_cmd_imm(struct mlx4_dev *dev, u64 in_param, u64 *out_param, - u32 in_modifier, u8 op_modifier, u16 op, - unsigned long timeout, int native) -{ - return __mlx4_cmd(dev, in_param, out_param, 1, in_modifier, - op_modifier, op, timeout, native); -} - -struct mlx4_cmd_mailbox *mlx4_alloc_cmd_mailbox(struct mlx4_dev *dev); -void mlx4_free_cmd_mailbox(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox); - -u32 mlx4_comm_get_version(void); -int mlx4_set_vf_mac(struct mlx4_dev *dev, int port, int vf, u8 *mac); -int mlx4_set_vf_vlan(struct mlx4_dev *dev, int port, int vf, u16 vlan, u8 qos); -int mlx4_set_vf_spoofchk(struct mlx4_dev *dev, int port, int vf, bool setting); -int mlx4_set_vf_link_state(struct mlx4_dev *dev, int port, int vf, int link_state); -int mlx4_get_vf_link_state(struct mlx4_dev *dev, int port, int vf); -/* - * mlx4_get_slave_default_vlan - - * retrun true if VST ( default vlan) - * if VST will fill vlan & qos (if not NULL) - */ -bool mlx4_get_slave_default_vlan(struct mlx4_dev *dev, int port, int slave, u16 *vlan, u8 *qos); - -enum { - IFLA_VF_LINK_STATE_AUTO, /* link state of the uplink */ - IFLA_VF_LINK_STATE_ENABLE, /* link always up */ - IFLA_VF_LINK_STATE_DISABLE, /* link always down */ - __IFLA_VF_LINK_STATE_MAX, -}; - -#define MLX4_COMM_GET_IF_REV(cmd_chan_ver) (u8)((cmd_chan_ver) >> 8) - -#endif /* MLX4_CMD_H */ Property changes on: stable/11/sys/ofed/include/linux/mlx4/cmd.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/include/linux/mlx4 =================================================================== --- stable/11/sys/ofed/include/linux/mlx4 (revision 329158) +++ stable/11/sys/ofed/include/linux/mlx4 (nonexistent) Property changes on: stable/11/sys/ofed/include/linux/mlx4 ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11/sys/ofed/include/linux =================================================================== --- stable/11/sys/ofed/include/linux (revision 329158) +++ stable/11/sys/ofed/include/linux (nonexistent) Property changes on: stable/11/sys/ofed/include/linux ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/11 =================================================================== --- stable/11 (revision 329158) +++ stable/11 (revision 329159) Property changes on: stable/11 ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head:r306486,310425,310488,324200-324201,324621,325841