Index: stable/10/sys/conf/files
===================================================================
--- stable/10/sys/conf/files	(revision 270296)
+++ stable/10/sys/conf/files	(revision 270297)
@@ -1,3935 +1,3937 @@
 # $FreeBSD$
 #
 # The long compile-with and dependency lines are required because of
 # limitations in config: backslash-newline doesn't work in strings, and
 # dependency lines other than the first are silently ignored.
 #
 acpi_quirks.h			optional acpi				   \
 	dependency	"$S/tools/acpi_quirks2h.awk $S/dev/acpica/acpi_quirks" \
 	compile-with	"${AWK} -f $S/tools/acpi_quirks2h.awk $S/dev/acpica/acpi_quirks" \
 	no-obj no-implicit-rule before-depend				   \
 	clean		"acpi_quirks.h"
 #
 # The 'fdt_dtb_file' target covers an actual DTB file name, which is derived
 # from the specified source (DTS) file: <platform>.dts -> <platform>.dtb
 #
 fdt_dtb_file			optional fdt fdt_dtb_static \
 	compile-with "sh $S/tools/fdt/make_dtb.sh $S ${FDT_DTS_FILE} ${.CURDIR}/${FDT_DTS_FILE:R}.dtb" \
 	no-obj no-implicit-rule before-depend	\
 	clean		"${FDT_DTS_FILE:R}.dtb"
 fdt_static_dtb.h		optional fdt fdt_dtb_static \
 	compile-with "sh $S/tools/fdt/make_dtbh.sh ${FDT_DTS_FILE} ${.CURDIR}" \
 	dependency	"fdt_dtb_file" \
 	no-obj no-implicit-rule before-depend \
 	clean		"fdt_static_dtb.h"
 feeder_eq_gen.h			optional sound				   \
 	dependency	"$S/tools/sound/feeder_eq_mkfilter.awk"		   \
 	compile-with	"${AWK} -f $S/tools/sound/feeder_eq_mkfilter.awk -- ${FEEDER_EQ_PRESETS} > feeder_eq_gen.h" \
 	no-obj no-implicit-rule before-depend				   \
 	clean		"feeder_eq_gen.h"
 feeder_rate_gen.h		optional sound				   \
 	dependency	"$S/tools/sound/feeder_rate_mkfilter.awk"	   \
 	compile-with	"${AWK} -f $S/tools/sound/feeder_rate_mkfilter.awk -- ${FEEDER_RATE_PRESETS} > feeder_rate_gen.h" \
 	no-obj no-implicit-rule before-depend				   \
 	clean		"feeder_rate_gen.h"
 snd_fxdiv_gen.h			optional sound				   \
 	dependency	"$S/tools/sound/snd_fxdiv_gen.awk"		   \
 	compile-with	"${AWK} -f $S/tools/sound/snd_fxdiv_gen.awk -- > snd_fxdiv_gen.h" \
 	no-obj no-implicit-rule before-depend				   \
 	clean		"snd_fxdiv_gen.h"
 miidevs.h			optional miibus | mii			   \
 	dependency	"$S/tools/miidevs2h.awk $S/dev/mii/miidevs"	   \
 	compile-with	"${AWK} -f $S/tools/miidevs2h.awk $S/dev/mii/miidevs" \
 	no-obj no-implicit-rule before-depend				   \
 	clean		"miidevs.h"
 pccarddevs.h			standard				   \
 	dependency	"$S/tools/pccarddevs2h.awk $S/dev/pccard/pccarddevs" \
 	compile-with	"${AWK} -f $S/tools/pccarddevs2h.awk $S/dev/pccard/pccarddevs" \
 	no-obj no-implicit-rule before-depend				   \
 	clean		"pccarddevs.h"
 teken_state.h		optional sc | vt				   \
 	dependency	"$S/teken/gensequences $S/teken/sequences" \
 	compile-with	"${AWK} -f $S/teken/gensequences $S/teken/sequences > teken_state.h" \
 	no-obj no-implicit-rule before-depend				   \
 	clean		"teken_state.h"
 usbdevs.h			optional usb				   \
 	dependency	"$S/tools/usbdevs2h.awk $S/dev/usb/usbdevs" \
 	compile-with	"${AWK} -f $S/tools/usbdevs2h.awk $S/dev/usb/usbdevs -h" \
 	no-obj no-implicit-rule before-depend				   \
 	clean		"usbdevs.h"
 usbdevs_data.h			optional usb				   \
 	dependency	"$S/tools/usbdevs2h.awk $S/dev/usb/usbdevs" \
 	compile-with	"${AWK} -f $S/tools/usbdevs2h.awk $S/dev/usb/usbdevs -d" \
 	no-obj no-implicit-rule before-depend				   \
 	clean		"usbdevs_data.h"
 cam/cam.c			optional scbus
 cam/cam_compat.c		optional scbus
 cam/cam_periph.c		optional scbus
 cam/cam_queue.c			optional scbus
 cam/cam_sim.c			optional scbus
 cam/cam_xpt.c			optional scbus
 cam/ata/ata_all.c		optional scbus
 cam/ata/ata_xpt.c		optional scbus
 cam/ata/ata_pmp.c		optional scbus
 cam/scsi/scsi_xpt.c		optional scbus
 cam/scsi/scsi_all.c		optional scbus
 cam/scsi/scsi_cd.c		optional cd
 cam/scsi/scsi_ch.c		optional ch
 cam/ata/ata_da.c		optional ada | da
 cam/ctl/ctl.c			optional ctl
 cam/ctl/ctl_backend.c		optional ctl
 cam/ctl/ctl_backend_block.c	optional ctl
 cam/ctl/ctl_backend_ramdisk.c	optional ctl
 cam/ctl/ctl_cmd_table.c		optional ctl
 cam/ctl/ctl_frontend.c		optional ctl
 cam/ctl/ctl_frontend_cam_sim.c	optional ctl
 cam/ctl/ctl_frontend_internal.c	optional ctl
 cam/ctl/ctl_frontend_iscsi.c	optional ctl
 cam/ctl/ctl_scsi_all.c		optional ctl
 cam/ctl/ctl_tpc.c		optional ctl
 cam/ctl/ctl_tpc_local.c		optional ctl
 cam/ctl/ctl_error.c		optional ctl
 cam/ctl/ctl_util.c		optional ctl
 cam/ctl/scsi_ctl.c		optional ctl
 cam/scsi/scsi_da.c		optional da
 cam/scsi/scsi_low.c		optional ct | ncv | nsp | stg
 cam/scsi/scsi_pass.c		optional pass
 cam/scsi/scsi_pt.c		optional pt
 cam/scsi/scsi_sa.c		optional sa
 cam/scsi/scsi_enc.c		optional ses
 cam/scsi/scsi_enc_ses.c		optional ses
 cam/scsi/scsi_enc_safte.c	optional ses
 cam/scsi/scsi_sg.c		optional sg
 cam/scsi/scsi_targ_bh.c		optional targbh
 cam/scsi/scsi_target.c		optional targ
 cam/scsi/smp_all.c		optional scbus
 # shared between zfs and dtrace
 cddl/compat/opensolaris/kern/opensolaris.c				optional zfs compile-with "${ZFS_C}"
 cddl/compat/opensolaris/kern/opensolaris_cmn_err.c			optional zfs compile-with "${ZFS_C}"
 cddl/compat/opensolaris/kern/opensolaris_kmem.c				optional zfs compile-with "${ZFS_C}"
 cddl/compat/opensolaris/kern/opensolaris_misc.c				optional zfs compile-with "${ZFS_C}"
 cddl/compat/opensolaris/kern/opensolaris_sunddi.c			optional zfs compile-with "${ZFS_C}"
 # zfs specific
 cddl/compat/opensolaris/kern/opensolaris_acl.c				optional zfs compile-with "${ZFS_C}"
 cddl/compat/opensolaris/kern/opensolaris_kobj.c				optional zfs compile-with "${ZFS_C}"
 cddl/compat/opensolaris/kern/opensolaris_kstat.c			optional zfs compile-with "${ZFS_C}"
 cddl/compat/opensolaris/kern/opensolaris_lookup.c			optional zfs compile-with "${ZFS_C}"
 cddl/compat/opensolaris/kern/opensolaris_policy.c			optional zfs compile-with "${ZFS_C}"
 cddl/compat/opensolaris/kern/opensolaris_string.c			optional zfs compile-with "${ZFS_C}"
 cddl/compat/opensolaris/kern/opensolaris_sysevent.c			optional zfs compile-with "${ZFS_C}"
 cddl/compat/opensolaris/kern/opensolaris_taskq.c			optional zfs compile-with "${ZFS_C}"
 cddl/compat/opensolaris/kern/opensolaris_uio.c				optional zfs compile-with "${ZFS_C}"
 cddl/compat/opensolaris/kern/opensolaris_vfs.c				optional zfs compile-with "${ZFS_C}"
 cddl/compat/opensolaris/kern/opensolaris_vm.c				optional zfs compile-with "${ZFS_C}"
 cddl/compat/opensolaris/kern/opensolaris_zone.c				optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/common/acl/acl_common.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/common/avl/avl.c				optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/common/nvpair/fnvpair.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/common/nvpair/nvpair.c				optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/common/nvpair/nvpair_alloc_fixed.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/common/unicode/u8_textprep.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/common/zfs/zfeature_common.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/common/zfs/zfs_comutil.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/common/zfs/zfs_deleg.c				optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/common/zfs/zfs_fletcher.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/common/zfs/zfs_ioctl_compat.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/common/zfs/zfs_namecheck.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/common/zfs/zfs_prop.c				optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/common/zfs/zpool_prop.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/common/zfs/zprop_common.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/gfs.c				optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/vnode.c				optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/blkptr.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/bplist.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/bpobj.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/bptree.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/ddt.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/ddt_zap.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_diff.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_object.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_send.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_traverse.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_zfetch.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c			optional zfs compile-with "${ZFS_C}" \
 	warning "kernel contains CDDL licensed ZFS filesystem"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dnode_sync.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_bookmark.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_deadlist.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_deleg.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_destroy.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_prop.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_userhold.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_synctask.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/gzip.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/lz4.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/lzjb.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/refcount.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/rrwlock.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/sa.c				optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/sha256.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/spa_config.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/spa_errlog.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/spa_history.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/space_map.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/space_reftree.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/trim_map.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/uberblock.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/unique.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_cache.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_file.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_geom.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_label.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_mirror.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_missing.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_queue.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_raidz.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_root.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zap.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zap_leaf.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zap_micro.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfeature.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_byteswap.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ctldir.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_debug.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_dir.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_fm.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_fuid.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_log.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_onexit.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_replay.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_rlock.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_sa.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zil.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zio_checksum.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zio_compress.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zio_inject.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zle.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zrlock.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/os/callb.c				optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/os/fm.c				optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/os/list.c				optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/os/nvpair_alloc_system.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/zmod/adler32.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/zmod/deflate.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/zmod/inffast.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/zmod/inflate.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/zmod/inftrees.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/zmod/opensolaris_crc32.c		optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/zmod/trees.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/zmod/zmod.c				optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/zmod/zmod_subr.c			optional zfs compile-with "${ZFS_C}"
 cddl/contrib/opensolaris/uts/common/zmod/zutil.c			optional zfs compile-with "${ZFS_C}"
 compat/freebsd32/freebsd32_capability.c	optional compat_freebsd32
 compat/freebsd32/freebsd32_ioctl.c	optional compat_freebsd32
 compat/freebsd32/freebsd32_misc.c	optional compat_freebsd32
 compat/freebsd32/freebsd32_syscalls.c	optional compat_freebsd32
 compat/freebsd32/freebsd32_sysent.c	optional compat_freebsd32
 contrib/altq/altq/altq_cbq.c		optional altq
 contrib/altq/altq/altq_cdnr.c		optional altq
 contrib/altq/altq/altq_hfsc.c		optional altq
 contrib/altq/altq/altq_priq.c		optional altq
 contrib/altq/altq/altq_red.c		optional altq
 contrib/altq/altq/altq_rio.c		optional altq
 contrib/altq/altq/altq_rmclass.c	optional altq
 contrib/altq/altq/altq_subr.c		optional altq
 contrib/dev/acpica/components/debugger/dbcmds.c		optional acpi acpi_debug
 contrib/dev/acpica/components/debugger/dbconvert.c	optional acpi acpi_debug
 contrib/dev/acpica/components/debugger/dbdisply.c	optional acpi acpi_debug
 contrib/dev/acpica/components/debugger/dbexec.c		optional acpi acpi_debug
 contrib/dev/acpica/components/debugger/dbfileio.c	optional acpi acpi_debug
 contrib/dev/acpica/components/debugger/dbhistry.c	optional acpi acpi_debug
 contrib/dev/acpica/components/debugger/dbinput.c	optional acpi acpi_debug
 contrib/dev/acpica/components/debugger/dbmethod.c	optional acpi acpi_debug
 contrib/dev/acpica/components/debugger/dbnames.c	optional acpi acpi_debug
 contrib/dev/acpica/components/debugger/dbstats.c	optional acpi acpi_debug
 contrib/dev/acpica/components/debugger/dbutils.c	optional acpi acpi_debug
 contrib/dev/acpica/components/debugger/dbxface.c	optional acpi acpi_debug
 contrib/dev/acpica/components/disassembler/dmbuffer.c	optional acpi acpi_debug
 contrib/dev/acpica/components/disassembler/dmdeferred.c	optional acpi acpi_debug
 contrib/dev/acpica/components/disassembler/dmnames.c	optional acpi acpi_debug
 contrib/dev/acpica/components/disassembler/dmopcode.c	optional acpi acpi_debug
 contrib/dev/acpica/components/disassembler/dmobject.c	optional acpi acpi_debug
 contrib/dev/acpica/components/disassembler/dmresrc.c	optional acpi acpi_debug
 contrib/dev/acpica/components/disassembler/dmresrcl.c	optional acpi acpi_debug
 contrib/dev/acpica/components/disassembler/dmresrcl2.c	optional acpi acpi_debug
 contrib/dev/acpica/components/disassembler/dmresrcs.c	optional acpi acpi_debug
 contrib/dev/acpica/components/disassembler/dmutils.c	optional acpi acpi_debug
 contrib/dev/acpica/components/disassembler/dmwalk.c	optional acpi acpi_debug
 contrib/dev/acpica/components/dispatcher/dsargs.c	optional acpi
 contrib/dev/acpica/components/dispatcher/dscontrol.c	optional acpi
 contrib/dev/acpica/components/dispatcher/dsfield.c	optional acpi
 contrib/dev/acpica/components/dispatcher/dsinit.c	optional acpi
 contrib/dev/acpica/components/dispatcher/dsmethod.c	optional acpi
 contrib/dev/acpica/components/dispatcher/dsmthdat.c	optional acpi
 contrib/dev/acpica/components/dispatcher/dsobject.c	optional acpi
 contrib/dev/acpica/components/dispatcher/dsopcode.c	optional acpi
 contrib/dev/acpica/components/dispatcher/dsutils.c	optional acpi
 contrib/dev/acpica/components/dispatcher/dswexec.c	optional acpi
 contrib/dev/acpica/components/dispatcher/dswload.c	optional acpi
 contrib/dev/acpica/components/dispatcher/dswload2.c	optional acpi
 contrib/dev/acpica/components/dispatcher/dswscope.c	optional acpi
 contrib/dev/acpica/components/dispatcher/dswstate.c	optional acpi
 contrib/dev/acpica/components/events/evevent.c		optional acpi
 contrib/dev/acpica/components/events/evglock.c		optional acpi
 contrib/dev/acpica/components/events/evgpe.c		optional acpi
 contrib/dev/acpica/components/events/evgpeblk.c		optional acpi
 contrib/dev/acpica/components/events/evgpeinit.c	optional acpi
 contrib/dev/acpica/components/events/evgpeutil.c	optional acpi
 contrib/dev/acpica/components/events/evhandler.c	optional acpi
 contrib/dev/acpica/components/events/evmisc.c		optional acpi
 contrib/dev/acpica/components/events/evregion.c		optional acpi
 contrib/dev/acpica/components/events/evrgnini.c		optional acpi
 contrib/dev/acpica/components/events/evsci.c		optional acpi
 contrib/dev/acpica/components/events/evxface.c		optional acpi
 contrib/dev/acpica/components/events/evxfevnt.c		optional acpi
 contrib/dev/acpica/components/events/evxfgpe.c		optional acpi
 contrib/dev/acpica/components/events/evxfregn.c		optional acpi
 contrib/dev/acpica/components/executer/exconfig.c	optional acpi
 contrib/dev/acpica/components/executer/exconvrt.c	optional acpi
 contrib/dev/acpica/components/executer/excreate.c	optional acpi
 contrib/dev/acpica/components/executer/exdebug.c	optional acpi
 contrib/dev/acpica/components/executer/exdump.c		optional acpi
 contrib/dev/acpica/components/executer/exfield.c	optional acpi
 contrib/dev/acpica/components/executer/exfldio.c	optional acpi
 contrib/dev/acpica/components/executer/exmisc.c		optional acpi
 contrib/dev/acpica/components/executer/exmutex.c	optional acpi
 contrib/dev/acpica/components/executer/exnames.c	optional acpi
 contrib/dev/acpica/components/executer/exoparg1.c	optional acpi
 contrib/dev/acpica/components/executer/exoparg2.c	optional acpi
 contrib/dev/acpica/components/executer/exoparg3.c	optional acpi
 contrib/dev/acpica/components/executer/exoparg6.c	optional acpi
 contrib/dev/acpica/components/executer/exprep.c		optional acpi
 contrib/dev/acpica/components/executer/exregion.c	optional acpi
 contrib/dev/acpica/components/executer/exresnte.c	optional acpi
 contrib/dev/acpica/components/executer/exresolv.c	optional acpi
 contrib/dev/acpica/components/executer/exresop.c	optional acpi
 contrib/dev/acpica/components/executer/exstore.c	optional acpi
 contrib/dev/acpica/components/executer/exstoren.c	optional acpi
 contrib/dev/acpica/components/executer/exstorob.c	optional acpi
 contrib/dev/acpica/components/executer/exsystem.c	optional acpi
 contrib/dev/acpica/components/executer/exutils.c	optional acpi
 contrib/dev/acpica/components/hardware/hwacpi.c		optional acpi
 contrib/dev/acpica/components/hardware/hwesleep.c	optional acpi
 contrib/dev/acpica/components/hardware/hwgpe.c		optional acpi
 contrib/dev/acpica/components/hardware/hwpci.c		optional acpi
 contrib/dev/acpica/components/hardware/hwregs.c		optional acpi
 contrib/dev/acpica/components/hardware/hwsleep.c	optional acpi
 contrib/dev/acpica/components/hardware/hwtimer.c	optional acpi
 contrib/dev/acpica/components/hardware/hwvalid.c	optional acpi
 contrib/dev/acpica/components/hardware/hwxface.c	optional acpi
 contrib/dev/acpica/components/hardware/hwxfsleep.c	optional acpi
 contrib/dev/acpica/components/namespace/nsaccess.c	optional acpi
 contrib/dev/acpica/components/namespace/nsalloc.c	optional acpi
 contrib/dev/acpica/components/namespace/nsarguments.c	optional acpi
 contrib/dev/acpica/components/namespace/nsconvert.c	optional acpi
 contrib/dev/acpica/components/namespace/nsdump.c	optional acpi
 contrib/dev/acpica/components/namespace/nseval.c	optional acpi
 contrib/dev/acpica/components/namespace/nsinit.c	optional acpi
 contrib/dev/acpica/components/namespace/nsload.c	optional acpi
 contrib/dev/acpica/components/namespace/nsnames.c	optional acpi
 contrib/dev/acpica/components/namespace/nsobject.c	optional acpi
 contrib/dev/acpica/components/namespace/nsparse.c	optional acpi
 contrib/dev/acpica/components/namespace/nspredef.c	optional acpi
 contrib/dev/acpica/components/namespace/nsprepkg.c	optional acpi
 contrib/dev/acpica/components/namespace/nsrepair.c	optional acpi
 contrib/dev/acpica/components/namespace/nsrepair2.c	optional acpi
 contrib/dev/acpica/components/namespace/nssearch.c	optional acpi
 contrib/dev/acpica/components/namespace/nsutils.c	optional acpi
 contrib/dev/acpica/components/namespace/nswalk.c	optional acpi
 contrib/dev/acpica/components/namespace/nsxfeval.c	optional acpi
 contrib/dev/acpica/components/namespace/nsxfname.c	optional acpi
 contrib/dev/acpica/components/namespace/nsxfobj.c	optional acpi
 contrib/dev/acpica/components/parser/psargs.c		optional acpi
 contrib/dev/acpica/components/parser/psloop.c		optional acpi
 contrib/dev/acpica/components/parser/psobject.c		optional acpi
 contrib/dev/acpica/components/parser/psopcode.c		optional acpi
 contrib/dev/acpica/components/parser/psopinfo.c		optional acpi
 contrib/dev/acpica/components/parser/psparse.c		optional acpi
 contrib/dev/acpica/components/parser/psscope.c		optional acpi
 contrib/dev/acpica/components/parser/pstree.c		optional acpi
 contrib/dev/acpica/components/parser/psutils.c		optional acpi
 contrib/dev/acpica/components/parser/pswalk.c		optional acpi
 contrib/dev/acpica/components/parser/psxface.c		optional acpi
 contrib/dev/acpica/components/resources/rsaddr.c	optional acpi
 contrib/dev/acpica/components/resources/rscalc.c	optional acpi
 contrib/dev/acpica/components/resources/rscreate.c	optional acpi
 contrib/dev/acpica/components/resources/rsdump.c	optional acpi
 contrib/dev/acpica/components/resources/rsdumpinfo.c	optional acpi
 contrib/dev/acpica/components/resources/rsinfo.c	optional acpi
 contrib/dev/acpica/components/resources/rsio.c		optional acpi
 contrib/dev/acpica/components/resources/rsirq.c		optional acpi
 contrib/dev/acpica/components/resources/rslist.c	optional acpi
 contrib/dev/acpica/components/resources/rsmemory.c	optional acpi
 contrib/dev/acpica/components/resources/rsmisc.c	optional acpi
 contrib/dev/acpica/components/resources/rsserial.c	optional acpi
 contrib/dev/acpica/components/resources/rsutils.c	optional acpi
 contrib/dev/acpica/components/resources/rsxface.c	optional acpi
 contrib/dev/acpica/components/tables/tbfadt.c		optional acpi
 contrib/dev/acpica/components/tables/tbfind.c		optional acpi
 contrib/dev/acpica/components/tables/tbinstal.c		optional acpi
 contrib/dev/acpica/components/tables/tbprint.c		optional acpi
 contrib/dev/acpica/components/tables/tbutils.c		optional acpi
 contrib/dev/acpica/components/tables/tbxface.c		optional acpi
 contrib/dev/acpica/components/tables/tbxfload.c		optional acpi
 contrib/dev/acpica/components/tables/tbxfroot.c		optional acpi
 contrib/dev/acpica/components/utilities/utaddress.c	optional acpi
 contrib/dev/acpica/components/utilities/utalloc.c	optional acpi
 contrib/dev/acpica/components/utilities/utbuffer.c	optional acpi
 contrib/dev/acpica/components/utilities/utcache.c	optional acpi
 contrib/dev/acpica/components/utilities/utcopy.c	optional acpi
 contrib/dev/acpica/components/utilities/utdebug.c	optional acpi
 contrib/dev/acpica/components/utilities/utdecode.c	optional acpi
 contrib/dev/acpica/components/utilities/utdelete.c	optional acpi
 contrib/dev/acpica/components/utilities/uterror.c	optional acpi
 contrib/dev/acpica/components/utilities/uteval.c	optional acpi
 contrib/dev/acpica/components/utilities/utexcep.c	optional acpi
 contrib/dev/acpica/components/utilities/utglobal.c	optional acpi
 contrib/dev/acpica/components/utilities/utids.c		optional acpi
 contrib/dev/acpica/components/utilities/utinit.c	optional acpi
 contrib/dev/acpica/components/utilities/utlock.c	optional acpi
 contrib/dev/acpica/components/utilities/utmath.c	optional acpi
 contrib/dev/acpica/components/utilities/utmisc.c	optional acpi
 contrib/dev/acpica/components/utilities/utmutex.c	optional acpi
 contrib/dev/acpica/components/utilities/utobject.c	optional acpi
 contrib/dev/acpica/components/utilities/utosi.c		optional acpi
 contrib/dev/acpica/components/utilities/utownerid.c	optional acpi
 contrib/dev/acpica/components/utilities/utpredef.c	optional acpi
 contrib/dev/acpica/components/utilities/utresrc.c	optional acpi
 contrib/dev/acpica/components/utilities/utstate.c	optional acpi
 contrib/dev/acpica/components/utilities/utstring.c	optional acpi
 contrib/dev/acpica/components/utilities/utxface.c	optional acpi
 contrib/dev/acpica/components/utilities/utxferror.c	optional acpi
 contrib/dev/acpica/components/utilities/utxfinit.c	optional acpi
 #contrib/dev/acpica/components/utilities/utxfmutex.c	optional acpi
 contrib/ipfilter/netinet/fil.c	optional ipfilter inet \
 	compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN} -Wno-unused -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/ip_auth.c optional ipfilter inet \
 	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/ip_fil_freebsd.c optional ipfilter inet \
 	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/ip_frag.c optional ipfilter inet \
 	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/ip_log.c optional ipfilter inet \
 	compile-with "${NORMAL_C} -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/ip_nat.c optional ipfilter inet \
 	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/ip_proxy.c optional ipfilter inet \
 	compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN} -Wno-unused -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/ip_state.c optional ipfilter inet \
 	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/ip_lookup.c optional ipfilter inet \
 	compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN} -Wno-unused -Wno-error -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/ip_pool.c optional ipfilter inet \
 	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/ip_htable.c optional ipfilter inet \
 	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/ip_sync.c optional ipfilter inet \
 	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/mlfk_ipl.c optional ipfilter inet \
 	compile-with "${NORMAL_C} -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/ip_nat6.c optional ipfilter inet \
 	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/ip_rules.c optional ipfilter inet \
 	compile-with "${NORMAL_C} -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/ip_scan.c optional ipfilter inet \
 	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/ip_dstlist.c optional ipfilter inet \
 	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
 contrib/ipfilter/netinet/radix_ipf.c optional ipfilter inet \
 	compile-with "${NORMAL_C} -I$S/contrib/ipfilter"
 contrib/libfdt/fdt.c		optional fdt
 contrib/libfdt/fdt_ro.c		optional fdt
 contrib/libfdt/fdt_rw.c		optional fdt
 contrib/libfdt/fdt_strerror.c	optional fdt
 contrib/libfdt/fdt_sw.c		optional fdt
 contrib/libfdt/fdt_wip.c	optional fdt
 contrib/ngatm/netnatm/api/cc_conn.c optional ngatm_ccatm \
 	compile-with "${NORMAL_C_NOWERROR} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/api/cc_data.c optional ngatm_ccatm \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/api/cc_dump.c optional ngatm_ccatm \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/api/cc_port.c optional ngatm_ccatm \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/api/cc_sig.c optional ngatm_ccatm \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/api/cc_user.c optional ngatm_ccatm \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/api/unisap.c optional ngatm_ccatm \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/misc/straddr.c optional ngatm_atmbase \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/misc/unimsg_common.c optional ngatm_atmbase \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/msg/traffic.c optional ngatm_atmbase \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/msg/uni_ie.c optional ngatm_atmbase \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/msg/uni_msg.c optional ngatm_atmbase \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/saal/saal_sscfu.c	optional ngatm_sscfu \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/saal/saal_sscop.c	optional ngatm_sscop \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/sig/sig_call.c optional ngatm_uni \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/sig/sig_coord.c optional ngatm_uni \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/sig/sig_party.c optional ngatm_uni \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/sig/sig_print.c optional ngatm_uni \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/sig/sig_reset.c optional ngatm_uni \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/sig/sig_uni.c optional ngatm_uni \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/sig/sig_unimsgcpy.c optional ngatm_uni \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 contrib/ngatm/netnatm/sig/sig_verify.c optional ngatm_uni \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 crypto/blowfish/bf_ecb.c	optional ipsec
 crypto/blowfish/bf_skey.c	optional crypto | ipsec
 crypto/camellia/camellia.c	optional crypto | ipsec
 crypto/camellia/camellia-api.c	optional crypto | ipsec
 crypto/des/des_ecb.c		optional crypto | ipsec | netsmb
 crypto/des/des_setkey.c		optional crypto | ipsec | netsmb
 crypto/rc4/rc4.c		optional netgraph_mppc_encryption | kgssapi
 crypto/rijndael/rijndael-alg-fst.c optional crypto | geom_bde | \
 					 ipsec | random | wlan_ccmp
 crypto/rijndael/rijndael-api-fst.c optional geom_bde | random
 crypto/rijndael/rijndael-api.c	optional crypto | ipsec | wlan_ccmp
 crypto/sha1.c			optional carp | crypto | ipsec | \
 					 netgraph_mppc_encryption | sctp
 crypto/sha2/sha2.c		optional crypto | geom_bde | ipsec | random | \
 					 sctp | zfs
 crypto/siphash/siphash.c	optional inet | inet6
 crypto/siphash/siphash_test.c	optional inet | inet6
 ddb/db_access.c			optional ddb
 ddb/db_break.c			optional ddb
 ddb/db_capture.c		optional ddb
 ddb/db_command.c		optional ddb
 ddb/db_examine.c		optional ddb
 ddb/db_expr.c			optional ddb
 ddb/db_input.c			optional ddb
 ddb/db_lex.c			optional ddb
 ddb/db_main.c			optional ddb
 ddb/db_output.c			optional ddb
 ddb/db_print.c			optional ddb
 ddb/db_ps.c			optional ddb
 ddb/db_run.c			optional ddb
 ddb/db_script.c			optional ddb
 ddb/db_sym.c			optional ddb
 ddb/db_thread.c			optional ddb
 ddb/db_textdump.c		optional ddb
 ddb/db_variables.c		optional ddb
 ddb/db_watch.c			optional ddb
 ddb/db_write_cmd.c		optional ddb
 #dev/dpt/dpt_control.c		optional dpt
 dev/aac/aac.c			optional aac
 dev/aac/aac_cam.c		optional aacp aac
 dev/aac/aac_debug.c		optional aac
 dev/aac/aac_disk.c		optional aac
 dev/aac/aac_linux.c		optional aac compat_linux
 dev/aac/aac_pci.c		optional aac pci
 dev/aacraid/aacraid.c		optional aacraid
 dev/aacraid/aacraid_cam.c	optional aacraid scbus
 dev/aacraid/aacraid_debug.c	optional aacraid
 dev/aacraid/aacraid_linux.c	optional aacraid compat_linux
 dev/aacraid/aacraid_pci.c	optional aacraid pci
 dev/acpi_support/acpi_wmi.c	optional acpi_wmi acpi
 dev/acpi_support/acpi_asus.c	optional acpi_asus acpi
 dev/acpi_support/acpi_asus_wmi.c	optional acpi_asus_wmi acpi
 dev/acpi_support/acpi_fujitsu.c	optional acpi_fujitsu acpi
 dev/acpi_support/acpi_hp.c	optional acpi_hp acpi
 dev/acpi_support/acpi_ibm.c	optional acpi_ibm acpi
 dev/acpi_support/acpi_panasonic.c optional acpi_panasonic acpi
 dev/acpi_support/acpi_sony.c	optional acpi_sony acpi
 dev/acpi_support/acpi_toshiba.c	optional acpi_toshiba acpi
 dev/acpi_support/atk0110.c	optional aibs acpi
 dev/acpica/Osd/OsdDebug.c	optional acpi
 dev/acpica/Osd/OsdHardware.c	optional acpi
 dev/acpica/Osd/OsdInterrupt.c	optional acpi
 dev/acpica/Osd/OsdMemory.c	optional acpi
 dev/acpica/Osd/OsdSchedule.c	optional acpi
 dev/acpica/Osd/OsdStream.c	optional acpi
 dev/acpica/Osd/OsdSynch.c	optional acpi
 dev/acpica/Osd/OsdTable.c	optional acpi
 dev/acpica/acpi.c		optional acpi
 dev/acpica/acpi_acad.c		optional acpi
 dev/acpica/acpi_battery.c	optional acpi
 dev/acpica/acpi_button.c	optional acpi
 dev/acpica/acpi_cmbat.c		optional acpi
 dev/acpica/acpi_cpu.c		optional acpi
 dev/acpica/acpi_ec.c		optional acpi
 dev/acpica/acpi_hpet.c		optional acpi
 dev/acpica/acpi_isab.c		optional acpi isa
 dev/acpica/acpi_lid.c		optional acpi
 dev/acpica/acpi_package.c	optional acpi
 dev/acpica/acpi_pci.c		optional acpi pci
 dev/acpica/acpi_pci_link.c	optional acpi pci
 dev/acpica/acpi_pcib.c		optional acpi pci
 dev/acpica/acpi_pcib_acpi.c	optional acpi pci
 dev/acpica/acpi_pcib_pci.c	optional acpi pci
 dev/acpica/acpi_perf.c		optional acpi
 dev/acpica/acpi_powerres.c	optional acpi
 dev/acpica/acpi_quirk.c		optional acpi
 dev/acpica/acpi_resource.c	optional acpi
 dev/acpica/acpi_smbat.c		optional acpi
 dev/acpica/acpi_thermal.c	optional acpi
 dev/acpica/acpi_throttle.c	optional acpi
 dev/acpica/acpi_timer.c		optional acpi
 dev/acpica/acpi_video.c		optional acpi_video acpi
 dev/acpica/acpi_dock.c		optional acpi_dock acpi
 dev/adlink/adlink.c		optional adlink
 dev/advansys/adv_eisa.c		optional adv eisa
 dev/advansys/adv_pci.c		optional adv pci
 dev/advansys/advansys.c		optional adv
 dev/advansys/advlib.c		optional adv
 dev/advansys/advmcode.c		optional adv
 dev/advansys/adw_pci.c		optional adw pci
 dev/advansys/adwcam.c		optional adw
 dev/advansys/adwlib.c		optional adw
 dev/advansys/adwmcode.c		optional adw
 dev/ae/if_ae.c			optional ae pci
 dev/age/if_age.c		optional age pci
 dev/agp/agp.c			optional agp pci
 dev/agp/agp_if.m		optional agp pci
 dev/aha/aha.c			optional aha
 dev/aha/aha_isa.c		optional aha isa
 dev/aha/aha_mca.c		optional aha mca
 dev/ahb/ahb.c			optional ahb eisa
 dev/ahci/ahci.c			optional ahci pci
 dev/ahci/ahciem.c		optional ahci pci
 dev/aic/aic.c			optional aic
 dev/aic/aic_pccard.c		optional aic pccard
 dev/aic7xxx/ahc_eisa.c		optional ahc eisa
 dev/aic7xxx/ahc_isa.c		optional ahc isa
 dev/aic7xxx/ahc_pci.c		optional ahc pci \
 	compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}"
 dev/aic7xxx/ahd_pci.c		optional ahd pci \
 	compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}"
 dev/aic7xxx/aic7770.c		optional ahc
 dev/aic7xxx/aic79xx.c		optional ahd pci
 dev/aic7xxx/aic79xx_osm.c	optional ahd pci
 dev/aic7xxx/aic79xx_pci.c	optional ahd pci
 dev/aic7xxx/aic79xx_reg_print.c	optional ahd pci ahd_reg_pretty_print
 dev/aic7xxx/aic7xxx.c		optional ahc
 dev/aic7xxx/aic7xxx_93cx6.c	optional ahc
 dev/aic7xxx/aic7xxx_osm.c	optional ahc
 dev/aic7xxx/aic7xxx_pci.c	optional ahc pci
 dev/aic7xxx/aic7xxx_reg_print.c	optional ahc ahc_reg_pretty_print
 dev/alc/if_alc.c		optional alc pci
 dev/ale/if_ale.c		optional ale pci
 dev/altera/avgen/altera_avgen.c		optional altera_avgen
 dev/altera/avgen/altera_avgen_fdt.c	optional altera_avgen fdt
 dev/altera/avgen/altera_avgen_nexus.c	optional altera_avgen
 dev/altera/sdcard/altera_sdcard.c	optional altera_sdcard
 dev/altera/sdcard/altera_sdcard_disk.c	optional altera_sdcard
 dev/altera/sdcard/altera_sdcard_io.c	optional altera_sdcard
 dev/altera/sdcard/altera_sdcard_fdt.c	optional altera_sdcard fdt
 dev/altera/sdcard/altera_sdcard_nexus.c	optional altera_sdcard
 dev/amr/amr.c			optional amr
 dev/amr/amr_cam.c		optional amrp amr
 dev/amr/amr_disk.c		optional amr
 dev/amr/amr_linux.c		optional amr compat_linux
 dev/amr/amr_pci.c		optional amr pci
 dev/an/if_an.c			optional an
 dev/an/if_an_isa.c		optional an isa
 dev/an/if_an_pccard.c		optional an pccard
 dev/an/if_an_pci.c		optional an pci
 dev/asr/asr.c			optional asr pci \
 	compile-with "${NORMAL_C} ${NO_WARRAY_BOUNDS}"
 #
 dev/ata/ata_if.m		optional ata | atacore
 dev/ata/ata-all.c		optional ata | atacore
 dev/ata/ata-dma.c		optional ata | atacore
 dev/ata/ata-lowlevel.c		optional ata | atacore
 dev/ata/ata-sata.c		optional ata | atacore
 dev/ata/ata-card.c		optional ata pccard | atapccard
 dev/ata/ata-cbus.c		optional ata pc98 | atapc98
 dev/ata/ata-isa.c		optional ata isa | ataisa
 dev/ata/ata-pci.c		optional ata pci | atapci
 dev/ata/chipsets/ata-ahci.c	optional ata pci | ataahci | ataacerlabs | \
 					 ataati | ataintel | atajmicron | \
 					 atavia | atanvidia
 dev/ata/chipsets/ata-acard.c	optional ata pci | ataacard
 dev/ata/chipsets/ata-acerlabs.c	optional ata pci | ataacerlabs
 dev/ata/chipsets/ata-adaptec.c	optional ata pci | ataadaptec
 dev/ata/chipsets/ata-amd.c	optional ata pci | ataamd
 dev/ata/chipsets/ata-ati.c	optional ata pci | ataati
 dev/ata/chipsets/ata-cenatek.c	optional ata pci | atacenatek
 dev/ata/chipsets/ata-cypress.c	optional ata pci | atacypress
 dev/ata/chipsets/ata-cyrix.c	optional ata pci | atacyrix
 dev/ata/chipsets/ata-highpoint.c	optional ata pci | atahighpoint
 dev/ata/chipsets/ata-intel.c	optional ata pci | ataintel
 dev/ata/chipsets/ata-ite.c	optional ata pci | ataite
 dev/ata/chipsets/ata-jmicron.c	optional ata pci | atajmicron
 dev/ata/chipsets/ata-marvell.c	optional ata pci | atamarvell | ataadaptec
 dev/ata/chipsets/ata-micron.c	optional ata pci | atamicron
 dev/ata/chipsets/ata-national.c	optional ata pci | atanational
 dev/ata/chipsets/ata-netcell.c	optional ata pci | atanetcell
 dev/ata/chipsets/ata-nvidia.c	optional ata pci | atanvidia
 dev/ata/chipsets/ata-promise.c	optional ata pci | atapromise
 dev/ata/chipsets/ata-serverworks.c	optional ata pci | ataserverworks
 dev/ata/chipsets/ata-siliconimage.c	optional ata pci | atasiliconimage | ataati
 dev/ata/chipsets/ata-sis.c	optional ata pci | atasis
 dev/ata/chipsets/ata-via.c	optional ata pci | atavia
 #
 dev/ath/if_ath_pci.c		optional ath_pci pci \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 #
 dev/ath/if_ath_ahb.c		optional ath_ahb \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 #
 dev/ath/if_ath.c		optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/if_ath_beacon.c		optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/if_ath_btcoex.c		optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/if_ath_debug.c		optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/if_ath_keycache.c	optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/if_ath_led.c		optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/if_ath_lna_div.c	optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/if_ath_tx.c		optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/if_ath_tx_edma.c	optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/if_ath_tx_ht.c		optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/if_ath_tdma.c		optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/if_ath_sysctl.c		optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/if_ath_rx.c		optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/if_ath_rx_edma.c	optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/if_ath_spectral.c	optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/ah_osdep.c		optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 #
 dev/ath/ath_hal/ah.c		optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/ath_hal/ah_eeprom_v1.c	optional ath_hal | ath_ar5210 \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/ath_hal/ah_eeprom_v3.c	optional ath_hal | ath_ar5211 | ath_ar5212 \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/ath_hal/ah_eeprom_v14.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/ath_hal/ah_eeprom_v4k.c \
 	optional ath_hal | ath_ar9285 \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/ath_hal/ah_eeprom_9287.c \
 	optional ath_hal | ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/ath_hal/ah_regdomain.c	optional ath \
 	compile-with "${NORMAL_C} ${NO_WSHIFT_COUNT_NEGATIVE} ${NO_WSHIFT_COUNT_OVERFLOW} -I$S/dev/ath"
 # ar5210
 dev/ath/ath_hal/ar5210/ar5210_attach.c		optional ath_hal | ath_ar5210 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5210/ar5210_beacon.c		optional ath_hal | ath_ar5210 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5210/ar5210_interrupts.c	optional ath_hal | ath_ar5210 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5210/ar5210_keycache.c	optional ath_hal | ath_ar5210 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5210/ar5210_misc.c		optional ath_hal | ath_ar5210 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5210/ar5210_phy.c		optional ath_hal | ath_ar5210 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5210/ar5210_power.c		optional ath_hal | ath_ar5210 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5210/ar5210_recv.c		optional ath_hal | ath_ar5210 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5210/ar5210_reset.c		optional ath_hal | ath_ar5210 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5210/ar5210_xmit.c		optional ath_hal | ath_ar5210 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 # ar5211
 dev/ath/ath_hal/ar5211/ar5211_attach.c		optional ath_hal | ath_ar5211 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5211/ar5211_beacon.c		optional ath_hal | ath_ar5211 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5211/ar5211_interrupts.c	optional ath_hal | ath_ar5211 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5211/ar5211_keycache.c	optional ath_hal | ath_ar5211 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5211/ar5211_misc.c		optional ath_hal | ath_ar5211 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5211/ar5211_phy.c		optional ath_hal | ath_ar5211 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5211/ar5211_power.c		optional ath_hal | ath_ar5211 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5211/ar5211_recv.c		optional ath_hal | ath_ar5211 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5211/ar5211_reset.c		optional ath_hal | ath_ar5211 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5211/ar5211_xmit.c		optional ath_hal | ath_ar5211 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 # ar5212
 dev/ath/ath_hal/ar5212/ar5212_ani.c \
 	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \
 	ath_ar9285 ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_attach.c \
 	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \
 	ath_ar9285 ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_beacon.c \
 	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \
 	ath_ar9285 ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_eeprom.c \
 	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \
 	ath_ar9285 ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_gpio.c \
 	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \
 	ath_ar9285 ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_interrupts.c \
 	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \
 	ath_ar9285 ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_keycache.c \
 	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \
 	ath_ar9285 ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_misc.c \
 	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \
 	ath_ar9285 ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_phy.c \
 	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \
 	ath_ar9285 ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_power.c \
 	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \
 	ath_ar9285 ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_recv.c \
 	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \
 	ath_ar9285 ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_reset.c \
 	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \
 	ath_ar9285 ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_rfgain.c \
 	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \
 	ath_ar9285 ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_xmit.c \
 	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \
 	ath_ar9285 ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 # ar5416 (depends on ar5212)
 dev/ath/ath_hal/ar5416/ar5416_ani.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_attach.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_beacon.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_btcoex.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_cal.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_cal_iq.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_cal_adcgain.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_cal_adcdc.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_eeprom.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_gpio.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_interrupts.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_keycache.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_misc.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_phy.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_power.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_radar.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_recv.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_reset.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_spectral.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_xmit.c \
 	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \
 	ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 # ar9130 (depends upon ar5416) - also requires AH_SUPPORT_AR9130
 #
 # Since this is an embedded MAC SoC, there's no need to compile it into the
 # default HAL.
 dev/ath/ath_hal/ar9001/ar9130_attach.c optional ath_ar9130 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar9001/ar9130_phy.c optional ath_ar9130 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar9001/ar9130_eeprom.c optional ath_ar9130 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 # ar9160 (depends on ar5416)
 dev/ath/ath_hal/ar9001/ar9160_attach.c optional ath_hal | ath_ar9160 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 # ar9280 (depends on ar5416)
 dev/ath/ath_hal/ar9002/ar9280_attach.c optional ath_hal | ath_ar9280 | \
 	ath_ar9285 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar9002/ar9280_olc.c optional ath_hal | ath_ar9280 | \
 	ath_ar9285 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 # ar9285 (depends on ar5416 and ar9280)
 dev/ath/ath_hal/ar9002/ar9285_attach.c optional ath_hal | ath_ar9285 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar9002/ar9285_btcoex.c optional ath_hal | ath_ar9285 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar9002/ar9285_reset.c optional ath_hal | ath_ar9285 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar9002/ar9285_cal.c optional ath_hal | ath_ar9285 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar9002/ar9285_phy.c optional ath_hal | ath_ar9285 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar9002/ar9285_diversity.c optional ath_hal | ath_ar9285 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 # ar9287 (depends on ar5416)
 dev/ath/ath_hal/ar9002/ar9287_attach.c optional ath_hal | ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar9002/ar9287_reset.c optional ath_hal | ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar9002/ar9287_cal.c optional ath_hal | ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar9002/ar9287_olc.c optional ath_hal | ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 
 # ar9300
 contrib/dev/ath/ath_hal/ar9300/ar9300_ani.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_attach.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_beacon.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_eeprom.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal ${NO_WCONSTANT_CONVERSION}"
 contrib/dev/ath/ath_hal/ar9300/ar9300_freebsd.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_gpio.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_interrupts.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_keycache.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_mci.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_misc.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_paprd.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_phy.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_power.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_radar.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_radio.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_recv.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_recv_ds.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_reset.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal ${NO_WSOMETIMES_UNINITIALIZED} -Wno-unused-function"
 contrib/dev/ath/ath_hal/ar9300/ar9300_stub.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_stub_funcs.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_timer.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_xmit.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 contrib/dev/ath/ath_hal/ar9300/ar9300_xmit_ds.c optional ath_hal | ath_ar9300 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
 
 # rf backends
 dev/ath/ath_hal/ar5212/ar2316.c	optional ath_rf2316 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar2317.c	optional ath_rf2317 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar2413.c	optional ath_hal | ath_rf2413 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar2425.c	optional ath_hal | ath_rf2425 | ath_rf2417 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5111.c	optional ath_hal | ath_rf5111 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5112.c	optional ath_hal | ath_rf5112 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5413.c	optional ath_hal | ath_rf5413 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar2133.c optional ath_hal | ath_ar5416 | \
 	ath_ar9130 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar9002/ar9280.c optional ath_hal | ath_ar9280 | ath_ar9285 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar9002/ar9285.c optional ath_hal | ath_ar9285 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar9002/ar9287.c optional ath_hal | ath_ar9287 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 
 # ath rate control algorithms
 dev/ath/ath_rate/amrr/amrr.c	optional ath_rate_amrr \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/ath_rate/onoe/onoe.c	optional ath_rate_onoe \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/ath_rate/sample/sample.c	optional ath_rate_sample \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 # ath DFS modules
 dev/ath/ath_dfs/null/dfs_null.c	optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 #
 dev/bce/if_bce.c		optional bce
 dev/bfe/if_bfe.c		optional bfe
 dev/bge/if_bge.c		optional bge
 dev/bktr/bktr_audio.c		optional bktr pci
 dev/bktr/bktr_card.c		optional bktr pci
 dev/bktr/bktr_core.c		optional bktr pci
 dev/bktr/bktr_i2c.c		optional bktr pci smbus
 dev/bktr/bktr_os.c		optional bktr pci
 dev/bktr/bktr_tuner.c		optional bktr pci
 dev/bktr/msp34xx.c		optional bktr pci
 dev/buslogic/bt.c		optional bt
 dev/buslogic/bt_eisa.c		optional bt eisa
 dev/buslogic/bt_isa.c		optional bt isa
 dev/buslogic/bt_mca.c		optional bt mca
 dev/buslogic/bt_pci.c		optional bt pci
 dev/bwi/bwimac.c		optional bwi
 dev/bwi/bwiphy.c		optional bwi
 dev/bwi/bwirf.c			optional bwi
 dev/bwi/if_bwi.c		optional bwi
 dev/bwi/if_bwi_pci.c		optional bwi pci
 # XXX Work around clang warning, until maintainer approves fix.
 dev/bwn/if_bwn.c		optional bwn siba_bwn \
 	compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}"
 dev/cardbus/cardbus.c		optional cardbus
 dev/cardbus/cardbus_cis.c	optional cardbus
 dev/cardbus/cardbus_device.c	optional cardbus
 dev/cas/if_cas.c		optional cas
 dev/cfi/cfi_bus_fdt.c		optional cfi fdt
 dev/cfi/cfi_bus_nexus.c		optional cfi
 dev/cfi/cfi_core.c		optional cfi
 dev/cfi/cfi_dev.c		optional cfi
 dev/cfi/cfi_disk.c		optional cfid
 dev/ciss/ciss.c			optional ciss
 dev/cm/smc90cx6.c		optional cm
 dev/cmx/cmx.c			optional cmx
 dev/cmx/cmx_pccard.c		optional cmx pccard
 dev/cpufreq/ichss.c		optional cpufreq
 dev/cs/if_cs.c			optional cs
 dev/cs/if_cs_isa.c		optional cs isa
 dev/cs/if_cs_pccard.c		optional cs pccard
 dev/cxgb/cxgb_main.c		optional cxgb pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgb"
 dev/cxgb/cxgb_sge.c		optional cxgb pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgb"
 dev/cxgb/common/cxgb_mc5.c	optional cxgb pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgb"
 dev/cxgb/common/cxgb_vsc7323.c	optional cxgb pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgb"
 dev/cxgb/common/cxgb_vsc8211.c	optional cxgb pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgb"
 dev/cxgb/common/cxgb_ael1002.c	optional cxgb pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgb"
 dev/cxgb/common/cxgb_aq100x.c	optional cxgb pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgb"
 dev/cxgb/common/cxgb_mv88e1xxx.c	optional cxgb pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgb"
 dev/cxgb/common/cxgb_xgmac.c	optional cxgb pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgb"
 dev/cxgb/common/cxgb_t3_hw.c	optional cxgb pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgb"
 dev/cxgb/common/cxgb_tn1010.c	optional cxgb pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgb"
 dev/cxgb/sys/uipc_mvec.c	optional cxgb pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgb"
 dev/cxgb/cxgb_t3fw.c		optional cxgb cxgb_t3fw \
 	compile-with "${NORMAL_C} -I$S/dev/cxgb"
 dev/cxgbe/t4_main.c		optional cxgbe pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
+dev/cxgbe/t4_netmap.c		optional cxgbe pci \
+	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
 dev/cxgbe/t4_sge.c		optional cxgbe pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
 dev/cxgbe/t4_l2t.c		optional cxgbe pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
 dev/cxgbe/t4_tracer.c		optional cxgbe pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
 dev/cxgbe/common/t4_hw.c	optional cxgbe pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
 t4fw_cfg.c		optional cxgbe					\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk t4fw_cfg.fw:t4fw_cfg t4fw_cfg_uwire.fw:t4fw_cfg_uwire t4fw.fw:t4fw -mt4fw_cfg -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"t4fw_cfg.c"
 t4fw_cfg.fwo		optional cxgbe					\
 	dependency	"t4fw_cfg.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"t4fw_cfg.fwo"
 t4fw_cfg.fw		optional cxgbe					\
 	dependency	"$S/dev/cxgbe/firmware/t4fw_cfg.txt"		\
 	compile-with	"${CP} ${.ALLSRC} ${.TARGET}"			\
 	no-obj no-implicit-rule						\
 	clean		"t4fw_cfg.fw"
 t4fw_cfg_uwire.fwo	optional cxgbe					\
 	dependency	"t4fw_cfg_uwire.fw"				\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"t4fw_cfg_uwire.fwo"
 t4fw_cfg_uwire.fw	optional cxgbe					\
 	dependency	"$S/dev/cxgbe/firmware/t4fw_cfg_uwire.txt"	\
 	compile-with	"${CP} ${.ALLSRC} ${.TARGET}"			\
 	no-obj no-implicit-rule						\
 	clean		"t4fw_cfg_uwire.fw"
 t4fw.fwo		optional cxgbe					\
 	dependency	"t4fw.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"t4fw.fwo"
 t4fw.fw			optional cxgbe					\
 	dependency	"$S/dev/cxgbe/firmware/t4fw-1.11.27.0.bin.uu"	\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"t4fw.fw"
 t5fw_cfg.c		optional cxgbe					\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk t5fw_cfg.fw:t5fw_cfg t5fw.fw:t5fw -mt5fw_cfg -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"t5fw_cfg.c"
 t5fw_cfg.fwo		optional cxgbe					\
 	dependency	"t5fw_cfg.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"t5fw_cfg.fwo"
 t5fw_cfg.fw		optional cxgbe					\
 	dependency	"$S/dev/cxgbe/firmware/t5fw_cfg.txt"		\
 	compile-with	"${CP} ${.ALLSRC} ${.TARGET}"			\
 	no-obj no-implicit-rule						\
 	clean		"t5fw_cfg.fw"
 t5fw.fwo		optional cxgbe					\
 	dependency	"t5fw.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"t5fw.fwo"
 t5fw.fw			optional cxgbe					\
 	dependency	"$S/dev/cxgbe/firmware/t5fw-1.11.27.0.bin.uu"	\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"t5fw.fw"
 dev/cy/cy.c			optional cy
 dev/cy/cy_isa.c			optional cy isa
 dev/cy/cy_pci.c			optional cy pci
 dev/dc/if_dc.c			optional dc pci
 dev/dc/dcphy.c			optional dc pci
 dev/dc/pnphy.c			optional dc pci
 dev/dcons/dcons.c		optional dcons
 dev/dcons/dcons_crom.c		optional dcons_crom
 dev/dcons/dcons_os.c		optional dcons
 dev/de/if_de.c			optional de pci
 dev/digi/CX.c			optional digi_CX
 dev/digi/CX_PCI.c		optional digi_CX_PCI
 dev/digi/EPCX.c			optional digi_EPCX
 dev/digi/EPCX_PCI.c		optional digi_EPCX_PCI
 dev/digi/Xe.c			optional digi_Xe
 dev/digi/Xem.c			optional digi_Xem
 dev/digi/Xr.c			optional digi_Xr
 dev/digi/digi.c			optional digi
 dev/digi/digi_isa.c		optional digi isa
 dev/digi/digi_pci.c		optional digi pci
 dev/dpt/dpt_eisa.c		optional dpt eisa
 dev/dpt/dpt_pci.c		optional dpt pci
 dev/dpt/dpt_scsi.c		optional dpt
 dev/drm/ati_pcigart.c		optional drm
 dev/drm/drm_agpsupport.c	optional drm
 dev/drm/drm_auth.c		optional drm
 dev/drm/drm_bufs.c		optional drm
 dev/drm/drm_context.c		optional drm
 dev/drm/drm_dma.c		optional drm
 dev/drm/drm_drawable.c		optional drm
 dev/drm/drm_drv.c		optional drm
 dev/drm/drm_fops.c		optional drm
 dev/drm/drm_hashtab.c		optional drm
 dev/drm/drm_ioctl.c		optional drm
 dev/drm/drm_irq.c		optional drm
 dev/drm/drm_lock.c		optional drm
 dev/drm/drm_memory.c		optional drm
 dev/drm/drm_mm.c		optional drm
 dev/drm/drm_pci.c		optional drm
 dev/drm/drm_scatter.c		optional drm
 dev/drm/drm_sman.c		optional drm
 dev/drm/drm_sysctl.c		optional drm
 dev/drm/drm_vm.c		optional drm
 dev/drm/i915_dma.c		optional i915drm
 dev/drm/i915_drv.c		optional i915drm
 dev/drm/i915_irq.c		optional i915drm
 dev/drm/i915_mem.c		optional i915drm
 dev/drm/i915_suspend.c		optional i915drm
 dev/drm/mach64_dma.c		optional mach64drm
 dev/drm/mach64_drv.c		optional mach64drm
 dev/drm/mach64_irq.c		optional mach64drm
 dev/drm/mach64_state.c		optional mach64drm
 dev/drm/mga_dma.c		optional mgadrm
 dev/drm/mga_drv.c		optional mgadrm
 dev/drm/mga_irq.c		optional mgadrm
 dev/drm/mga_state.c		optional mgadrm
 dev/drm/mga_warp.c		optional mgadrm
 dev/drm/r128_cce.c		optional r128drm \
 	compile-with "${NORMAL_C} ${NO_WUNUSED_VALUE} ${NO_WCONSTANT_CONVERSION}"
 dev/drm/r128_drv.c		optional r128drm
 dev/drm/r128_irq.c		optional r128drm
 dev/drm/r128_state.c		optional r128drm \
 	compile-with "${NORMAL_C} ${NO_WUNUSED_VALUE}"
 dev/drm/r300_cmdbuf.c		optional radeondrm
 dev/drm/r600_blit.c		optional radeondrm
 dev/drm/r600_cp.c		optional radeondrm \
 	compile-with "${NORMAL_C} ${NO_WUNUSED_VALUE} ${NO_WCONSTANT_CONVERSION}"
 dev/drm/radeon_cp.c		optional radeondrm \
 	compile-with "${NORMAL_C} ${NO_WUNUSED_VALUE} ${NO_WCONSTANT_CONVERSION}"
 dev/drm/radeon_cs.c		optional radeondrm
 dev/drm/radeon_drv.c		optional radeondrm
 dev/drm/radeon_irq.c		optional radeondrm
 dev/drm/radeon_mem.c		optional radeondrm
 dev/drm/radeon_state.c		optional radeondrm
 dev/drm/savage_bci.c		optional savagedrm
 dev/drm/savage_drv.c		optional savagedrm
 dev/drm/savage_state.c		optional savagedrm
 dev/drm/sis_drv.c		optional sisdrm
 dev/drm/sis_ds.c		optional sisdrm
 dev/drm/sis_mm.c		optional sisdrm
 dev/drm/tdfx_drv.c		optional tdfxdrm
 dev/drm/via_dma.c		optional viadrm
 dev/drm/via_dmablit.c		optional viadrm
 dev/drm/via_drv.c		optional viadrm
 dev/drm/via_irq.c		optional viadrm
 dev/drm/via_map.c		optional viadrm
 dev/drm/via_mm.c		optional viadrm
 dev/drm/via_verifier.c		optional viadrm
 dev/drm/via_video.c		optional viadrm
 dev/ed/if_ed.c			optional ed
 dev/ed/if_ed_novell.c		optional ed
 dev/ed/if_ed_rtl80x9.c		optional ed
 dev/ed/if_ed_pccard.c		optional ed pccard
 dev/ed/if_ed_pci.c		optional ed pci
 dev/eisa/eisa_if.m		standard
 dev/eisa/eisaconf.c		optional eisa
 dev/e1000/if_em.c		optional em \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/if_lem.c		optional em \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/if_igb.c		optional igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_80003es2lan.c	optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_82540.c		optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_82541.c		optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_82542.c		optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_82543.c		optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_82571.c		optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_82575.c		optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_ich8lan.c	optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_i210.c		optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_api.c		optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_mac.c		optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_manage.c	optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_nvm.c		optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_phy.c		optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_vf.c		optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_mbx.c		optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/e1000/e1000_osdep.c		optional em | igb \
 	compile-with "${NORMAL_C} -I$S/dev/e1000"
 dev/et/if_et.c			optional et
 dev/en/if_en_pci.c		optional en pci
 dev/en/midway.c			optional en
 dev/ep/if_ep.c			optional ep
 dev/ep/if_ep_eisa.c		optional ep eisa
 dev/ep/if_ep_isa.c		optional ep isa
 dev/ep/if_ep_mca.c		optional ep mca
 dev/ep/if_ep_pccard.c		optional ep pccard
 dev/esp/esp_pci.c		optional esp pci
 dev/esp/ncr53c9x.c		optional esp
 dev/etherswitch/arswitch/arswitch.c		optional arswitch
 dev/etherswitch/arswitch/arswitch_reg.c		optional arswitch
 dev/etherswitch/arswitch/arswitch_phy.c		optional arswitch
 dev/etherswitch/arswitch/arswitch_8216.c	optional arswitch
 dev/etherswitch/arswitch/arswitch_8226.c	optional arswitch
 dev/etherswitch/arswitch/arswitch_8316.c	optional arswitch
 dev/etherswitch/arswitch/arswitch_7240.c	optional arswitch
 dev/etherswitch/arswitch/arswitch_vlans.c	optional arswitch
 dev/etherswitch/etherswitch.c		optional etherswitch
 dev/etherswitch/etherswitch_if.m	optional etherswitch
 dev/etherswitch/ip17x/ip17x.c		optional ip17x
 dev/etherswitch/ip17x/ip175c.c		optional ip17x
 dev/etherswitch/ip17x/ip175d.c		optional ip17x
 dev/etherswitch/ip17x/ip17x_phy.c	optional ip17x
 dev/etherswitch/ip17x/ip17x_vlans.c	optional ip17x
 dev/etherswitch/mdio_if.m		optional miiproxy
 dev/etherswitch/mdio.c			optional miiproxy
 dev/etherswitch/miiproxy.c		optional miiproxy
 dev/etherswitch/rtl8366/rtl8366rb.c	optional rtl8366rb
 dev/etherswitch/ukswitch/ukswitch.c	optional ukswitch
 dev/ex/if_ex.c			optional ex
 dev/ex/if_ex_isa.c		optional ex isa
 dev/ex/if_ex_pccard.c		optional ex pccard
 dev/exca/exca.c			optional cbb
 dev/fatm/if_fatm.c		optional fatm pci
 dev/fb/fbd.c			optional fbd | vt
 dev/fb/fb_if.m			standard
 dev/fb/splash.c			optional sc splash
 dev/fdt/fdt_common.c		optional fdt
 dev/fdt/fdt_slicer.c		optional fdt cfi | fdt nand
 dev/fdt/fdt_static_dtb.S	optional fdt fdt_dtb_static \
 	dependency	"$S/boot/fdt/dts/${MACHINE}/${FDT_DTS_FILE}"
 dev/fdt/simplebus.c		optional fdt
 dev/fe/if_fe.c			optional fe
 dev/fe/if_fe_pccard.c		optional fe pccard
 dev/filemon/filemon.c		optional filemon
 dev/firewire/firewire.c		optional firewire
 dev/firewire/fwcrom.c		optional firewire
 dev/firewire/fwdev.c		optional firewire
 dev/firewire/fwdma.c		optional firewire
 dev/firewire/fwmem.c		optional firewire
 dev/firewire/fwohci.c		optional firewire
 dev/firewire/fwohci_pci.c	optional firewire pci
 dev/firewire/if_fwe.c		optional fwe
 dev/firewire/if_fwip.c		optional fwip
 dev/firewire/sbp.c		optional sbp
 dev/firewire/sbp_targ.c		optional sbp_targ
 dev/flash/at45d.c		optional at45d
 dev/flash/mx25l.c		optional mx25l
 dev/fxp/if_fxp.c		optional fxp
 dev/fxp/inphy.c			optional fxp
 dev/gem/if_gem.c		optional gem
 dev/gem/if_gem_pci.c		optional gem pci
 dev/gem/if_gem_sbus.c		optional gem sbus
 dev/gpio/gpiobus.c		optional gpio				\
 	dependency	"gpiobus_if.h"
 dev/gpio/gpioc.c		optional gpio				\
 	dependency	"gpio_if.h"
 dev/gpio/gpioiic.c		optional gpioiic
 dev/gpio/gpioled.c		optional gpioled
 dev/gpio/gpio_if.m		optional gpio
 dev/gpio/gpiobus_if.m		optional gpio
 dev/gpio/ofw_gpiobus.c		optional fdt gpio
 dev/hatm/if_hatm.c		optional hatm pci
 dev/hatm/if_hatm_intr.c		optional hatm pci
 dev/hatm/if_hatm_ioctl.c	optional hatm pci
 dev/hatm/if_hatm_rx.c		optional hatm pci
 dev/hatm/if_hatm_tx.c		optional hatm pci
 dev/hifn/hifn7751.c		optional hifn
 dev/hme/if_hme.c		optional hme
 dev/hme/if_hme_pci.c		optional hme pci
 dev/hme/if_hme_sbus.c		optional hme sbus
 dev/hptiop/hptiop.c		optional hptiop scbus
 dev/hwpmc/hwpmc_logging.c	optional hwpmc
 dev/hwpmc/hwpmc_mod.c		optional hwpmc
 dev/hwpmc/hwpmc_soft.c		optional hwpmc
 dev/ichsmb/ichsmb.c		optional ichsmb
 dev/ichsmb/ichsmb_pci.c		optional ichsmb pci
 dev/ida/ida.c			optional ida
 dev/ida/ida_disk.c		optional ida
 dev/ida/ida_eisa.c		optional ida eisa
 dev/ida/ida_pci.c		optional ida pci
 dev/ie/if_ie.c			optional ie isa nowerror
 dev/ie/if_ie_isa.c		optional ie isa
 dev/ieee488/ibfoo.c		optional pcii | tnt4882
 dev/ieee488/pcii.c		optional pcii
 dev/ieee488/tnt4882.c		optional tnt4882
 dev/ieee488/upd7210.c		optional pcii | tnt4882
 dev/iicbus/ad7418.c		optional ad7418
 dev/iicbus/ds133x.c		optional ds133x
 dev/iicbus/ds1374.c		optional ds1374
 dev/iicbus/ds1672.c		optional ds1672
 dev/iicbus/icee.c		optional icee
 dev/iicbus/if_ic.c		optional ic
 dev/iicbus/iic.c		optional iic
 dev/iicbus/iicbb.c		optional iicbb
 dev/iicbus/iicbb_if.m		optional iicbb
 dev/iicbus/iicbus.c		optional iicbus
 dev/iicbus/iicbus_if.m		optional iicbus
 dev/iicbus/iiconf.c		optional iicbus
 dev/iicbus/iicsmb.c		optional iicsmb				\
 	dependency	"iicbus_if.h"
 dev/iicbus/iicoc.c		optional iicoc
 dev/iicbus/pcf8563.c		optional pcf8563
 dev/iicbus/s35390a.c		optional s35390a
 dev/iir/iir.c			optional iir
 dev/iir/iir_ctrl.c		optional iir
 dev/iir/iir_pci.c		optional iir pci
 # XXX Work around clang warning, until maintainer approves fix.
 dev/ips/ips.c			optional ips \
 	compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}"
 dev/ips/ips_commands.c		optional ips
 dev/ips/ips_disk.c		optional ips
 dev/ips/ips_ioctl.c		optional ips
 dev/ips/ips_pci.c		optional ips pci
 dev/ipw/if_ipw.c		optional ipw
 ipwbssfw.c			optional ipwbssfw | ipwfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk ipw_bss.fw:ipw_bss:130 -lintel_ipw -mipw_bss -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"ipwbssfw.c"
 ipw_bss.fwo			optional ipwbssfw | ipwfw		\
 	dependency	"ipw_bss.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"ipw_bss.fwo"
 ipw_bss.fw			optional ipwbssfw | ipwfw		\
 	dependency	"$S/contrib/dev/ipw/ipw2100-1.3.fw.uu"		\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"ipw_bss.fw"
 ipwibssfw.c			optional ipwibssfw | ipwfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk ipw_ibss.fw:ipw_ibss:130 -lintel_ipw -mipw_ibss -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"ipwibssfw.c"
 ipw_ibss.fwo			optional ipwibssfw | ipwfw		\
 	dependency	"ipw_ibss.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"ipw_ibss.fwo"
 ipw_ibss.fw			optional ipwibssfw | ipwfw		\
 	dependency	"$S/contrib/dev/ipw/ipw2100-1.3-i.fw.uu"	\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"ipw_ibss.fw"
 ipwmonitorfw.c			optional ipwmonitorfw | ipwfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk ipw_monitor.fw:ipw_monitor:130 -lintel_ipw -mipw_monitor -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"ipwmonitorfw.c"
 ipw_monitor.fwo			optional ipwmonitorfw | ipwfw		\
 	dependency	"ipw_monitor.fw"				\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"ipw_monitor.fwo"
 ipw_monitor.fw			optional ipwmonitorfw | ipwfw		\
 	dependency	"$S/contrib/dev/ipw/ipw2100-1.3-p.fw.uu"	\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"ipw_monitor.fw"
 dev/iscsi/icl.c			optional iscsi | ctl 
 dev/iscsi/icl_proxy.c		optional iscsi | ctl
 dev/iscsi/iscsi.c		optional iscsi scbus
 dev/iscsi_initiator/iscsi.c	optional iscsi_initiator scbus
 dev/iscsi_initiator/iscsi_subr.c	optional iscsi_initiator scbus
 dev/iscsi_initiator/isc_cam.c	optional iscsi_initiator scbus
 dev/iscsi_initiator/isc_soc.c	optional iscsi_initiator scbus
 dev/iscsi_initiator/isc_sm.c	optional iscsi_initiator scbus
 dev/iscsi_initiator/isc_subr.c	optional iscsi_initiator scbus
 dev/isp/isp.c			optional isp
 dev/isp/isp_freebsd.c		optional isp
 dev/isp/isp_library.c		optional isp
 dev/isp/isp_pci.c		optional isp pci
 dev/isp/isp_sbus.c		optional isp sbus
 dev/isp/isp_target.c		optional isp
 dev/ispfw/ispfw.c		optional ispfw
 dev/iwi/if_iwi.c		optional iwi
 iwibssfw.c			optional iwibssfw | iwifw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk iwi_bss.fw:iwi_bss:300 -lintel_iwi -miwi_bss -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"iwibssfw.c"
 iwi_bss.fwo			optional iwibssfw | iwifw		\
 	dependency	"iwi_bss.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"iwi_bss.fwo"
 iwi_bss.fw			optional iwibssfw | iwifw		\
 	dependency	"$S/contrib/dev/iwi/ipw2200-bss.fw.uu"		\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"iwi_bss.fw"
 iwiibssfw.c			optional iwiibssfw | iwifw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk iwi_ibss.fw:iwi_ibss:300 -lintel_iwi -miwi_ibss -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"iwiibssfw.c"
 iwi_ibss.fwo			optional iwiibssfw | iwifw		\
 	dependency	"iwi_ibss.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"iwi_ibss.fwo"
 iwi_ibss.fw			optional iwiibssfw | iwifw		\
 	dependency	"$S/contrib/dev/iwi/ipw2200-ibss.fw.uu"		\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"iwi_ibss.fw"
 iwimonitorfw.c			optional iwimonitorfw | iwifw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk iwi_monitor.fw:iwi_monitor:300 -lintel_iwi -miwi_monitor -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"iwimonitorfw.c"
 iwi_monitor.fwo			optional iwimonitorfw | iwifw		\
 	dependency	"iwi_monitor.fw"				\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"iwi_monitor.fwo"
 iwi_monitor.fw			optional iwimonitorfw | iwifw		\
 	dependency	"$S/contrib/dev/iwi/ipw2200-sniffer.fw.uu"	\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"iwi_monitor.fw"
 dev/iwn/if_iwn.c		optional iwn
 iwn1000fw.c			optional iwn1000fw | iwnfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk iwn1000.fw:iwn1000fw -miwn1000fw -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"iwn1000fw.c"
 iwn1000fw.fwo			optional iwn1000fw | iwnfw		\
 	dependency	"iwn1000.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"iwn1000fw.fwo"
 iwn1000.fw			optional iwn1000fw | iwnfw		\
 	dependency	"$S/contrib/dev/iwn/iwlwifi-1000-39.31.5.1.fw.uu" \
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"iwn1000.fw"
 iwn2000fw.c			optional iwn2000fw | iwnfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk iwn2000.fw:iwn2000fw -miwn2000fw -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"iwn2000fw.c"
 iwn2000fw.fwo			optional iwn2000fw | iwnfw		\
 	dependency	"iwn2000.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"iwn2000fw.fwo"
 iwn2000.fw			optional iwn2000fw | iwnfw		\
 	dependency	"$S/contrib/dev/iwn/iwlwifi-2000-18.168.6.1.fw.uu" \
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"iwn2000.fw"
 iwn2030fw.c			optional iwn2030fw | iwnfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk iwn2030.fw:iwn2030fw -miwn2030fw -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"iwn2030fw.c"
 iwn2030fw.fwo			optional iwn2030fw | iwnfw		\
 	dependency	"iwn2030.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"iwn2030fw.fwo"
 iwn2030.fw			optional iwn2030fw | iwnfw		\
 	dependency	"$S/contrib/dev/iwn/iwnwifi-2030-18.168.6.1.fw.uu" \
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"iwn2030.fw"
 iwn4965fw.c			optional iwn4965fw | iwnfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk iwn4965.fw:iwn4965fw -miwn4965fw -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"iwn4965fw.c"
 iwn4965fw.fwo			optional iwn4965fw | iwnfw		\
 	dependency	"iwn4965.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"iwn4965fw.fwo"
 iwn4965.fw			optional iwn4965fw | iwnfw		\
 	dependency	"$S/contrib/dev/iwn/iwlwifi-4965-228.61.2.24.fw.uu" \
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"iwn4965.fw"
 iwn5000fw.c			optional iwn5000fw | iwnfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk iwn5000.fw:iwn5000fw -miwn5000fw -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"iwn5000fw.c"
 iwn5000fw.fwo		optional iwn5000fw | iwnfw			\
 	dependency	"iwn5000.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"iwn5000fw.fwo"
 iwn5000.fw			optional iwn5000fw | iwnfw		\
 	dependency	"$S/contrib/dev/iwn/iwlwifi-5000-8.83.5.1.fw.uu" \
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"iwn5000.fw"
 iwn5150fw.c			optional iwn5150fw | iwnfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk iwn5150.fw:iwn5150fw -miwn5150fw -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"iwn5150fw.c"
 iwn5150fw.fwo			optional iwn5150fw | iwnfw		\
 	dependency	"iwn5150.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"iwn5150fw.fwo"
 iwn5150.fw			optional iwn5150fw | iwnfw		\
 	dependency	"$S/contrib/dev/iwn/iwlwifi-5150-8.24.2.2.fw.uu"\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"iwn5150.fw"
 iwn6000fw.c			optional iwn6000fw | iwnfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk iwn6000.fw:iwn6000fw -miwn6000fw -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"iwn6000fw.c"
 iwn6000fw.fwo			optional iwn6000fw | iwnfw		\
 	dependency	"iwn6000.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"iwn6000fw.fwo"
 iwn6000.fw			optional iwn6000fw | iwnfw		\
 	dependency	"$S/contrib/dev/iwn/iwlwifi-6000-9.221.4.1.fw.uu" \
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"iwn6000.fw"
 iwn6000g2afw.c			optional iwn6000g2afw | iwnfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk iwn6000g2a.fw:iwn6000g2afw -miwn6000g2afw -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"iwn6000g2afw.c"
 iwn6000g2afw.fwo		optional iwn6000g2afw | iwnfw		\
 	dependency	"iwn6000g2a.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"iwn6000g2afw.fwo"
 iwn6000g2a.fw			optional iwn6000g2afw | iwnfw		\
 	dependency	"$S/contrib/dev/iwn/iwlwifi-6000g2a-17.168.5.2.fw.uu" \
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"iwn6000g2a.fw"
 iwn6000g2bfw.c			optional iwn6000g2bfw | iwnfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk iwn6000g2b.fw:iwn6000g2bfw -miwn6000g2bfw -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"iwn6000g2bfw.c"
 iwn6000g2bfw.fwo		optional iwn6000g2bfw | iwnfw		\
 	dependency	"iwn6000g2b.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"iwn6000g2bfw.fwo"
 iwn6000g2b.fw			optional iwn6000g2bfw | iwnfw		\
 	dependency	"$S/contrib/dev/iwn/iwlwifi-6000g2b-17.168.5.2.fw.uu" \
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"iwn6000g2b.fw"
 iwn6050fw.c			optional iwn6050fw | iwnfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk iwn6050.fw:iwn6050fw -miwn6050fw -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"iwn6050fw.c"
 iwn6050fw.fwo			optional iwn6050fw | iwnfw		\
 	dependency	"iwn6050.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"iwn6050fw.fwo"
 iwn6050.fw			optional iwn6050fw | iwnfw		\
 	dependency	"$S/contrib/dev/iwn/iwlwifi-6050-41.28.5.1.fw.uu" \
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"iwn6050.fw"
 dev/ixgb/if_ixgb.c		optional ixgb
 dev/ixgb/ixgb_ee.c		optional ixgb
 dev/ixgb/ixgb_hw.c		optional ixgb
 dev/ixgbe/ixgbe.c		optional ixgbe inet \
 	compile-with "${NORMAL_C} -I$S/dev/ixgbe -DSMP -DIXGBE_FDIR"
 dev/ixgbe/ixv.c			optional ixgbe inet \
 	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
 dev/ixgbe/ixgbe_phy.c		optional ixgbe inet \
 	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
 dev/ixgbe/ixgbe_api.c		optional ixgbe inet \
 	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
 dev/ixgbe/ixgbe_common.c	optional ixgbe inet \
 	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
 dev/ixgbe/ixgbe_mbx.c		optional ixgbe inet \
 	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
 dev/ixgbe/ixgbe_vf.c		optional ixgbe inet \
 	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
 dev/ixgbe/ixgbe_82598.c		optional ixgbe inet \
 	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
 dev/ixgbe/ixgbe_82599.c		optional ixgbe inet \
 	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
 dev/ixgbe/ixgbe_x540.c		optional ixgbe inet \
 	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
 dev/ixgbe/ixgbe_dcb.c		optional ixgbe inet \
 	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
 dev/ixgbe/ixgbe_dcb_82598.c	optional ixgbe inet \
 	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
 dev/ixgbe/ixgbe_dcb_82599.c	optional ixgbe inet \
 	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
 dev/jme/if_jme.c		optional jme pci
 dev/joy/joy.c			optional joy
 dev/joy/joy_isa.c		optional joy isa
 dev/joy/joy_pccard.c		optional joy pccard
 dev/kbdmux/kbdmux.c		optional kbdmux
 dev/ksyms/ksyms.c		optional ksyms
 dev/le/am7990.c			optional le
 dev/le/am79900.c		optional le
 dev/le/if_le_pci.c		optional le pci
 dev/le/lance.c			optional le
 dev/led/led.c			standard
 dev/lge/if_lge.c		optional lge
 dev/lmc/if_lmc.c		optional lmc
 dev/malo/if_malo.c		optional malo
 dev/malo/if_malohal.c		optional malo
 dev/malo/if_malo_pci.c		optional malo pci
 dev/mc146818/mc146818.c		optional mc146818
 dev/mca/mca_bus.c		optional mca
 dev/mcd/mcd.c			optional mcd isa nowerror
 dev/mcd/mcd_isa.c		optional mcd isa nowerror
 dev/md/md.c			optional md
 dev/mem/memdev.c		optional mem
 dev/mem/memutil.c		optional mem
 dev/mfi/mfi.c			optional mfi
 dev/mfi/mfi_debug.c		optional mfi
 dev/mfi/mfi_pci.c		optional mfi pci
 dev/mfi/mfi_disk.c		optional mfi
 dev/mfi/mfi_syspd.c		optional mfi
 dev/mfi/mfi_tbolt.c		optional mfi
 dev/mfi/mfi_linux.c		optional mfi compat_linux
 dev/mfi/mfi_cam.c		optional mfip scbus
 dev/mii/acphy.c			optional miibus | acphy
 dev/mii/amphy.c			optional miibus | amphy
 dev/mii/atphy.c			optional miibus | atphy
 dev/mii/axphy.c			optional miibus | axphy
 dev/mii/bmtphy.c		optional miibus | bmtphy
 dev/mii/brgphy.c		optional miibus | brgphy
 dev/mii/ciphy.c			optional miibus | ciphy
 dev/mii/e1000phy.c		optional miibus | e1000phy
 dev/mii/gentbi.c		optional miibus | gentbi
 dev/mii/icsphy.c		optional miibus | icsphy
 dev/mii/ip1000phy.c		optional miibus | ip1000phy
 dev/mii/jmphy.c			optional miibus | jmphy
 dev/mii/lxtphy.c		optional miibus | lxtphy
 dev/mii/mii.c			optional miibus | mii
 dev/mii/mii_bitbang.c		optional miibus | mii_bitbang
 dev/mii/mii_physubr.c		optional miibus | mii
 dev/mii/miibus_if.m		optional miibus | mii
 dev/mii/mlphy.c			optional miibus | mlphy
 dev/mii/nsgphy.c		optional miibus | nsgphy
 dev/mii/nsphy.c			optional miibus | nsphy
 dev/mii/nsphyter.c		optional miibus | nsphyter
 dev/mii/pnaphy.c		optional miibus | pnaphy
 dev/mii/qsphy.c			optional miibus | qsphy
 dev/mii/rdcphy.c		optional miibus | rdcphy
 dev/mii/rgephy.c		optional miibus | rgephy
 dev/mii/rlphy.c			optional miibus | rlphy
 dev/mii/rlswitch.c		optional rlswitch
 dev/mii/smcphy.c		optional miibus | smcphy
 dev/mii/smscphy.c		optional miibus | smscphy
 dev/mii/tdkphy.c		optional miibus | tdkphy
 dev/mii/tlphy.c			optional miibus | tlphy
 dev/mii/truephy.c		optional miibus | truephy
 dev/mii/ukphy.c			optional miibus | mii
 dev/mii/ukphy_subr.c		optional miibus | mii
 dev/mii/xmphy.c			optional miibus | xmphy
 dev/mk48txx/mk48txx.c		optional mk48txx
 dev/mlx/mlx.c			optional mlx
 dev/mlx/mlx_disk.c		optional mlx
 dev/mlx/mlx_pci.c		optional mlx pci
 dev/mly/mly.c			optional mly
 dev/mmc/mmc.c			optional mmc
 dev/mmc/mmcbr_if.m		standard
 dev/mmc/mmcbus_if.m		standard
 dev/mmc/mmcsd.c			optional mmcsd
 dev/mn/if_mn.c			optional mn pci
 dev/mpr/mpr.c			optional mpr
 dev/mpr/mpr_config.c		optional mpr
 # XXX Work around clang warning, until maintainer approves fix.
 dev/mpr/mpr_mapping.c		optional mpr \
 	compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}"
 dev/mpr/mpr_pci.c		optional mpr pci
 dev/mpr/mpr_sas.c		optional mpr \
 	compile-with "${NORMAL_C} ${NO_WUNNEEDED_INTERNAL_DECL}"
 dev/mpr/mpr_sas_lsi.c		optional mpr
 dev/mpr/mpr_table.c		optional mpr
 dev/mpr/mpr_user.c		optional mpr
 dev/mps/mps.c			optional mps
 dev/mps/mps_config.c		optional mps
 # XXX Work around clang warning, until maintainer approves fix.
 dev/mps/mps_mapping.c		optional mps \
 	compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}"
 dev/mps/mps_pci.c		optional mps pci
 dev/mps/mps_sas.c		optional mps \
 	compile-with "${NORMAL_C} ${NO_WUNNEEDED_INTERNAL_DECL}"
 dev/mps/mps_sas_lsi.c		optional mps
 dev/mps/mps_table.c		optional mps
 dev/mps/mps_user.c		optional mps
 dev/mpt/mpt.c			optional mpt
 dev/mpt/mpt_cam.c		optional mpt
 dev/mpt/mpt_debug.c		optional mpt
 dev/mpt/mpt_pci.c		optional mpt pci
 dev/mpt/mpt_raid.c		optional mpt
 dev/mpt/mpt_user.c		optional mpt
 dev/mrsas/mrsas.c		optional mrsas
 dev/mrsas/mrsas_cam.c		optional mrsas
 dev/mrsas/mrsas_ioctl.c		optional mrsas
 dev/mrsas/mrsas_fp.c		optional mrsas
 dev/msk/if_msk.c		optional msk
 dev/mvs/mvs.c			optional mvs
 dev/mvs/mvs_if.m		optional mvs
 dev/mvs/mvs_pci.c		optional mvs pci
 dev/mwl/if_mwl.c		optional mwl
 dev/mwl/if_mwl_pci.c		optional mwl pci
 dev/mwl/mwlhal.c		optional mwl
 mwlfw.c				optional mwlfw				\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk mw88W8363.fw:mw88W8363fw mwlboot.fw:mwlboot -mmwl -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"mwlfw.c"
 mw88W8363.fwo		optional mwlfw					\
 	dependency	"mw88W8363.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"mw88W8363.fwo"
 mw88W8363.fw		optional mwlfw					\
 	dependency	"$S/contrib/dev/mwl/mw88W8363.fw.uu"		\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"mw88W8363.fw"
 mwlboot.fwo		optional mwlfw					\
 	dependency	"mwlboot.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"mwlboot.fwo"
 mwlboot.fw		optional mwlfw					\
 	dependency	"$S/contrib/dev/mwl/mwlboot.fw.uu"		\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"mwlboot.fw"
 dev/mxge/if_mxge.c		optional mxge pci
 dev/mxge/mxge_eth_z8e.c		optional mxge pci
 dev/mxge/mxge_ethp_z8e.c	optional mxge pci
 dev/mxge/mxge_rss_eth_z8e.c	optional mxge pci
 dev/mxge/mxge_rss_ethp_z8e.c	optional mxge pci
 dev/my/if_my.c			optional my
 dev/nand/nand.c			optional nand
 dev/nand/nand_bbt.c		optional nand
 dev/nand/nand_cdev.c		optional nand
 dev/nand/nand_generic.c		optional nand
 dev/nand/nand_geom.c		optional nand
 dev/nand/nand_id.c		optional nand
 dev/nand/nandbus.c		optional nand
 dev/nand/nandbus_if.m		optional nand
 dev/nand/nand_if.m		optional nand
 dev/nand/nandsim.c		optional nandsim nand
 dev/nand/nandsim_chip.c		optional nandsim nand
 dev/nand/nandsim_ctrl.c		optional nandsim nand
 dev/nand/nandsim_log.c		optional nandsim nand
 dev/nand/nandsim_swap.c		optional nandsim nand
 dev/nand/nfc_if.m		optional nand
 dev/ncv/ncr53c500.c		optional ncv
 dev/ncv/ncr53c500_pccard.c	optional ncv pccard
 dev/netmap/netmap.c		optional netmap
 dev/netmap/netmap_freebsd.c	optional netmap
 dev/netmap/netmap_generic.c	optional netmap
 dev/netmap/netmap_mbq.c		optional netmap
 dev/netmap/netmap_mem2.c	optional netmap
 dev/netmap/netmap_monitor.c	optional netmap
 dev/netmap/netmap_offloadings.c	optional netmap
 dev/netmap/netmap_pipe.c	optional netmap
 dev/netmap/netmap_vale.c	optional netmap
 dev/nge/if_nge.c		optional nge
 dev/nxge/if_nxge.c		optional nxge \
 	compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}"
 dev/nxge/xgehal/xgehal-device.c	optional nxge \
 	compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}"
 dev/nxge/xgehal/xgehal-mm.c	optional nxge
 dev/nxge/xgehal/xge-queue.c	optional nxge
 dev/nxge/xgehal/xgehal-driver.c	optional nxge \
 	compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}"
 dev/nxge/xgehal/xgehal-ring.c	optional nxge \
 	compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}"
 dev/nxge/xgehal/xgehal-channel.c	optional nxge \
 	compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}"
 dev/nxge/xgehal/xgehal-fifo.c	optional nxge \
 	compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}"
 dev/nxge/xgehal/xgehal-stats.c	optional nxge \
 	compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}"
 dev/nxge/xgehal/xgehal-config.c	optional nxge
 dev/nxge/xgehal/xgehal-mgmt.c	optional nxge \
 	compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}"
 dev/nmdm/nmdm.c			optional nmdm
 dev/nsp/nsp.c			optional nsp
 dev/nsp/nsp_pccard.c		optional nsp pccard
 dev/null/null.c			standard
 dev/oce/oce_hw.c		optional oce pci
 dev/oce/oce_if.c		optional oce pci
 dev/oce/oce_mbox.c		optional oce pci
 dev/oce/oce_queue.c		optional oce pci
 dev/oce/oce_sysctl.c		optional oce pci
 dev/oce/oce_util.c		optional oce pci
 dev/ofw/ofw_bus_if.m		optional fdt
 dev/ofw/ofw_bus_subr.c		optional fdt
 dev/ofw/ofw_fdt.c		optional fdt
 dev/ofw/ofw_if.m		optional fdt
 dev/ofw/ofw_iicbus.c		optional fdt iicbus
 dev/ofw/ofwbus.c		optional fdt
 dev/ofw/openfirm.c		optional fdt
 dev/ofw/openfirmio.c		optional fdt
 dev/patm/if_patm.c		optional patm pci
 dev/patm/if_patm_attach.c	optional patm pci
 dev/patm/if_patm_intr.c		optional patm pci
 dev/patm/if_patm_ioctl.c	optional patm pci
 dev/patm/if_patm_rtables.c	optional patm pci
 dev/patm/if_patm_rx.c		optional patm pci
 dev/patm/if_patm_tx.c		optional patm pci
 dev/pbio/pbio.c			optional pbio isa
 dev/pccard/card_if.m		standard
 dev/pccard/pccard.c		optional pccard
 dev/pccard/pccard_cis.c		optional pccard
 dev/pccard/pccard_cis_quirks.c	optional pccard
 dev/pccard/pccard_device.c	optional pccard
 dev/pccard/power_if.m		standard
 dev/pccbb/pccbb.c		optional cbb
 dev/pccbb/pccbb_isa.c		optional cbb isa
 dev/pccbb/pccbb_pci.c		optional cbb pci
 dev/pcf/pcf.c			optional pcf
 dev/pci/eisa_pci.c		optional pci eisa
 dev/pci/fixup_pci.c		optional pci
 dev/pci/hostb_pci.c		optional pci
 dev/pci/ignore_pci.c		optional pci
 dev/pci/isa_pci.c		optional pci isa
 dev/pci/pci.c			optional pci
 dev/pci/pci_if.m		standard
 dev/pci/pci_pci.c		optional pci
 dev/pci/pci_subr.c		optional pci
 dev/pci/pci_user.c		optional pci
 dev/pci/pcib_if.m		standard
 dev/pci/vga_pci.c		optional pci
 dev/pcn/if_pcn.c		optional pcn pci
 dev/pdq/if_fea.c		optional fea eisa
 dev/pdq/if_fpa.c		optional fpa pci
 dev/pdq/pdq.c			optional nowerror fea eisa | fpa pci
 dev/pdq/pdq_ifsubr.c		optional nowerror fea eisa | fpa pci
 dev/ppbus/if_plip.c		optional plip
 dev/ppbus/immio.c		optional vpo
 dev/ppbus/lpbb.c		optional lpbb
 dev/ppbus/lpt.c			optional lpt
 dev/ppbus/pcfclock.c		optional pcfclock
 dev/ppbus/ppb_1284.c		optional ppbus
 dev/ppbus/ppb_base.c		optional ppbus
 dev/ppbus/ppb_msq.c		optional ppbus
 dev/ppbus/ppbconf.c		optional ppbus
 dev/ppbus/ppbus_if.m		optional ppbus
 dev/ppbus/ppi.c			optional ppi
 dev/ppbus/pps.c			optional pps
 dev/ppbus/vpo.c			optional vpo
 dev/ppbus/vpoio.c		optional vpo
 dev/ppc/ppc.c			optional ppc
 dev/ppc/ppc_acpi.c		optional ppc acpi
 dev/ppc/ppc_isa.c		optional ppc isa
 dev/ppc/ppc_pci.c		optional ppc pci
 dev/ppc/ppc_puc.c		optional ppc puc
 dev/pst/pst-iop.c		optional pst
 dev/pst/pst-pci.c		optional pst pci
 dev/pst/pst-raid.c		optional pst
 dev/pty/pty.c			optional pty
 dev/puc/puc.c			optional puc
 dev/puc/puc_cfg.c		optional puc
 dev/puc/puc_pccard.c		optional puc pccard
 dev/puc/puc_pci.c		optional puc pci
 dev/puc/pucdata.c		optional puc pci
 dev/quicc/quicc_core.c		optional quicc
 dev/ral/rt2560.c		optional ral
 dev/ral/rt2661.c		optional ral
 dev/ral/rt2860.c		optional ral
 dev/ral/if_ral_pci.c		optional ral pci
 rt2561fw.c			optional rt2561fw | ralfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk rt2561.fw:rt2561fw -mrt2561 -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"rt2561fw.c"
 rt2561fw.fwo			optional rt2561fw | ralfw		\
 	dependency	"rt2561.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"rt2561fw.fwo"
 rt2561.fw			optional rt2561fw | ralfw		\
 	dependency	"$S/contrib/dev/ral/rt2561.fw.uu"		\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"rt2561.fw"
 rt2561sfw.c			optional rt2561sfw | ralfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk rt2561s.fw:rt2561sfw -mrt2561s -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"rt2561sfw.c"
 rt2561sfw.fwo			optional rt2561sfw | ralfw		\
 	dependency	"rt2561s.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"rt2561sfw.fwo"
 rt2561s.fw			optional rt2561sfw | ralfw		\
 	dependency	"$S/contrib/dev/ral/rt2561s.fw.uu"		\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"rt2561s.fw"
 rt2661fw.c			optional rt2661fw | ralfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk rt2661.fw:rt2661fw -mrt2661 -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"rt2661fw.c"
 rt2661fw.fwo			optional rt2661fw | ralfw		\
 	dependency	"rt2661.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"rt2661fw.fwo"
 rt2661.fw			optional rt2661fw | ralfw		\
 	dependency	"$S/contrib/dev/ral/rt2661.fw.uu"		\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"rt2661.fw"
 rt2860fw.c			optional rt2860fw | ralfw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk rt2860.fw:rt2860fw -mrt2860 -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"rt2860fw.c"
 rt2860fw.fwo			optional rt2860fw | ralfw		\
 	dependency	"rt2860.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"rt2860fw.fwo"
 rt2860.fw			optional rt2860fw | ralfw		\
 	dependency	"$S/contrib/dev/ral/rt2860.fw.uu"		\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"rt2860.fw"
 dev/random/harvest.c		standard
 dev/random/dummy_rng.c		standard
 dev/random/random_adaptors.c	standard
 dev/random/live_entropy_sources.c	optional random
 dev/random/random_harvestq.c	optional random
 dev/random/randomdev.c		optional random
 dev/random/randomdev_soft.c	optional random
 dev/random/yarrow.c		optional random
 dev/random/hash.c		optional random
 dev/random/rwfile.c		optional random
 dev/rc/rc.c			optional rc
 dev/re/if_re.c			optional re
 dev/rndtest/rndtest.c		optional rndtest
 dev/rp/rp.c			optional rp
 dev/rp/rp_isa.c			optional rp isa
 dev/rp/rp_pci.c			optional rp pci
 dev/safe/safe.c			optional safe
 dev/scc/scc_if.m		optional scc
 dev/scc/scc_bfe_ebus.c		optional scc ebus
 dev/scc/scc_bfe_quicc.c		optional scc quicc
 dev/scc/scc_bfe_sbus.c		optional scc fhc | scc sbus
 dev/scc/scc_core.c		optional scc
 dev/scc/scc_dev_quicc.c		optional scc quicc
 dev/scc/scc_dev_sab82532.c	optional scc
 dev/scc/scc_dev_z8530.c		optional scc
 dev/scd/scd.c			optional scd isa
 dev/scd/scd_isa.c		optional scd isa
 dev/sdhci/sdhci.c		optional sdhci
 dev/sdhci/sdhci_if.m		optional sdhci
 dev/sdhci/sdhci_pci.c		optional sdhci pci
 dev/sf/if_sf.c			optional sf pci
 dev/sge/if_sge.c		optional sge pci
 dev/si/si.c			optional si
 dev/si/si2_z280.c		optional si
 dev/si/si3_t225.c		optional si
 dev/si/si_eisa.c		optional si eisa
 dev/si/si_isa.c			optional si isa
 dev/si/si_pci.c			optional si pci
 dev/siba/siba.c			optional siba
 dev/siba/siba_bwn.c		optional siba_bwn pci
 dev/siba/siba_cc.c		optional siba
 dev/siba/siba_core.c		optional siba | siba_bwn pci
 dev/siba/siba_pcib.c		optional siba pci
 dev/siis/siis.c			optional siis pci
 dev/sis/if_sis.c		optional sis pci
 dev/sk/if_sk.c			optional sk pci
 dev/smbus/smb.c			optional smb
 dev/smbus/smbconf.c		optional smbus
 dev/smbus/smbus.c		optional smbus
 dev/smbus/smbus_if.m		optional smbus
 dev/smc/if_smc.c		optional smc
 dev/sn/if_sn.c			optional sn
 dev/sn/if_sn_isa.c		optional sn isa
 dev/sn/if_sn_pccard.c		optional sn pccard
 dev/snp/snp.c			optional snp
 dev/sound/clone.c		optional sound
 dev/sound/unit.c		optional sound
 dev/sound/isa/ad1816.c		optional snd_ad1816 isa
 dev/sound/isa/ess.c		optional snd_ess isa
 dev/sound/isa/gusc.c		optional snd_gusc isa
 dev/sound/isa/mss.c		optional snd_mss isa
 dev/sound/isa/sb16.c		optional snd_sb16 isa
 dev/sound/isa/sb8.c		optional snd_sb8 isa
 dev/sound/isa/sbc.c		optional snd_sbc isa
 dev/sound/isa/sndbuf_dma.c	optional sound isa
 dev/sound/pci/als4000.c		optional snd_als4000 pci
 dev/sound/pci/atiixp.c		optional snd_atiixp pci
 dev/sound/pci/cmi.c		optional snd_cmi pci
 dev/sound/pci/cs4281.c		optional snd_cs4281 pci
 dev/sound/pci/csa.c		optional snd_csa pci
 dev/sound/pci/csapcm.c		optional snd_csa pci
 dev/sound/pci/ds1.c		optional snd_ds1 pci
 dev/sound/pci/emu10k1.c		optional snd_emu10k1 pci
 dev/sound/pci/emu10kx.c		optional snd_emu10kx pci
 dev/sound/pci/emu10kx-pcm.c	optional snd_emu10kx pci
 dev/sound/pci/emu10kx-midi.c	optional snd_emu10kx pci
 dev/sound/pci/envy24.c		optional snd_envy24 pci
 dev/sound/pci/envy24ht.c	optional snd_envy24ht pci
 dev/sound/pci/es137x.c		optional snd_es137x pci
 dev/sound/pci/fm801.c		optional snd_fm801 pci
 dev/sound/pci/ich.c		optional snd_ich pci
 dev/sound/pci/maestro.c		optional snd_maestro pci
 dev/sound/pci/maestro3.c	optional snd_maestro3 pci
 dev/sound/pci/neomagic.c	optional snd_neomagic pci
 dev/sound/pci/solo.c		optional snd_solo pci
 dev/sound/pci/spicds.c		optional snd_spicds pci
 dev/sound/pci/t4dwave.c		optional snd_t4dwave pci
 dev/sound/pci/via8233.c		optional snd_via8233 pci
 dev/sound/pci/via82c686.c	optional snd_via82c686 pci
 dev/sound/pci/vibes.c		optional snd_vibes pci
 dev/sound/pci/hda/hdaa.c	optional snd_hda pci
 dev/sound/pci/hda/hdaa_patches.c	optional snd_hda pci
 dev/sound/pci/hda/hdac.c	optional snd_hda pci
 dev/sound/pci/hda/hdac_if.m	optional snd_hda pci
 dev/sound/pci/hda/hdacc.c	optional snd_hda pci
 dev/sound/pci/hdspe.c		optional snd_hdspe pci
 dev/sound/pci/hdspe-pcm.c	optional snd_hdspe pci
 dev/sound/pcm/ac97.c		optional sound
 dev/sound/pcm/ac97_if.m		optional sound
 dev/sound/pcm/ac97_patch.c	optional sound
 dev/sound/pcm/buffer.c		optional sound	\
 	dependency	"snd_fxdiv_gen.h"
 dev/sound/pcm/channel.c		optional sound
 dev/sound/pcm/channel_if.m	optional sound
 dev/sound/pcm/dsp.c		optional sound
 dev/sound/pcm/feeder.c		optional sound
 dev/sound/pcm/feeder_chain.c	optional sound
 dev/sound/pcm/feeder_eq.c	optional sound	\
 	dependency	"feeder_eq_gen.h"	\
 	dependency	"snd_fxdiv_gen.h"
 dev/sound/pcm/feeder_if.m	optional sound
 dev/sound/pcm/feeder_format.c	optional sound  \
 	dependency	"snd_fxdiv_gen.h"
 dev/sound/pcm/feeder_matrix.c	optional sound  \
 	dependency	"snd_fxdiv_gen.h"
 dev/sound/pcm/feeder_mixer.c	optional sound  \
 	dependency	"snd_fxdiv_gen.h"
 dev/sound/pcm/feeder_rate.c	optional sound	\
 	dependency	"feeder_rate_gen.h"	\
 	dependency	"snd_fxdiv_gen.h"
 dev/sound/pcm/feeder_volume.c	optional sound  \
 	dependency	"snd_fxdiv_gen.h"
 dev/sound/pcm/mixer.c		optional sound
 dev/sound/pcm/mixer_if.m	optional sound
 dev/sound/pcm/sndstat.c		optional sound
 dev/sound/pcm/sound.c		optional sound
 dev/sound/pcm/vchan.c		optional sound
 dev/sound/usb/uaudio.c		optional snd_uaudio usb
 dev/sound/usb/uaudio_pcm.c	optional snd_uaudio usb
 dev/sound/midi/midi.c		optional sound
 dev/sound/midi/mpu401.c		optional sound
 dev/sound/midi/mpu_if.m		optional sound
 dev/sound/midi/mpufoi_if.m	optional sound
 dev/sound/midi/sequencer.c	optional sound
 dev/sound/midi/synth_if.m	optional sound
 dev/spibus/ofw_spibus.c		optional fdt spibus
 dev/spibus/spibus.c		optional spibus				\
 	dependency	"spibus_if.h"
 dev/spibus/spibus_if.m		optional spibus
 dev/ste/if_ste.c		optional ste pci
 dev/stg/tmc18c30.c		optional stg
 dev/stg/tmc18c30_isa.c		optional stg isa
 dev/stg/tmc18c30_pccard.c	optional stg pccard
 dev/stg/tmc18c30_pci.c		optional stg pci
 dev/stg/tmc18c30_subr.c		optional stg
 dev/stge/if_stge.c		optional stge
 dev/streams/streams.c		optional streams
 dev/sym/sym_hipd.c		optional sym				\
 	dependency	"$S/dev/sym/sym_{conf,defs}.h"
 dev/syscons/blank/blank_saver.c	optional blank_saver
 dev/syscons/daemon/daemon_saver.c optional daemon_saver
 dev/syscons/dragon/dragon_saver.c optional dragon_saver
 dev/syscons/fade/fade_saver.c	optional fade_saver
 dev/syscons/fire/fire_saver.c	optional fire_saver
 dev/syscons/green/green_saver.c	optional green_saver
 dev/syscons/logo/logo.c		optional logo_saver
 dev/syscons/logo/logo_saver.c	optional logo_saver
 dev/syscons/rain/rain_saver.c	optional rain_saver
 dev/syscons/schistory.c		optional sc
 dev/syscons/scmouse.c		optional sc
 dev/syscons/scterm.c		optional sc
 dev/syscons/scvidctl.c		optional sc
 dev/syscons/snake/snake_saver.c	optional snake_saver
 dev/syscons/star/star_saver.c	optional star_saver
 dev/syscons/syscons.c		optional sc
 dev/syscons/sysmouse.c		optional sc
 dev/syscons/warp/warp_saver.c	optional warp_saver
 dev/tdfx/tdfx_linux.c		optional tdfx_linux tdfx compat_linux
 dev/tdfx/tdfx_pci.c		optional tdfx pci
 dev/ti/if_ti.c			optional ti pci
 dev/tl/if_tl.c			optional tl pci
 dev/trm/trm.c			optional trm
 dev/twa/tw_cl_init.c		optional twa \
 	compile-with "${NORMAL_C} -I$S/dev/twa"
 dev/twa/tw_cl_intr.c		optional twa \
 	compile-with "${NORMAL_C} -I$S/dev/twa"
 dev/twa/tw_cl_io.c		optional twa \
 	compile-with "${NORMAL_C} -I$S/dev/twa"
 dev/twa/tw_cl_misc.c		optional twa \
 	compile-with "${NORMAL_C} -I$S/dev/twa"
 dev/twa/tw_osl_cam.c		optional twa \
 	compile-with "${NORMAL_C} -I$S/dev/twa"
 dev/twa/tw_osl_freebsd.c	optional twa \
 	compile-with "${NORMAL_C} -I$S/dev/twa"
 dev/twe/twe.c			optional twe
 dev/twe/twe_freebsd.c		optional twe
 dev/tws/tws.c			optional tws
 dev/tws/tws_cam.c		optional tws
 dev/tws/tws_hdm.c		optional tws
 dev/tws/tws_services.c		optional tws
 dev/tws/tws_user.c		optional tws
 dev/tx/if_tx.c			optional tx
 dev/txp/if_txp.c		optional txp
 dev/uart/uart_bus_acpi.c	optional uart acpi
 #dev/uart/uart_bus_cbus.c	optional uart cbus
 dev/uart/uart_bus_ebus.c	optional uart ebus
 dev/uart/uart_bus_fdt.c		optional uart fdt
 dev/uart/uart_bus_isa.c		optional uart isa
 dev/uart/uart_bus_pccard.c	optional uart pccard
 dev/uart/uart_bus_pci.c		optional uart pci
 dev/uart/uart_bus_puc.c		optional uart puc
 dev/uart/uart_bus_scc.c		optional uart scc
 dev/uart/uart_core.c		optional uart
 dev/uart/uart_dbg.c		optional uart gdb
 dev/uart/uart_dev_ns8250.c	optional uart uart_ns8250
 dev/uart/uart_dev_pl011.c	optional uart pl011
 dev/uart/uart_dev_quicc.c	optional uart quicc
 dev/uart/uart_dev_sab82532.c	optional uart uart_sab82532
 dev/uart/uart_dev_sab82532.c	optional uart scc
 dev/uart/uart_dev_z8530.c	optional uart uart_z8530
 dev/uart/uart_dev_z8530.c	optional uart scc
 dev/uart/uart_if.m		optional uart
 dev/uart/uart_subr.c		optional uart
 dev/uart/uart_tty.c		optional uart
 dev/ubsec/ubsec.c		optional ubsec
 #
 # USB controller drivers
 #
 dev/usb/controller/at91dci.c		optional at91dci
 dev/usb/controller/at91dci_atmelarm.c	optional at91dci at91rm9200
 dev/usb/controller/musb_otg.c		optional musb
 dev/usb/controller/musb_otg_atmelarm.c	optional musb at91rm9200
 dev/usb/controller/dwc_otg.c		optional dwcotg
 dev/usb/controller/ehci.c		optional ehci
 dev/usb/controller/ehci_pci.c		optional ehci pci
 dev/usb/controller/ohci.c		optional ohci
 dev/usb/controller/ohci_atmelarm.c	optional ohci at91rm9200
 dev/usb/controller/ohci_pci.c		optional ohci pci
 dev/usb/controller/uhci.c		optional uhci
 dev/usb/controller/uhci_pci.c		optional uhci pci
 dev/usb/controller/xhci.c		optional xhci
 dev/usb/controller/xhci_pci.c		optional xhci pci
 dev/usb/controller/uss820dci.c		optional uss820dci
 dev/usb/controller/uss820dci_atmelarm.c	optional uss820dci at91rm9200
 dev/usb/controller/usb_controller.c	optional usb
 #
 # USB storage drivers
 #
 dev/usb/storage/umass.c		optional umass
 dev/usb/storage/urio.c		optional urio
 dev/usb/storage/ustorage_fs.c	optional usfs
 #
 # USB core
 #
 dev/usb/usb_busdma.c		optional usb
 dev/usb/usb_compat_linux.c	optional usb
 dev/usb/usb_core.c		optional usb
 dev/usb/usb_debug.c		optional usb
 dev/usb/usb_dev.c		optional usb
 dev/usb/usb_device.c		optional usb
 dev/usb/usb_dynamic.c		optional usb
 dev/usb/usb_error.c		optional usb
 dev/usb/usb_generic.c		optional usb
 dev/usb/usb_handle_request.c	optional usb
 dev/usb/usb_hid.c		optional usb
 dev/usb/usb_hub.c		optional usb
 dev/usb/usb_if.m		optional usb
 dev/usb/usb_lookup.c		optional usb
 dev/usb/usb_mbuf.c		optional usb
 dev/usb/usb_msctest.c		optional usb
 dev/usb/usb_parse.c		optional usb
 dev/usb/usb_pf.c		optional usb
 dev/usb/usb_process.c		optional usb
 dev/usb/usb_request.c		optional usb
 dev/usb/usb_transfer.c		optional usb
 dev/usb/usb_util.c		optional usb
 #
 # USB network drivers
 #
 dev/usb/net/if_aue.c		optional aue
 dev/usb/net/if_axe.c		optional axe
 dev/usb/net/if_axge.c		optional axge
 dev/usb/net/if_cdce.c		optional cdce
 dev/usb/net/if_cue.c		optional cue
 dev/usb/net/if_ipheth.c		optional ipheth
 dev/usb/net/if_kue.c		optional kue
 dev/usb/net/if_mos.c		optional mos
 dev/usb/net/if_rue.c		optional rue
 dev/usb/net/if_smsc.c		optional smsc
 dev/usb/net/if_udav.c		optional udav
 dev/usb/net/if_usie.c		optional usie
 dev/usb/net/if_urndis.c		optional urndis
 dev/usb/net/ruephy.c		optional rue
 dev/usb/net/usb_ethernet.c	optional aue | axe | axge | cdce | cue | kue | \
 					 mos | rue | smsc | udav | ipheth | \
 					 urndis
 dev/usb/net/uhso.c		optional uhso
 #
 # USB WLAN drivers
 #
 dev/usb/wlan/if_rsu.c		optional rsu
 rsu-rtl8712fw.c			optional rsu-rtl8712fw | rsufw		\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk rsu-rtl8712fw.fw:rsu-rtl8712fw:120 -mrsu-rtl8712fw -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"rsu-rtl8712fw.c"
 rsu-rtl8712fw.fwo		optional rsu-rtl8712fw | rsufw		\
 	dependency	"rsu-rtl8712fw.fw"				\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"rsu-rtl8712fw.fwo"
 rsu-rtl8712fw.fw		optional rsu-rtl8712.fw | rsufw		\
 	dependency	"$S/contrib/dev/rsu/rsu-rtl8712fw.fw.uu"	\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"rsu-rtl8712fw.fw"
 dev/usb/wlan/if_rum.c		optional rum
 dev/usb/wlan/if_run.c		optional run
 runfw.c				optional runfw							\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk run.fw:runfw -mrunfw -c${.TARGET}"	\
 	no-implicit-rule before-depend local							\
 	clean		"runfw.c"
 runfw.fwo			optional runfw							\
 	dependency	"run.fw"								\
 	compile-with	"${NORMAL_FWO}"								\
 	no-implicit-rule									\
 	clean		"runfw.fwo"
 run.fw				optional runfw							\
 	dependency	"$S/contrib/dev/run/rt2870.fw.uu"					\
 	compile-with	"${NORMAL_FW}"								\
 	no-obj no-implicit-rule									\
 	clean		"run.fw"
 dev/usb/wlan/if_uath.c		optional uath
 dev/usb/wlan/if_upgt.c		optional upgt
 dev/usb/wlan/if_ural.c		optional ural
 dev/usb/wlan/if_urtw.c		optional urtw
 dev/usb/wlan/if_urtwn.c		optional urtwn
 urtwn-rtl8188eufw.c		optional urtwn-rtl8188eufw | urtwnfw	\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk urtwn-rtl8188eufw.fw:urtwn-rtl8188eufw:111 -murtwn-rtl8188eufw -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"urtwn-rtl8188eufw.c"
 urtwn-rtl8188eufw.fwo		optional urtwn-rtl8188eufw | urtwnfw	\
 	dependency	"urtwn-rtl8188eufw.fw"				\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"urtwn-rtl8188eufw.fwo"
 urtwn-rtl8188eufw.fw		optional urtwn-rtl8188eufw | urtwnfw	\
 	dependency	"$S/contrib/dev/urtwn/urtwn-rtl8188eufw.fw.uu"	\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"urtwn-rtl8188eufw.fw"
 urtwn-rtl8192cfwT.c		optional urtwn-rtl8192cfwT | urtwnfw	\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk urtwn-rtl8192cfwT.fw:urtwn-rtl8192cfwT:111 -murtwn-rtl8192cfwT -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"urtwn-rtl8192cfwT.c"
 urtwn-rtl8192cfwT.fwo		optional urtwn-rtl8192cfwT | urtwnfw	\
 	dependency	"urtwn-rtl8192cfwT.fw"				\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"urtwn-rtl8192cfwT.fwo"
 urtwn-rtl8192cfwT.fw		optional urtwn-rtl8192cfwT | urtwnfw	\
 	dependency	"$S/contrib/dev/urtwn/urtwn-rtl8192cfwT.fw.uu"	\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"urtwn-rtl8192cfwT.fw"
 urtwn-rtl8192cfwU.c		optional urtwn-rtl8192cfwU | urtwnfw	\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk urtwn-rtl8192cfwU.fw:urtwn-rtl8192cfwU:111 -murtwn-rtl8192cfwU -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"urtwn-rtl8192cfwU.c"
 urtwn-rtl8192cfwU.fwo		optional urtwn-rtl8192cfwU | urtwnfw	\
 	dependency	"urtwn-rtl8192cfwU.fw"				\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"urtwn-rtl8192cfwU.fwo"
 urtwn-rtl8192cfwU.fw		optional urtwn-rtl8192cfwU | urtwnfw	\
 	dependency	"$S/contrib/dev/urtwn/urtwn-rtl8192cfwU.fw.uu"	\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"urtwn-rtl8192cfwU.fw"
 
 dev/usb/wlan/if_zyd.c		optional zyd
 #
 # USB serial and parallel port drivers
 #
 dev/usb/serial/u3g.c		optional u3g
 dev/usb/serial/uark.c		optional uark
 dev/usb/serial/ubsa.c		optional ubsa
 dev/usb/serial/ubser.c		optional ubser
 dev/usb/serial/uchcom.c		optional uchcom
 dev/usb/serial/ucycom.c		optional ucycom
 dev/usb/serial/ufoma.c		optional ufoma
 dev/usb/serial/uftdi.c		optional uftdi
 dev/usb/serial/ugensa.c		optional ugensa
 dev/usb/serial/uipaq.c		optional uipaq
 dev/usb/serial/ulpt.c		optional ulpt
 dev/usb/serial/umcs.c		optional umcs
 dev/usb/serial/umct.c		optional umct
 dev/usb/serial/umodem.c		optional umodem
 dev/usb/serial/umoscom.c	optional umoscom
 dev/usb/serial/uplcom.c		optional uplcom
 dev/usb/serial/uslcom.c		optional uslcom
 dev/usb/serial/uvisor.c		optional uvisor
 dev/usb/serial/uvscom.c		optional uvscom
 dev/usb/serial/usb_serial.c 	optional ucom | u3g | uark | ubsa | ubser | \
 					 uchcom | ucycom | ufoma | uftdi | \
 					 ugensa | uipaq | umcs | umct | \
 					 umodem | umoscom | uplcom | usie | \
 					 uslcom | uvisor | uvscom
 #
 # USB misc drivers
 #
 dev/usb/misc/ufm.c		optional ufm
 dev/usb/misc/udbp.c		optional udbp
 #
 # USB input drivers
 #
 dev/usb/input/atp.c		optional atp
 dev/usb/input/uep.c		optional uep
 dev/usb/input/uhid.c		optional uhid
 dev/usb/input/ukbd.c		optional ukbd
 dev/usb/input/ums.c		optional ums
 dev/usb/input/wsp.c		optional wsp
 #
 # USB quirks
 #
 dev/usb/quirk/usb_quirk.c	optional usb
 #
 # USB templates
 #
 dev/usb/template/usb_template.c		optional usb_template
 dev/usb/template/usb_template_audio.c	optional usb_template
 dev/usb/template/usb_template_cdce.c	optional usb_template
 dev/usb/template/usb_template_kbd.c	optional usb_template
 dev/usb/template/usb_template_modem.c	optional usb_template
 dev/usb/template/usb_template_mouse.c	optional usb_template
 dev/usb/template/usb_template_msc.c	optional usb_template
 dev/usb/template/usb_template_mtp.c	optional usb_template
 dev/usb/template/usb_template_phone.c	optional usb_template
 #
 # USB END
 #
 dev/utopia/idtphy.c		optional utopia
 dev/utopia/suni.c		optional utopia
 dev/utopia/utopia.c		optional utopia
 dev/vge/if_vge.c		optional vge
 
 dev/vkbd/vkbd.c			optional vkbd
 dev/vr/if_vr.c			optional vr pci
 dev/vt/colors/vt_termcolors.c	optional vt
 dev/vt/font/vt_font_default.c	optional vt
 dev/vt/font/vt_mouse_cursor.c	optional vt
 dev/vt/hw/efifb/efifb.c		optional vt_efifb
 dev/vt/hw/fb/vt_fb.c		optional vt
 dev/vt/hw/vga/vt_vga.c		optional vt vt_vga
 dev/vt/logo/logo_freebsd.c	optional vt splash
 dev/vt/vt_buf.c			optional vt
 dev/vt/vt_consolectl.c		optional vt
 dev/vt/vt_core.c		optional vt
 dev/vt/vt_font.c		optional vt
 dev/vt/vt_sysmouse.c		optional vt
 dev/vte/if_vte.c		optional vte pci
 dev/vx/if_vx.c			optional vx
 dev/vx/if_vx_eisa.c		optional vx eisa
 dev/vx/if_vx_pci.c		optional vx pci
 dev/vxge/vxge.c				optional vxge
 dev/vxge/vxgehal/vxgehal-ifmsg.c	optional vxge
 dev/vxge/vxgehal/vxgehal-mrpcim.c	optional vxge
 dev/vxge/vxgehal/vxge-queue.c		optional vxge
 dev/vxge/vxgehal/vxgehal-ring.c		optional vxge
 dev/vxge/vxgehal/vxgehal-swapper.c	optional vxge
 dev/vxge/vxgehal/vxgehal-mgmt.c		optional vxge
 dev/vxge/vxgehal/vxgehal-srpcim.c	optional vxge
 dev/vxge/vxgehal/vxgehal-config.c	optional vxge
 dev/vxge/vxgehal/vxgehal-blockpool.c	optional vxge
 dev/vxge/vxgehal/vxgehal-doorbells.c	optional vxge
 dev/vxge/vxgehal/vxgehal-mgmtaux.c	optional vxge
 dev/vxge/vxgehal/vxgehal-device.c	optional vxge
 dev/vxge/vxgehal/vxgehal-mm.c		optional vxge
 dev/vxge/vxgehal/vxgehal-driver.c	optional vxge
 dev/vxge/vxgehal/vxgehal-virtualpath.c	optional vxge
 dev/vxge/vxgehal/vxgehal-channel.c	optional vxge
 dev/vxge/vxgehal/vxgehal-fifo.c		optional vxge
 dev/watchdog/watchdog.c		standard
 dev/wb/if_wb.c			optional wb pci
 dev/wds/wd7000.c		optional wds isa
 dev/wi/if_wi.c			optional wi
 dev/wi/if_wi_pccard.c		optional wi pccard
 dev/wi/if_wi_pci.c		optional wi pci
 dev/wl/if_wl.c			optional wl isa
 dev/wpi/if_wpi.c		optional wpi pci
 wpifw.c			optional wpifw					\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk wpi.fw:wpifw:153229 -mwpi -c${.TARGET}" \
 	no-implicit-rule before-depend local				\
 	clean		"wpifw.c"
 wpifw.fwo			optional wpifw				\
 	dependency	"wpi.fw"					\
 	compile-with	"${NORMAL_FWO}"					\
 	no-implicit-rule						\
 	clean		"wpifw.fwo"
 wpi.fw			optional wpifw					\
 	dependency	"$S/contrib/dev/wpi/iwlwifi-3945-15.32.2.9.fw.uu"	\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"wpi.fw"
 dev/xe/if_xe.c			optional xe
 dev/xe/if_xe_pccard.c		optional xe pccard
 dev/xen/balloon/balloon.c	optional xen | xenhvm
 dev/xen/blkfront/blkfront.c	optional xen | xenhvm
 dev/xen/blkback/blkback.c	optional xen | xenhvm
 dev/xen/console/console.c	optional xen
 dev/xen/console/xencons_ring.c	optional xen
 dev/xen/control/control.c	optional xen | xenhvm
 dev/xen/netback/netback.c	optional xen | xenhvm
 dev/xen/netfront/netfront.c	optional xen | xenhvm
 dev/xen/xenpci/xenpci.c		optional xenpci
 dev/xen/timer/timer.c		optional xen | xenhvm
 dev/xl/if_xl.c			optional xl pci
 dev/xl/xlphy.c			optional xl pci
 fs/deadfs/dead_vnops.c		standard
 fs/devfs/devfs_devs.c		standard
 fs/devfs/devfs_dir.c		standard
 fs/devfs/devfs_rule.c		standard
 fs/devfs/devfs_vfsops.c		standard
 fs/devfs/devfs_vnops.c		standard
 fs/fdescfs/fdesc_vfsops.c	optional fdescfs
 fs/fdescfs/fdesc_vnops.c	optional fdescfs
 fs/fifofs/fifo_vnops.c		standard
 fs/fuse/fuse_device.c		optional fuse
 fs/fuse/fuse_file.c		optional fuse
 fs/fuse/fuse_internal.c		optional fuse
 fs/fuse/fuse_io.c		optional fuse
 fs/fuse/fuse_ipc.c		optional fuse
 fs/fuse/fuse_main.c		optional fuse
 fs/fuse/fuse_node.c		optional fuse
 fs/fuse/fuse_vfsops.c		optional fuse
 fs/fuse/fuse_vnops.c		optional fuse
 fs/msdosfs/msdosfs_conv.c	optional msdosfs
 fs/msdosfs/msdosfs_denode.c	optional msdosfs
 fs/msdosfs/msdosfs_fat.c	optional msdosfs
 fs/msdosfs/msdosfs_fileno.c	optional msdosfs
 fs/msdosfs/msdosfs_iconv.c	optional msdosfs_iconv
 fs/msdosfs/msdosfs_lookup.c	optional msdosfs
 fs/msdosfs/msdosfs_vfsops.c	optional msdosfs
 fs/msdosfs/msdosfs_vnops.c	optional msdosfs
 fs/nandfs/bmap.c		optional nandfs
 fs/nandfs/nandfs_alloc.c	optional nandfs
 fs/nandfs/nandfs_bmap.c		optional nandfs
 fs/nandfs/nandfs_buffer.c	optional nandfs
 fs/nandfs/nandfs_cleaner.c	optional nandfs
 fs/nandfs/nandfs_cpfile.c	optional nandfs
 fs/nandfs/nandfs_dat.c		optional nandfs
 fs/nandfs/nandfs_dir.c		optional nandfs
 fs/nandfs/nandfs_ifile.c	optional nandfs
 fs/nandfs/nandfs_segment.c	optional nandfs
 fs/nandfs/nandfs_subr.c		optional nandfs
 fs/nandfs/nandfs_sufile.c	optional nandfs
 fs/nandfs/nandfs_vfsops.c	optional nandfs
 fs/nandfs/nandfs_vnops.c	optional nandfs
 fs/nfs/nfs_commonkrpc.c		optional nfscl | nfsd
 fs/nfs/nfs_commonsubs.c		optional nfscl | nfsd
 fs/nfs/nfs_commonport.c		optional nfscl | nfsd
 fs/nfs/nfs_commonacl.c		optional nfscl | nfsd
 fs/nfsclient/nfs_clcomsubs.c	optional nfscl
 fs/nfsclient/nfs_clsubs.c	optional nfscl
 fs/nfsclient/nfs_clstate.c	optional nfscl
 fs/nfsclient/nfs_clkrpc.c	optional nfscl
 fs/nfsclient/nfs_clrpcops.c	optional nfscl
 fs/nfsclient/nfs_clvnops.c	optional nfscl
 fs/nfsclient/nfs_clnode.c	optional nfscl
 fs/nfsclient/nfs_clvfsops.c	optional nfscl
 fs/nfsclient/nfs_clport.c	optional nfscl
 fs/nfsclient/nfs_clbio.c	optional nfscl
 fs/nfsclient/nfs_clnfsiod.c	optional nfscl
 fs/nfsserver/nfs_fha_new.c	optional nfsd inet
 fs/nfsserver/nfs_nfsdsocket.c	optional nfsd inet
 fs/nfsserver/nfs_nfsdsubs.c	optional nfsd inet
 fs/nfsserver/nfs_nfsdstate.c	optional nfsd inet
 fs/nfsserver/nfs_nfsdkrpc.c	optional nfsd inet
 fs/nfsserver/nfs_nfsdserv.c	optional nfsd inet
 fs/nfsserver/nfs_nfsdport.c	optional nfsd inet
 fs/nfsserver/nfs_nfsdcache.c	optional nfsd inet
 fs/nullfs/null_subr.c		optional nullfs
 fs/nullfs/null_vfsops.c		optional nullfs
 fs/nullfs/null_vnops.c		optional nullfs
 fs/procfs/procfs.c		optional procfs
 fs/procfs/procfs_ctl.c		optional procfs
 fs/procfs/procfs_dbregs.c	optional procfs
 fs/procfs/procfs_fpregs.c	optional procfs
 fs/procfs/procfs_ioctl.c	optional procfs
 fs/procfs/procfs_map.c		optional procfs
 fs/procfs/procfs_mem.c		optional procfs
 fs/procfs/procfs_note.c		optional procfs
 fs/procfs/procfs_osrel.c	optional procfs
 fs/procfs/procfs_regs.c		optional procfs
 fs/procfs/procfs_rlimit.c	optional procfs
 fs/procfs/procfs_status.c	optional procfs
 fs/procfs/procfs_type.c		optional procfs
 fs/pseudofs/pseudofs.c		optional pseudofs
 fs/pseudofs/pseudofs_fileno.c	optional pseudofs
 fs/pseudofs/pseudofs_vncache.c	optional pseudofs
 fs/pseudofs/pseudofs_vnops.c	optional pseudofs
 fs/smbfs/smbfs_io.c		optional smbfs
 fs/smbfs/smbfs_node.c		optional smbfs
 fs/smbfs/smbfs_smb.c		optional smbfs
 fs/smbfs/smbfs_subr.c		optional smbfs
 fs/smbfs/smbfs_vfsops.c		optional smbfs
 fs/smbfs/smbfs_vnops.c		optional smbfs
 fs/udf/osta.c			optional udf
 fs/udf/udf_iconv.c		optional udf_iconv
 fs/udf/udf_vfsops.c		optional udf
 fs/udf/udf_vnops.c		optional udf
 fs/unionfs/union_subr.c		optional unionfs
 fs/unionfs/union_vfsops.c	optional unionfs
 fs/unionfs/union_vnops.c	optional unionfs
 fs/tmpfs/tmpfs_vnops.c		optional tmpfs
 fs/tmpfs/tmpfs_fifoops.c 	optional tmpfs
 fs/tmpfs/tmpfs_vfsops.c 	optional tmpfs
 fs/tmpfs/tmpfs_subr.c 		optional tmpfs
 gdb/gdb_cons.c			optional gdb
 gdb/gdb_main.c			optional gdb
 gdb/gdb_packet.c		optional gdb
 geom/bde/g_bde.c		optional geom_bde
 geom/bde/g_bde_crypt.c		optional geom_bde
 geom/bde/g_bde_lock.c		optional geom_bde
 geom/bde/g_bde_work.c		optional geom_bde
 geom/cache/g_cache.c		optional geom_cache
 geom/concat/g_concat.c		optional geom_concat
 geom/eli/g_eli.c		optional geom_eli
 geom/eli/g_eli_crypto.c		optional geom_eli
 geom/eli/g_eli_ctl.c		optional geom_eli
 geom/eli/g_eli_integrity.c	optional geom_eli
 geom/eli/g_eli_key.c		optional geom_eli
 geom/eli/g_eli_key_cache.c	optional geom_eli
 geom/eli/g_eli_privacy.c	optional geom_eli
 geom/eli/pkcs5v2.c		optional geom_eli
 geom/gate/g_gate.c		optional geom_gate
 geom/geom_aes.c			optional geom_aes
 geom/geom_bsd.c			optional geom_bsd
 geom/geom_bsd_enc.c		optional geom_bsd
 geom/geom_ccd.c			optional ccd | geom_ccd
 geom/geom_ctl.c			standard
 geom/geom_dev.c			standard
 geom/geom_disk.c		standard
 geom/geom_dump.c		standard
 geom/geom_event.c		standard
 geom/geom_fox.c			optional geom_fox
 geom/geom_flashmap.c		optional fdt cfi | fdt nand
 geom/geom_io.c			standard
 geom/geom_kern.c		standard
 geom/geom_map.c			optional geom_map
 geom/geom_mbr.c			optional geom_mbr
 geom/geom_mbr_enc.c		optional geom_mbr
 geom/geom_pc98.c		optional geom_pc98
 geom/geom_pc98_enc.c		optional geom_pc98
 geom/geom_redboot.c		optional geom_redboot
 geom/geom_slice.c		standard
 geom/geom_subr.c		standard
 geom/geom_sunlabel.c		optional geom_sunlabel
 geom/geom_sunlabel_enc.c	optional geom_sunlabel
 geom/geom_vfs.c			standard
 geom/geom_vol_ffs.c		optional geom_vol
 geom/journal/g_journal.c	optional geom_journal
 geom/journal/g_journal_ufs.c	optional geom_journal
 geom/label/g_label.c		optional geom_label
 geom/label/g_label_ext2fs.c	optional geom_label
 geom/label/g_label_iso9660.c	optional geom_label
 geom/label/g_label_msdosfs.c	optional geom_label
 geom/label/g_label_ntfs.c	optional geom_label
 geom/label/g_label_reiserfs.c	optional geom_label
 geom/label/g_label_ufs.c	optional geom_label
 geom/label/g_label_gpt.c	optional geom_label
 geom/label/g_label_disk_ident.c	optional geom_label
 geom/linux_lvm/g_linux_lvm.c	optional geom_linux_lvm
 geom/mirror/g_mirror.c		optional geom_mirror
 geom/mirror/g_mirror_ctl.c	optional geom_mirror
 geom/mountver/g_mountver.c	optional geom_mountver
 geom/multipath/g_multipath.c	optional geom_multipath
 geom/nop/g_nop.c		optional geom_nop
 geom/part/g_part.c		standard
 geom/part/g_part_if.m		standard
 geom/part/g_part_apm.c		optional geom_part_apm
 geom/part/g_part_bsd.c		optional geom_part_bsd
 geom/part/g_part_bsd64.c	optional geom_part_bsd64
 geom/part/g_part_ebr.c		optional geom_part_ebr
 geom/part/g_part_gpt.c		optional geom_part_gpt
 geom/part/g_part_ldm.c		optional geom_part_ldm
 geom/part/g_part_mbr.c		optional geom_part_mbr
 geom/part/g_part_pc98.c		optional geom_part_pc98
 geom/part/g_part_vtoc8.c	optional geom_part_vtoc8
 geom/raid/g_raid.c		optional geom_raid
 geom/raid/g_raid_ctl.c		optional geom_raid
 geom/raid/g_raid_md_if.m	optional geom_raid
 geom/raid/g_raid_tr_if.m	optional geom_raid
 geom/raid/md_ddf.c		optional geom_raid
 geom/raid/md_intel.c		optional geom_raid
 geom/raid/md_jmicron.c		optional geom_raid
 geom/raid/md_nvidia.c		optional geom_raid
 geom/raid/md_promise.c		optional geom_raid
 geom/raid/md_sii.c		optional geom_raid
 geom/raid/tr_concat.c		optional geom_raid
 geom/raid/tr_raid0.c		optional geom_raid
 geom/raid/tr_raid1.c		optional geom_raid
 geom/raid/tr_raid1e.c		optional geom_raid
 geom/raid/tr_raid5.c		optional geom_raid
 geom/raid3/g_raid3.c		optional geom_raid3
 geom/raid3/g_raid3_ctl.c	optional geom_raid3
 geom/shsec/g_shsec.c		optional geom_shsec
 geom/stripe/g_stripe.c		optional geom_stripe
 geom/uncompress/g_uncompress.c	optional geom_uncompress
 contrib/xz-embedded/freebsd/xz_malloc.c	\
 	optional xz_embedded | geom_uncompress \
 	compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/"
 contrib/xz-embedded/linux/lib/xz/xz_crc32.c \
 	optional xz_embedded | geom_uncompress \
 	compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/"
 contrib/xz-embedded/linux/lib/xz/xz_dec_bcj.c \
 	optional xz_embedded | geom_uncompress \
 	compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/"
 contrib/xz-embedded/linux/lib/xz/xz_dec_lzma2.c \
 	optional xz_embedded | geom_uncompress \
 	compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/"
 contrib/xz-embedded/linux/lib/xz/xz_dec_stream.c \
 	optional xz_embedded | geom_uncompress \
 	compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/"
 geom/uzip/g_uzip.c		optional geom_uzip
 geom/vinum/geom_vinum.c		optional geom_vinum
 geom/vinum/geom_vinum_create.c	optional geom_vinum
 geom/vinum/geom_vinum_drive.c	optional geom_vinum
 geom/vinum/geom_vinum_plex.c	optional geom_vinum
 geom/vinum/geom_vinum_volume.c	optional geom_vinum
 geom/vinum/geom_vinum_subr.c	optional geom_vinum
 geom/vinum/geom_vinum_raid5.c	optional geom_vinum
 geom/vinum/geom_vinum_share.c	optional geom_vinum
 geom/vinum/geom_vinum_list.c	optional geom_vinum
 geom/vinum/geom_vinum_rm.c	optional geom_vinum
 geom/vinum/geom_vinum_init.c	optional geom_vinum
 geom/vinum/geom_vinum_state.c	optional geom_vinum
 geom/vinum/geom_vinum_rename.c	optional geom_vinum
 geom/vinum/geom_vinum_move.c	optional geom_vinum
 geom/vinum/geom_vinum_events.c	optional geom_vinum
 geom/virstor/binstream.c	optional geom_virstor
 geom/virstor/g_virstor.c	optional geom_virstor
 geom/virstor/g_virstor_md.c	optional geom_virstor
 geom/zero/g_zero.c		optional geom_zero
 fs/ext2fs/ext2_alloc.c		optional ext2fs
 fs/ext2fs/ext2_balloc.c		optional ext2fs
 fs/ext2fs/ext2_bmap.c		optional ext2fs
 fs/ext2fs/ext2_extents.c	optional ext2fs
 fs/ext2fs/ext2_inode.c		optional ext2fs
 fs/ext2fs/ext2_inode_cnv.c	optional ext2fs
 fs/ext2fs/ext2_hash.c		optional ext2fs
 fs/ext2fs/ext2_htree.c		optional ext2fs
 fs/ext2fs/ext2_lookup.c		optional ext2fs
 fs/ext2fs/ext2_subr.c		optional ext2fs
 fs/ext2fs/ext2_vfsops.c		optional ext2fs
 fs/ext2fs/ext2_vnops.c		optional ext2fs
 gnu/fs/reiserfs/reiserfs_hashes.c	optional reiserfs \
 	warning "kernel contains GPL contaminated ReiserFS filesystem"
 gnu/fs/reiserfs/reiserfs_inode.c	optional reiserfs
 gnu/fs/reiserfs/reiserfs_item_ops.c	optional reiserfs
 gnu/fs/reiserfs/reiserfs_namei.c	optional reiserfs
 gnu/fs/reiserfs/reiserfs_prints.c	optional reiserfs
 gnu/fs/reiserfs/reiserfs_stree.c	optional reiserfs
 gnu/fs/reiserfs/reiserfs_vfsops.c	optional reiserfs
 gnu/fs/reiserfs/reiserfs_vnops.c	optional reiserfs
 #
 isa/isa_if.m			standard
 isa/isa_common.c		optional isa
 isa/isahint.c			optional isa
 isa/pnp.c			optional isa isapnp
 isa/pnpparse.c			optional isa isapnp
 fs/cd9660/cd9660_bmap.c	optional cd9660
 fs/cd9660/cd9660_lookup.c	optional cd9660
 fs/cd9660/cd9660_node.c	optional cd9660
 fs/cd9660/cd9660_rrip.c	optional cd9660
 fs/cd9660/cd9660_util.c	optional cd9660
 fs/cd9660/cd9660_vfsops.c	optional cd9660
 fs/cd9660/cd9660_vnops.c	optional cd9660
 fs/cd9660/cd9660_iconv.c	optional cd9660_iconv
 kern/bus_if.m			standard
 kern/clock_if.m			standard
 kern/cpufreq_if.m		standard
 kern/device_if.m		standard
 kern/imgact_elf.c		standard
 kern/imgact_elf32.c		optional compat_freebsd32
 kern/imgact_shell.c		standard
 kern/inflate.c			optional gzip
 kern/init_main.c		standard
 kern/init_sysent.c		standard
 kern/ksched.c			optional _kposix_priority_scheduling
 kern/kern_acct.c		standard
 kern/kern_alq.c			optional alq
 kern/kern_clock.c		standard
 kern/kern_condvar.c		standard
 kern/kern_conf.c		standard
 kern/kern_cons.c		standard
 kern/kern_cpu.c			standard
 kern/kern_cpuset.c		standard
 kern/kern_context.c		standard
 kern/kern_descrip.c		standard
 kern/kern_dtrace.c		optional kdtrace_hooks
 kern/kern_environment.c		standard
 kern/kern_et.c			standard
 kern/kern_event.c		standard
 kern/kern_exec.c		standard
 kern/kern_exit.c		standard
 kern/kern_fail.c		standard
 kern/kern_ffclock.c		standard
 kern/kern_fork.c		standard
 kern/kern_gzio.c		optional gzio
 kern/kern_hhook.c		standard
 kern/kern_idle.c		standard
 kern/kern_intr.c		standard
 kern/kern_jail.c		standard
 kern/kern_khelp.c		standard
 kern/kern_kthread.c		standard
 kern/kern_ktr.c			optional ktr
 kern/kern_ktrace.c		standard
 kern/kern_linker.c		standard
 kern/kern_lock.c		standard
 kern/kern_lockf.c		standard
 kern/kern_lockstat.c		optional kdtrace_hooks
 kern/kern_loginclass.c		standard
 kern/kern_malloc.c		standard
 kern/kern_mbuf.c		standard
 kern/kern_mib.c			standard
 kern/kern_module.c		standard
 kern/kern_mtxpool.c		standard
 kern/kern_mutex.c		standard
 kern/kern_ntptime.c		standard
 kern/kern_osd.c			standard
 kern/kern_physio.c		standard
 kern/kern_pmc.c			standard
 kern/kern_poll.c		optional device_polling
 kern/kern_priv.c		standard
 kern/kern_proc.c		standard
 kern/kern_prot.c		standard
 kern/kern_racct.c		standard
 kern/kern_rangelock.c		standard
 kern/kern_rctl.c		standard
 kern/kern_resource.c		standard
 kern/kern_rmlock.c		standard
 kern/kern_rwlock.c		standard
 kern/kern_sdt.c			optional kdtrace_hooks
 kern/kern_sema.c		standard
 kern/kern_sharedpage.c		standard
 kern/kern_shutdown.c		standard
 kern/kern_sig.c			standard
 kern/kern_switch.c		standard
 kern/kern_sx.c			standard
 kern/kern_synch.c		standard
 kern/kern_syscalls.c		standard
 kern/kern_sysctl.c		standard
 kern/kern_tc.c			standard
 kern/kern_thr.c			standard
 kern/kern_thread.c		standard
 kern/kern_time.c		standard
 kern/kern_timeout.c		standard
 kern/kern_umtx.c		standard
 kern/kern_uuid.c		standard
 kern/kern_xxx.c			standard
 kern/link_elf.c			standard
 kern/linker_if.m		standard
 kern/md4c.c			optional netsmb
 kern/md5c.c			standard
 kern/p1003_1b.c			standard
 kern/posix4_mib.c		standard
 kern/sched_4bsd.c		optional sched_4bsd
 kern/sched_ule.c		optional sched_ule
 kern/serdev_if.m		standard
 kern/stack_protector.c		standard \
 	compile-with "${NORMAL_C:N-fstack-protector*}"
 kern/subr_acl_nfs4.c		optional ufs_acl | zfs
 kern/subr_acl_posix1e.c		optional ufs_acl
 kern/subr_autoconf.c		standard
 kern/subr_blist.c		standard
 kern/subr_bus.c			standard
 kern/subr_bus_dma.c		standard
 kern/subr_bufring.c		standard
 kern/subr_capability.c		standard
 kern/subr_clock.c		standard
 kern/subr_counter.c		standard
 kern/subr_devstat.c		standard
 kern/subr_disk.c		standard
 kern/subr_eventhandler.c	standard
 kern/subr_fattime.c		standard
 kern/subr_firmware.c		optional firmware
 kern/subr_hash.c		standard
 kern/subr_hints.c		standard
 kern/subr_kdb.c			standard
 kern/subr_kobj.c		standard
 kern/subr_lock.c		standard
 kern/subr_log.c			standard
 kern/subr_mbpool.c		optional libmbpool
 kern/subr_mchain.c		optional libmchain
 kern/subr_module.c		standard
 kern/subr_msgbuf.c		standard
 kern/subr_param.c		standard
 kern/subr_pcpu.c		standard
 kern/subr_pctrie.c		standard
 kern/subr_power.c		standard
 kern/subr_prf.c			standard
 kern/subr_prof.c		standard
 kern/subr_rman.c		standard
 kern/subr_rtc.c			standard
 kern/subr_sbuf.c		standard
 kern/subr_scanf.c		standard
 kern/subr_sglist.c		standard
 kern/subr_sleepqueue.c		standard
 kern/subr_smp.c			standard
 kern/subr_stack.c		optional ddb | stack | ktr
 kern/subr_taskqueue.c		standard
 kern/subr_terminal.c		optional vt
 kern/subr_trap.c		standard
 kern/subr_turnstile.c		standard
 kern/subr_uio.c			standard
 kern/subr_unit.c		standard
 kern/subr_vmem.c		standard
 kern/subr_witness.c		optional witness
 kern/sys_capability.c		standard
 kern/sys_generic.c		standard
 kern/sys_pipe.c			standard
 kern/sys_procdesc.c		standard
 kern/sys_process.c		standard
 kern/sys_socket.c		standard
 kern/syscalls.c			standard
 kern/sysv_ipc.c			standard
 kern/sysv_msg.c			optional sysvmsg
 kern/sysv_sem.c			optional sysvsem
 kern/sysv_shm.c			optional sysvshm
 kern/tty.c			standard
 kern/tty_compat.c		optional compat_43tty
 kern/tty_info.c			standard
 kern/tty_inq.c			standard
 kern/tty_outq.c			standard
 kern/tty_pts.c			standard
 kern/tty_tty.c			standard
 kern/tty_ttydisc.c		standard
 kern/uipc_accf.c		optional inet
 kern/uipc_debug.c		optional ddb
 kern/uipc_domain.c		standard
 kern/uipc_mbuf.c		standard
 kern/uipc_mbuf2.c		standard
 kern/uipc_mqueue.c		optional p1003_1b_mqueue
 kern/uipc_sem.c			optional p1003_1b_semaphores
 kern/uipc_shm.c			standard
 kern/uipc_sockbuf.c		standard
 kern/uipc_socket.c		standard
 kern/uipc_syscalls.c		standard
 kern/uipc_usrreq.c		standard
 kern/vfs_acl.c			standard
 kern/vfs_aio.c			optional vfs_aio
 kern/vfs_bio.c			standard
 kern/vfs_cache.c		standard
 kern/vfs_cluster.c		standard
 kern/vfs_default.c		standard
 kern/vfs_export.c		standard
 kern/vfs_extattr.c		standard
 kern/vfs_hash.c			standard
 kern/vfs_init.c			standard
 kern/vfs_lookup.c		standard
 kern/vfs_mount.c		standard
 kern/vfs_mountroot.c		standard
 kern/vfs_subr.c			standard
 kern/vfs_syscalls.c		standard
 kern/vfs_vnops.c		standard
 #
 # Kernel GSS-API
 #
 gssd.h				optional kgssapi			\
 	dependency		"$S/kgssapi/gssd.x"			\
 	compile-with		"RPCGEN_CPP='${CPP}' rpcgen -hM $S/kgssapi/gssd.x | grep -v pthread.h > gssd.h" \
 	no-obj no-implicit-rule before-depend local			\
 	clean			"gssd.h"
 gssd_xdr.c			optional kgssapi			\
 	dependency		"$S/kgssapi/gssd.x gssd.h"		\
 	compile-with		"RPCGEN_CPP='${CPP}' rpcgen -c $S/kgssapi/gssd.x -o gssd_xdr.c" \
 	no-implicit-rule before-depend local				\
 	clean			"gssd_xdr.c"
 gssd_clnt.c			optional kgssapi			\
 	dependency		"$S/kgssapi/gssd.x gssd.h"		\
 	compile-with		"RPCGEN_CPP='${CPP}' rpcgen -lM $S/kgssapi/gssd.x | grep -v string.h > gssd_clnt.c" \
 	no-implicit-rule before-depend local				\
 	clean			"gssd_clnt.c"
 kgssapi/gss_accept_sec_context.c optional kgssapi
 kgssapi/gss_add_oid_set_member.c optional kgssapi
 kgssapi/gss_acquire_cred.c	optional kgssapi
 kgssapi/gss_canonicalize_name.c	optional kgssapi
 kgssapi/gss_create_empty_oid_set.c optional kgssapi
 kgssapi/gss_delete_sec_context.c optional kgssapi
 kgssapi/gss_display_status.c	optional kgssapi
 kgssapi/gss_export_name.c	optional kgssapi
 kgssapi/gss_get_mic.c		optional kgssapi
 kgssapi/gss_init_sec_context.c	optional kgssapi
 kgssapi/gss_impl.c		optional kgssapi
 kgssapi/gss_import_name.c	optional kgssapi
 kgssapi/gss_names.c		optional kgssapi
 kgssapi/gss_pname_to_uid.c	optional kgssapi
 kgssapi/gss_release_buffer.c	optional kgssapi
 kgssapi/gss_release_cred.c	optional kgssapi
 kgssapi/gss_release_name.c	optional kgssapi
 kgssapi/gss_release_oid_set.c	optional kgssapi
 kgssapi/gss_set_cred_option.c	optional kgssapi
 kgssapi/gss_test_oid_set_member.c optional kgssapi
 kgssapi/gss_unwrap.c		optional kgssapi
 kgssapi/gss_verify_mic.c	optional kgssapi
 kgssapi/gss_wrap.c		optional kgssapi
 kgssapi/gss_wrap_size_limit.c	optional kgssapi
 kgssapi/gssd_prot.c		optional kgssapi
 kgssapi/krb5/krb5_mech.c	optional kgssapi
 kgssapi/krb5/kcrypto.c		optional kgssapi
 kgssapi/krb5/kcrypto_aes.c	optional kgssapi
 kgssapi/krb5/kcrypto_arcfour.c	optional kgssapi
 kgssapi/krb5/kcrypto_des.c	optional kgssapi
 kgssapi/krb5/kcrypto_des3.c	optional kgssapi
 kgssapi/kgss_if.m		optional kgssapi
 kgssapi/gsstest.c		optional kgssapi_debug
 # These files in libkern/ are those needed by all architectures.  Some
 # of the files in libkern/ are only needed on some architectures, e.g.,
 # libkern/divdi3.c is needed by i386 but not alpha.  Also, some of these
 # routines may be optimized for a particular platform.  In either case,
 # the file should be moved to conf/files.<arch> from here.
 #
 libkern/arc4random.c		standard
 libkern/bcd.c			standard
 libkern/bsearch.c		standard
 libkern/crc32.c			standard
 libkern/flsll.c                 standard
 libkern/fnmatch.c		standard
 libkern/iconv.c			optional libiconv
 libkern/iconv_converter_if.m	optional libiconv
 libkern/iconv_ucs.c		optional libiconv
 libkern/iconv_xlat.c		optional libiconv
 libkern/iconv_xlat16.c		optional libiconv
 libkern/inet_aton.c		standard
 libkern/inet_ntoa.c		standard
 libkern/inet_ntop.c		standard
 libkern/inet_pton.c		standard
 libkern/jenkins_hash.c		standard
 libkern/mcount.c		optional profiling-routine
 libkern/memcchr.c		standard
 libkern/memchr.c		optional fdt
 libkern/memcmp.c		standard
 libkern/qsort.c			standard
 libkern/qsort_r.c		standard
 libkern/random.c		standard
 libkern/scanc.c			standard
 libkern/strcasecmp.c		standard
 libkern/strcat.c		standard
 libkern/strchr.c		standard
 libkern/strcmp.c		standard
 libkern/strcpy.c		standard
 libkern/strcspn.c		standard
 libkern/strdup.c		standard
 libkern/strlcat.c		standard
 libkern/strlcpy.c		standard
 libkern/strlen.c		standard
 libkern/strncmp.c		standard
 libkern/strncpy.c		standard
 libkern/strnlen.c		standard
 libkern/strrchr.c		standard
 libkern/strsep.c		standard
 libkern/strspn.c		standard
 libkern/strstr.c		standard
 libkern/strtol.c		standard
 libkern/strtoq.c		standard
 libkern/strtoul.c		standard
 libkern/strtouq.c		standard
 libkern/strvalid.c		standard
 net/bpf.c			standard
 net/bpf_buffer.c		optional bpf
 net/bpf_jitter.c		optional bpf_jitter
 net/bpf_filter.c		optional bpf | netgraph_bpf
 net/bpf_zerocopy.c		optional bpf
 net/bridgestp.c			optional bridge | if_bridge
 net/flowtable.c			optional flowtable inet | flowtable inet6
 net/ieee8023ad_lacp.c		optional lagg
 net/if.c			standard
 net/if_arcsubr.c		optional arcnet
 net/if_atmsubr.c		optional atm
 net/if_bridge.c			optional bridge inet | if_bridge inet
 net/if_clone.c			standard
 net/if_dead.c			standard
 net/if_debug.c			optional ddb
 net/if_disc.c			optional disc
 net/if_edsc.c			optional edsc
 net/if_ef.c			optional ef
 net/if_enc.c			optional enc ipsec inet | enc ipsec inet6
 net/if_epair.c			optional epair
 net/if_ethersubr.c		optional ether
 net/if_faith.c			optional faith
 net/if_fddisubr.c		optional fddi
 net/if_fwsubr.c			optional fwip
 net/if_gif.c			optional gif | netgraph_gif
 net/if_gre.c			optional gre inet
 net/if_iso88025subr.c		optional token
 net/if_lagg.c			optional lagg
 net/if_loop.c			optional loop
 net/if_llatbl.c			standard
 net/if_media.c			standard
 net/if_mib.c			standard
 net/if_spppfr.c			optional sppp | netgraph_sppp
 net/if_spppsubr.c		optional sppp | netgraph_sppp
 net/if_stf.c			optional stf inet inet6
 net/if_tun.c			optional tun
 net/if_tap.c			optional tap
 net/if_vlan.c			optional vlan
 net/mppcc.c			optional netgraph_mppc_compression
 net/mppcd.c			optional netgraph_mppc_compression
 net/netisr.c			standard
 net/pfil.c			optional ether | inet
 net/radix.c			standard
 net/radix_mpath.c		standard
 net/raw_cb.c			standard
 net/raw_usrreq.c		standard
 net/route.c			standard
 net/rtsock.c			standard
 net/slcompress.c		optional netgraph_vjc | sppp | \
 					 netgraph_sppp
 net/vnet.c			optional vimage
 net/zlib.c			optional crypto | geom_uzip | ipsec | \
 					 mxge | netgraph_deflate | \
 					 ddb_ctf | gzio | geom_uncompress
 net80211/ieee80211.c		optional wlan
 net80211/ieee80211_acl.c	optional wlan wlan_acl
 net80211/ieee80211_action.c	optional wlan
 net80211/ieee80211_ageq.c	optional wlan
 net80211/ieee80211_adhoc.c	optional wlan \
 	compile-with "${NORMAL_C} -Wno-unused-function"
 net80211/ieee80211_ageq.c	optional wlan
 net80211/ieee80211_amrr.c	optional wlan | wlan_amrr
 net80211/ieee80211_crypto.c	optional wlan \
 	compile-with "${NORMAL_C} -Wno-unused-function"
 net80211/ieee80211_crypto_ccmp.c optional wlan wlan_ccmp
 net80211/ieee80211_crypto_none.c optional wlan
 net80211/ieee80211_crypto_tkip.c optional wlan wlan_tkip
 net80211/ieee80211_crypto_wep.c	optional wlan wlan_wep
 net80211/ieee80211_ddb.c	optional wlan ddb
 net80211/ieee80211_dfs.c	optional wlan
 net80211/ieee80211_freebsd.c	optional wlan
 net80211/ieee80211_hostap.c	optional wlan \
 	compile-with "${NORMAL_C} -Wno-unused-function"
 net80211/ieee80211_ht.c		optional wlan
 net80211/ieee80211_hwmp.c	optional wlan ieee80211_support_mesh
 net80211/ieee80211_input.c	optional wlan
 net80211/ieee80211_ioctl.c	optional wlan
 net80211/ieee80211_mesh.c	optional wlan ieee80211_support_mesh \
 	compile-with "${NORMAL_C} -Wno-unused-function"
 net80211/ieee80211_monitor.c	optional wlan
 net80211/ieee80211_node.c	optional wlan
 net80211/ieee80211_output.c	optional wlan
 net80211/ieee80211_phy.c	optional wlan
 net80211/ieee80211_power.c	optional wlan
 net80211/ieee80211_proto.c	optional wlan
 net80211/ieee80211_radiotap.c	optional wlan
 net80211/ieee80211_ratectl.c	optional wlan
 net80211/ieee80211_ratectl_none.c optional wlan
 net80211/ieee80211_regdomain.c	optional wlan
 net80211/ieee80211_rssadapt.c	optional wlan wlan_rssadapt
 net80211/ieee80211_scan.c	optional wlan
 net80211/ieee80211_scan_sta.c	optional wlan
 net80211/ieee80211_sta.c	optional wlan \
 	compile-with "${NORMAL_C} -Wno-unused-function"
 net80211/ieee80211_superg.c	optional wlan ieee80211_support_superg
 net80211/ieee80211_tdma.c	optional wlan ieee80211_support_tdma
 net80211/ieee80211_wds.c	optional wlan
 net80211/ieee80211_xauth.c	optional wlan wlan_xauth
 net80211/ieee80211_alq.c	optional wlan ieee80211_alq
 netatalk/aarp.c			optional netatalk
 netatalk/at_control.c		optional netatalk
 netatalk/at_proto.c		optional netatalk
 netatalk/at_rmx.c		optional netatalk
 netatalk/ddp_input.c		optional netatalk
 netatalk/ddp_output.c		optional netatalk
 netatalk/ddp_pcb.c		optional netatalk
 netatalk/ddp_usrreq.c		optional netatalk
 netgraph/atm/ccatm/ng_ccatm.c	optional ngatm_ccatm \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 netgraph/atm/ng_atm.c		optional ngatm_atm
 netgraph/atm/ngatmbase.c	optional ngatm_atmbase \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 netgraph/atm/sscfu/ng_sscfu.c	optional ngatm_sscfu \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 netgraph/atm/sscop/ng_sscop.c optional ngatm_sscop \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 netgraph/atm/uni/ng_uni.c	optional ngatm_uni \
 	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
 netgraph/bluetooth/common/ng_bluetooth.c optional netgraph_bluetooth
 netgraph/bluetooth/drivers/bt3c/ng_bt3c_pccard.c optional netgraph_bluetooth_bt3c
 netgraph/bluetooth/drivers/h4/ng_h4.c optional netgraph_bluetooth_h4
 netgraph/bluetooth/drivers/ubt/ng_ubt.c optional netgraph_bluetooth_ubt usb
 netgraph/bluetooth/drivers/ubtbcmfw/ubtbcmfw.c optional netgraph_bluetooth_ubtbcmfw usb
 netgraph/bluetooth/hci/ng_hci_cmds.c optional netgraph_bluetooth_hci
 netgraph/bluetooth/hci/ng_hci_evnt.c optional netgraph_bluetooth_hci
 netgraph/bluetooth/hci/ng_hci_main.c optional netgraph_bluetooth_hci
 netgraph/bluetooth/hci/ng_hci_misc.c optional netgraph_bluetooth_hci
 netgraph/bluetooth/hci/ng_hci_ulpi.c optional netgraph_bluetooth_hci
 netgraph/bluetooth/l2cap/ng_l2cap_cmds.c optional netgraph_bluetooth_l2cap
 netgraph/bluetooth/l2cap/ng_l2cap_evnt.c optional netgraph_bluetooth_l2cap
 netgraph/bluetooth/l2cap/ng_l2cap_llpi.c optional netgraph_bluetooth_l2cap
 netgraph/bluetooth/l2cap/ng_l2cap_main.c optional netgraph_bluetooth_l2cap
 netgraph/bluetooth/l2cap/ng_l2cap_misc.c optional netgraph_bluetooth_l2cap
 netgraph/bluetooth/l2cap/ng_l2cap_ulpi.c optional netgraph_bluetooth_l2cap
 netgraph/bluetooth/socket/ng_btsocket.c optional netgraph_bluetooth_socket
 netgraph/bluetooth/socket/ng_btsocket_hci_raw.c	optional netgraph_bluetooth_socket
 netgraph/bluetooth/socket/ng_btsocket_l2cap.c optional netgraph_bluetooth_socket
 netgraph/bluetooth/socket/ng_btsocket_l2cap_raw.c optional netgraph_bluetooth_socket
 netgraph/bluetooth/socket/ng_btsocket_rfcomm.c optional netgraph_bluetooth_socket
 netgraph/bluetooth/socket/ng_btsocket_sco.c optional netgraph_bluetooth_socket
 netgraph/netflow/netflow.c	optional netgraph_netflow
 netgraph/netflow/netflow_v9.c	optional netgraph_netflow
 netgraph/netflow/ng_netflow.c	optional netgraph_netflow
 netgraph/ng_UI.c		optional netgraph_UI
 netgraph/ng_async.c		optional netgraph_async
 netgraph/ng_atmllc.c		optional netgraph_atmllc
 netgraph/ng_base.c		optional netgraph
 netgraph/ng_bpf.c		optional netgraph_bpf
 netgraph/ng_bridge.c		optional netgraph_bridge
 netgraph/ng_car.c		optional netgraph_car
 netgraph/ng_cisco.c		optional netgraph_cisco
 netgraph/ng_deflate.c		optional netgraph_deflate
 netgraph/ng_device.c		optional netgraph_device
 netgraph/ng_echo.c		optional netgraph_echo
 netgraph/ng_eiface.c		optional netgraph_eiface
 netgraph/ng_ether.c		optional netgraph_ether
 netgraph/ng_ether_echo.c	optional netgraph_ether_echo
 netgraph/ng_fec.c		optional netgraph_fec
 netgraph/ng_frame_relay.c	optional netgraph_frame_relay
 netgraph/ng_gif.c		optional netgraph_gif
 netgraph/ng_gif_demux.c		optional netgraph_gif_demux
 netgraph/ng_hole.c		optional netgraph_hole
 netgraph/ng_iface.c		optional netgraph_iface
 netgraph/ng_ip_input.c		optional netgraph_ip_input
 netgraph/ng_ipfw.c		optional netgraph_ipfw inet ipfirewall
 netgraph/ng_ksocket.c		optional netgraph_ksocket
 netgraph/ng_l2tp.c		optional netgraph_l2tp
 netgraph/ng_lmi.c		optional netgraph_lmi
 netgraph/ng_mppc.c		optional netgraph_mppc_compression | \
 					 netgraph_mppc_encryption
 netgraph/ng_nat.c		optional netgraph_nat inet libalias
 netgraph/ng_one2many.c		optional netgraph_one2many
 netgraph/ng_parse.c		optional netgraph
 netgraph/ng_patch.c		optional netgraph_patch
 netgraph/ng_pipe.c		optional netgraph_pipe
 netgraph/ng_ppp.c		optional netgraph_ppp
 netgraph/ng_pppoe.c		optional netgraph_pppoe
 netgraph/ng_pptpgre.c		optional netgraph_pptpgre
 netgraph/ng_pred1.c		optional netgraph_pred1
 netgraph/ng_rfc1490.c		optional netgraph_rfc1490
 netgraph/ng_socket.c		optional netgraph_socket
 netgraph/ng_split.c		optional netgraph_split
 netgraph/ng_sppp.c		optional netgraph_sppp
 netgraph/ng_tag.c		optional netgraph_tag
 netgraph/ng_tcpmss.c		optional netgraph_tcpmss
 netgraph/ng_tee.c		optional netgraph_tee
 netgraph/ng_tty.c		optional netgraph_tty
 netgraph/ng_vjc.c		optional netgraph_vjc
 netgraph/ng_vlan.c		optional netgraph_vlan
 netinet/accf_data.c		optional accept_filter_data inet
 netinet/accf_dns.c		optional accept_filter_dns inet
 netinet/accf_http.c		optional accept_filter_http inet
 netinet/if_atm.c		optional atm
 netinet/if_ether.c		optional inet ether
 netinet/igmp.c			optional inet
 netinet/in.c			optional inet
 netinet/in_debug.c		optional inet ddb
 netinet/in_kdtrace.c		optional inet | inet6
 netinet/ip_carp.c		optional inet carp | inet6 carp
 netinet/in_gif.c		optional gif inet | netgraph_gif inet
 netinet/ip_gre.c		optional gre inet
 netinet/ip_id.c			optional inet
 netinet/in_mcast.c		optional inet
 netinet/in_pcb.c		optional inet | inet6
 netinet/in_pcbgroup.c		optional inet pcbgroup | inet6 pcbgroup
 netinet/in_proto.c		optional inet | inet6
 netinet/in_rmx.c		optional inet
 netinet/ip_divert.c		optional inet ipdivert ipfirewall
 netinet/ip_ecn.c		optional inet | inet6
 netinet/ip_encap.c		optional inet | inet6
 netinet/ip_fastfwd.c		optional inet
 netinet/ip_icmp.c		optional inet | inet6
 netinet/ip_input.c		optional inet
 netinet/ip_ipsec.c		optional inet ipsec
 netinet/ip_mroute.c		optional mrouting inet
 netinet/ip_options.c		optional inet
 netinet/ip_output.c		optional inet
 netinet/raw_ip.c		optional inet | inet6
 netinet/cc/cc.c			optional inet | inet6
 netinet/cc/cc_newreno.c		optional inet | inet6
 netinet/sctp_asconf.c		optional inet sctp | inet6 sctp
 netinet/sctp_auth.c		optional inet sctp | inet6 sctp
 netinet/sctp_bsd_addr.c		optional inet sctp | inet6 sctp
 netinet/sctp_cc_functions.c	optional inet sctp | inet6 sctp
 netinet/sctp_crc32.c		optional inet sctp | inet6 sctp
 netinet/sctp_indata.c		optional inet sctp | inet6 sctp
 netinet/sctp_input.c		optional inet sctp | inet6 sctp
 netinet/sctp_output.c		optional inet sctp | inet6 sctp
 netinet/sctp_pcb.c		optional inet sctp | inet6 sctp
 netinet/sctp_peeloff.c		optional inet sctp | inet6 sctp
 netinet/sctp_ss_functions.c	optional inet sctp | inet6 sctp
 netinet/sctp_sysctl.c		optional inet sctp | inet6 sctp
 netinet/sctp_timer.c		optional inet sctp | inet6 sctp
 netinet/sctp_usrreq.c		optional inet sctp | inet6 sctp
 netinet/sctputil.c		optional inet sctp | inet6 sctp
 netinet/tcp_debug.c		optional tcpdebug
 netinet/tcp_hostcache.c		optional inet | inet6
 netinet/tcp_input.c		optional inet | inet6
 netinet/tcp_lro.c		optional inet | inet6
 netinet/tcp_output.c		optional inet | inet6
 netinet/tcp_offload.c		optional tcp_offload inet | tcp_offload inet6
 netinet/tcp_reass.c		optional inet | inet6
 netinet/tcp_sack.c		optional inet | inet6
 netinet/tcp_subr.c		optional inet | inet6
 netinet/tcp_syncache.c		optional inet | inet6
 netinet/tcp_timer.c		optional inet | inet6
 netinet/tcp_timewait.c		optional inet | inet6
 netinet/tcp_usrreq.c		optional inet | inet6
 netinet/udp_usrreq.c		optional inet | inet6
 netinet/libalias/alias.c	optional libalias inet | netgraph_nat inet
 netinet/libalias/alias_db.c	optional libalias inet | netgraph_nat inet
 netinet/libalias/alias_mod.c	optional libalias | netgraph_nat
 netinet/libalias/alias_proxy.c	optional libalias inet | netgraph_nat inet
 netinet/libalias/alias_util.c	optional libalias inet | netgraph_nat inet
 netinet/libalias/alias_sctp.c	optional libalias inet | netgraph_nat inet
 netinet6/dest6.c		optional inet6
 netinet6/frag6.c		optional inet6
 netinet6/icmp6.c		optional inet6
 netinet6/in6.c			optional inet6
 netinet6/in6_cksum.c		optional inet6
 netinet6/in6_gif.c		optional gif inet6 | netgraph_gif inet6
 netinet6/in6_ifattach.c		optional inet6
 netinet6/in6_mcast.c		optional inet6
 netinet6/in6_pcb.c		optional inet6
 netinet6/in6_pcbgroup.c		optional inet6 pcbgroup
 netinet6/in6_proto.c		optional inet6
 netinet6/in6_rmx.c		optional inet6
 netinet6/in6_src.c		optional inet6
 netinet6/ip6_forward.c		optional inet6
 netinet6/ip6_id.c		optional inet6
 netinet6/ip6_input.c		optional inet6
 netinet6/ip6_mroute.c		optional mrouting inet6
 netinet6/ip6_output.c		optional inet6
 netinet6/ip6_ipsec.c		optional inet6 ipsec
 netinet6/mld6.c			optional inet6
 netinet6/nd6.c			optional inet6
 netinet6/nd6_nbr.c		optional inet6
 netinet6/nd6_rtr.c		optional inet6
 netinet6/raw_ip6.c		optional inet6
 netinet6/route6.c		optional inet6
 netinet6/scope6.c		optional inet6
 netinet6/sctp6_usrreq.c		optional inet6 sctp
 netinet6/udp6_usrreq.c		optional inet6
 netipsec/ipsec.c		optional ipsec inet | ipsec inet6
 netipsec/ipsec_input.c		optional ipsec inet | ipsec inet6
 netipsec/ipsec_mbuf.c		optional ipsec inet | ipsec inet6
 netipsec/ipsec_output.c		optional ipsec inet | ipsec inet6
 netipsec/key.c			optional ipsec inet | ipsec inet6
 netipsec/key_debug.c		optional ipsec inet | ipsec inet6
 netipsec/keysock.c		optional ipsec inet | ipsec inet6
 netipsec/xform_ah.c		optional ipsec inet | ipsec inet6
 netipsec/xform_esp.c		optional ipsec inet | ipsec inet6
 netipsec/xform_ipcomp.c		optional ipsec inet | ipsec inet6
 netipsec/xform_ipip.c		optional ipsec inet | ipsec inet6
 netipsec/xform_tcp.c		optional ipsec inet tcp_signature | \
 					 ipsec inet6 tcp_signature
 netipx/ipx.c			optional ipx
 netipx/ipx_cksum.c		optional ipx
 netipx/ipx_input.c		optional ipx
 netipx/ipx_outputfl.c		optional ipx
 netipx/ipx_pcb.c		optional ipx
 netipx/ipx_proto.c		optional ipx
 netipx/ipx_usrreq.c		optional ipx
 netipx/spx_debug.c		optional ipx
 netipx/spx_reass.c		optional ipx
 netipx/spx_usrreq.c		optional ipx
 netnatm/natm.c			optional natm
 netnatm/natm_pcb.c		optional natm
 netnatm/natm_proto.c		optional natm
 netpfil/ipfw/dn_heap.c		optional inet dummynet
 netpfil/ipfw/dn_sched_fifo.c	optional inet dummynet
 netpfil/ipfw/dn_sched_prio.c	optional inet dummynet
 netpfil/ipfw/dn_sched_qfq.c	optional inet dummynet
 netpfil/ipfw/dn_sched_rr.c	optional inet dummynet
 netpfil/ipfw/dn_sched_wf2q.c	optional inet dummynet
 netpfil/ipfw/ip_dummynet.c	optional inet dummynet
 netpfil/ipfw/ip_dn_io.c		optional inet dummynet
 netpfil/ipfw/ip_dn_glue.c	optional inet dummynet
 netpfil/ipfw/ip_fw2.c		optional inet ipfirewall
 netpfil/ipfw/ip_fw_dynamic.c	optional inet ipfirewall
 netpfil/ipfw/ip_fw_log.c	optional inet ipfirewall
 netpfil/ipfw/ip_fw_pfil.c	optional inet ipfirewall
 netpfil/ipfw/ip_fw_sockopt.c	optional inet ipfirewall
 netpfil/ipfw/ip_fw_table.c	optional inet ipfirewall
 netpfil/ipfw/ip_fw_nat.c	optional inet ipfirewall_nat
 netpfil/pf/if_pflog.c		optional pflog pf inet
 netpfil/pf/if_pfsync.c		optional pfsync pf inet
 netpfil/pf/pf.c			optional pf inet
 netpfil/pf/pf_if.c		optional pf inet
 netpfil/pf/pf_ioctl.c		optional pf inet
 netpfil/pf/pf_lb.c		optional pf inet
 netpfil/pf/pf_norm.c		optional pf inet
 netpfil/pf/pf_osfp.c		optional pf inet
 netpfil/pf/pf_ruleset.c		optional pf inet
 netpfil/pf/pf_table.c		optional pf inet
 netpfil/pf/in4_cksum.c		optional pf inet
 netsmb/smb_conn.c		optional netsmb
 netsmb/smb_crypt.c		optional netsmb
 netsmb/smb_dev.c		optional netsmb
 netsmb/smb_iod.c		optional netsmb
 netsmb/smb_rq.c			optional netsmb
 netsmb/smb_smb.c		optional netsmb
 netsmb/smb_subr.c		optional netsmb
 netsmb/smb_trantcp.c		optional netsmb
 netsmb/smb_usr.c		optional netsmb
 nfs/bootp_subr.c		optional bootp nfsclient | bootp nfscl
 nfs/krpc_subr.c			optional bootp nfsclient | bootp nfscl
 nfs/nfs_common.c		optional nfsclient | nfsserver
 nfs/nfs_diskless.c		optional nfsclient nfs_root | nfscl nfs_root
 nfs/nfs_fha.c			optional nfsserver | nfsd
 nfs/nfs_lock.c			optional nfsclient | nfscl | nfslockd | nfsd
 nfsclient/nfs_bio.c		optional nfsclient
 nfsclient/nfs_node.c		optional nfsclient
 nfsclient/nfs_krpc.c		optional nfsclient
 nfsclient/nfs_subs.c		optional nfsclient
 nfsclient/nfs_nfsiod.c		optional nfsclient
 nfsclient/nfs_vfsops.c		optional nfsclient
 nfsclient/nfs_vnops.c		optional nfsclient
 nfsserver/nfs_fha_old.c		optional nfsserver
 nfsserver/nfs_serv.c		optional nfsserver
 nfsserver/nfs_srvkrpc.c		optional nfsserver
 nfsserver/nfs_srvsubs.c		optional nfsserver
 nfs/nfs_nfssvc.c		optional nfsserver | nfscl | nfsd
 nlm/nlm_advlock.c		optional nfslockd | nfsd
 nlm/nlm_prot_clnt.c		optional nfslockd | nfsd
 nlm/nlm_prot_impl.c		optional nfslockd | nfsd
 nlm/nlm_prot_server.c		optional nfslockd | nfsd
 nlm/nlm_prot_svc.c		optional nfslockd | nfsd
 nlm/nlm_prot_xdr.c		optional nfslockd | nfsd
 nlm/sm_inter_xdr.c		optional nfslockd | nfsd
 
 # OpenFabrics Enterprise Distribution (Infiniband)
 ofed/include/linux/linux_compat.c		optional ofed		\
 	no-depend compile-with "${OFED_C}"
 ofed/include/linux/linux_idr.c			optional ofed		\
 	no-depend compile-with "${OFED_C}"
 ofed/include/linux/linux_radix.c		optional ofed		\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/core/addr.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/agent.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/cache.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 # XXX Mad.c must be ordered before cm.c for sysinit sets to occur in
 # the correct order.
 ofed/drivers/infiniband/core/mad.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/cm.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/ -Wno-unused-function"
 ofed/drivers/infiniband/core/cma.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/device.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/fmr_pool.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/iwcm.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/local_sa.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/mad_rmpp.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/multicast.c	optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/notice.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/packer.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/sa_query.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/smi.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/sysfs.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/ucm.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/ucma.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/ud_header.c	optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/umem.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/user_mad.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/uverbs_cmd.c	optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/uverbs_main.c	optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/uverbs_marshall.c	optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 ofed/drivers/infiniband/core/verbs.c		optional ofed		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
 
 ofed/drivers/infiniband/ulp/ipoib/ipoib_cm.c	optional ipoib		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/"
 #ofed/drivers/infiniband/ulp/ipoib/ipoib_fs.c	optional ipoib		\
 #	no-depend							\
 #	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/"
 ofed/drivers/infiniband/ulp/ipoib/ipoib_ib.c	optional ipoib		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/"
 ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c	optional ipoib		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/"
 ofed/drivers/infiniband/ulp/ipoib/ipoib_multicast.c	optional ipoib	\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/"
 ofed/drivers/infiniband/ulp/ipoib/ipoib_verbs.c	optional ipoib		\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/"
 #ofed/drivers/infiniband/ulp/ipoib/ipoib_vlan.c	optional ipoib		\
 #	no-depend							\
 #	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/"
 
 ofed/drivers/infiniband/ulp/sdp/sdp_bcopy.c	optional sdp inet	\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/"
 ofed/drivers/infiniband/ulp/sdp/sdp_main.c	optional sdp inet 	\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/"
 ofed/drivers/infiniband/ulp/sdp/sdp_rx.c	optional sdp inet 	\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/"
 ofed/drivers/infiniband/ulp/sdp/sdp_cma.c	optional sdp inet 	\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/"
 ofed/drivers/infiniband/ulp/sdp/sdp_tx.c	optional sdp inet 	\
 	no-depend							\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/"
 
 ofed/drivers/infiniband/hw/mlx4/alias_GUID.c    optional mlx4ib         \
         no-depend obj-prefix "mlx4ib_"                                  \
         compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/"
 ofed/drivers/infiniband/hw/mlx4/mcg.c           optional mlx4ib         \
         no-depend obj-prefix "mlx4ib_"                                  \
         compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/"
 ofed/drivers/infiniband/hw/mlx4/sysfs.c         optional mlx4ib         \
         no-depend obj-prefix "mlx4ib_"                                  \
         compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/"
 ofed/drivers/infiniband/hw/mlx4/cm.c            optional mlx4ib         \
         no-depend obj-prefix "mlx4ib_"                                  \
         compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/"
 ofed/drivers/infiniband/hw/mlx4/ah.c		optional mlx4ib		\
 	no-depend obj-prefix "mlx4ib_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/"
 ofed/drivers/infiniband/hw/mlx4/cq.c		optional mlx4ib		\
 	no-depend obj-prefix "mlx4ib_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/"
 ofed/drivers/infiniband/hw/mlx4/doorbell.c	optional mlx4ib		\
 	no-depend obj-prefix "mlx4ib_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/"
 ofed/drivers/infiniband/hw/mlx4/mad.c		optional mlx4ib		\
 	no-depend obj-prefix "mlx4ib_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/"
 ofed/drivers/infiniband/hw/mlx4/main.c		optional mlx4ib		\
 	no-depend obj-prefix "mlx4ib_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/"
 ofed/drivers/infiniband/hw/mlx4/mr.c		optional mlx4ib		\
 	no-depend obj-prefix "mlx4ib_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/"
 ofed/drivers/infiniband/hw/mlx4/qp.c		optional mlx4ib		\
 	no-depend obj-prefix "mlx4ib_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/"
 ofed/drivers/infiniband/hw/mlx4/srq.c		optional mlx4ib		\
 	no-depend obj-prefix "mlx4ib_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/"
 ofed/drivers/infiniband/hw/mlx4/wc.c		optional mlx4ib		\
 	no-depend obj-prefix "mlx4ib_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/"
 
 ofed/drivers/net/mlx4/alloc.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/catas.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/cmd.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/cq.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/eq.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/fw.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/icm.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/intf.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/main.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/mcg.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/ -Wno-unused"
 ofed/drivers/net/mlx4/mr.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/pd.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/port.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/profile.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/qp.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/reset.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/sense.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/srq.c			optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/resource_tracker.c        optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/sys_tune.c		optional mlx4ib | mlxen	\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 
 ofed/drivers/net/mlx4/en_cq.c			optional mlxen		\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/en_frag.c			optional mlxen		\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/en_main.c			optional mlxen		\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/en_netdev.c		optional mlxen		\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/en_port.c			optional mlxen		\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/en_resources.c		optional mlxen		\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/en_rx.c			optional mlxen		\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 ofed/drivers/net/mlx4/en_tx.c			optional mlxen		\
 	no-depend obj-prefix "mlx4_"					\
 	compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/"
 
 ofed/drivers/infiniband/hw/mthca/mthca_allocator.c	optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_av.c		optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_catas.c		optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_cmd.c		optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_cq.c		optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_eq.c		optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_mad.c		optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_main.c		optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_mcg.c		optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_memfree.c	optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_mr.c		optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_pd.c		optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_profile.c	optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_provider.c	optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_qp.c		optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_reset.c		optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_srq.c		optional mthca	\
 	no-depend compile-with "${OFED_C}"
 ofed/drivers/infiniband/hw/mthca/mthca_uar.c		optional mthca	\
 	no-depend compile-with "${OFED_C}"
 
 # crypto support
 opencrypto/cast.c		optional crypto | ipsec
 opencrypto/criov.c		optional crypto
 opencrypto/crypto.c		optional crypto
 opencrypto/cryptodev.c		optional cryptodev
 opencrypto/cryptodev_if.m	optional crypto
 opencrypto/cryptosoft.c		optional crypto
 opencrypto/cryptodeflate.c	optional crypto
 opencrypto/rmd160.c		optional crypto | ipsec
 opencrypto/skipjack.c		optional crypto
 opencrypto/xform.c		optional crypto
 pci/alpm.c			optional alpm pci
 pci/amdpm.c			optional amdpm pci | nfpm pci
 pci/amdsmb.c			optional amdsmb pci
 pci/if_rl.c			optional rl pci
 pci/intpm.c			optional intpm pci
 pci/ncr.c			optional ncr pci \
 	compile-with "${NORMAL_C} -Wno-unused"
 pci/nfsmb.c			optional nfsmb pci
 pci/viapm.c			optional viapm pci
 rpc/auth_none.c			optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 rpc/auth_unix.c			optional krpc | nfslockd | nfsclient | nfscl | nfsd
 rpc/authunix_prot.c		optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 rpc/clnt_bck.c			optional krpc | nfslockd | nfsserver | nfscl | nfsd
 rpc/clnt_dg.c			optional krpc | nfslockd | nfsclient | nfscl | nfsd
 rpc/clnt_rc.c			optional krpc | nfslockd | nfsclient | nfscl | nfsd
 rpc/clnt_vc.c			optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 rpc/getnetconfig.c		optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 rpc/replay.c			optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 rpc/rpc_callmsg.c		optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 rpc/rpc_generic.c		optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 rpc/rpc_prot.c			optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 rpc/rpcb_clnt.c			optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 rpc/rpcb_prot.c			optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 rpc/svc.c			optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 rpc/svc_auth.c			optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 rpc/svc_auth_unix.c		optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 rpc/svc_dg.c			optional krpc | nfslockd | nfsserver | nfscl | nfsd
 rpc/svc_generic.c		optional krpc | nfslockd | nfsserver | nfscl | nfsd
 rpc/svc_vc.c			optional krpc | nfslockd | nfsserver | nfscl | nfsd
 rpc/rpcsec_gss/rpcsec_gss.c	optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi
 rpc/rpcsec_gss/rpcsec_gss_conf.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi
 rpc/rpcsec_gss/rpcsec_gss_misc.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi
 rpc/rpcsec_gss/rpcsec_gss_prot.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi
 rpc/rpcsec_gss/svc_rpcsec_gss.c	optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi
 security/audit/audit.c		optional audit
 security/audit/audit_arg.c	optional audit
 security/audit/audit_bsm.c	optional audit
 security/audit/audit_bsm_klib.c	optional audit
 security/audit/audit_pipe.c	optional audit
 security/audit/audit_syscalls.c	standard
 security/audit/audit_trigger.c	optional audit
 security/audit/audit_worker.c	optional audit
 security/audit/bsm_domain.c	optional audit
 security/audit/bsm_errno.c	optional audit
 security/audit/bsm_fcntl.c	optional audit
 security/audit/bsm_socket_type.c	optional audit
 security/audit/bsm_token.c	optional audit
 security/mac/mac_atalk.c	optional mac netatalk
 security/mac/mac_audit.c	optional mac audit
 security/mac/mac_cred.c		optional mac
 security/mac/mac_framework.c	optional mac
 security/mac/mac_inet.c		optional mac inet | mac inet6
 security/mac/mac_inet6.c	optional mac inet6
 security/mac/mac_label.c	optional mac
 security/mac/mac_net.c		optional mac
 security/mac/mac_pipe.c		optional mac
 security/mac/mac_posix_sem.c	optional mac
 security/mac/mac_posix_shm.c	optional mac
 security/mac/mac_priv.c		optional mac
 security/mac/mac_process.c	optional mac
 security/mac/mac_socket.c	optional mac
 security/mac/mac_syscalls.c	standard
 security/mac/mac_system.c	optional mac
 security/mac/mac_sysv_msg.c	optional mac
 security/mac/mac_sysv_sem.c	optional mac
 security/mac/mac_sysv_shm.c	optional mac
 security/mac/mac_vfs.c		optional mac
 security/mac_biba/mac_biba.c	optional mac_biba
 security/mac_bsdextended/mac_bsdextended.c	optional mac_bsdextended
 security/mac_bsdextended/ugidfw_system.c	optional mac_bsdextended
 security/mac_bsdextended/ugidfw_vnode.c		optional mac_bsdextended
 security/mac_ifoff/mac_ifoff.c	optional mac_ifoff
 security/mac_lomac/mac_lomac.c	optional mac_lomac
 security/mac_mls/mac_mls.c	optional mac_mls
 security/mac_none/mac_none.c	optional mac_none
 security/mac_partition/mac_partition.c optional mac_partition
 security/mac_portacl/mac_portacl.c optional mac_portacl
 security/mac_seeotheruids/mac_seeotheruids.c optional mac_seeotheruids
 security/mac_stub/mac_stub.c	optional mac_stub
 security/mac_test/mac_test.c	optional mac_test
 teken/teken.c			optional sc | vt
 ufs/ffs/ffs_alloc.c		optional ffs
 ufs/ffs/ffs_balloc.c		optional ffs
 ufs/ffs/ffs_inode.c		optional ffs
 ufs/ffs/ffs_snapshot.c		optional ffs
 ufs/ffs/ffs_softdep.c		optional ffs
 ufs/ffs/ffs_subr.c		optional ffs
 ufs/ffs/ffs_tables.c		optional ffs
 ufs/ffs/ffs_vfsops.c		optional ffs
 ufs/ffs/ffs_vnops.c		optional ffs
 ufs/ffs/ffs_rawread.c		optional ffs directio
 ufs/ffs/ffs_suspend.c		optional ffs
 ufs/ufs/ufs_acl.c		optional ffs
 ufs/ufs/ufs_bmap.c		optional ffs
 ufs/ufs/ufs_dirhash.c		optional ffs
 ufs/ufs/ufs_extattr.c		optional ffs
 ufs/ufs/ufs_gjournal.c		optional ffs UFS_GJOURNAL
 ufs/ufs/ufs_inode.c		optional ffs
 ufs/ufs/ufs_lookup.c		optional ffs
 ufs/ufs/ufs_quota.c		optional ffs
 ufs/ufs/ufs_vfsops.c		optional ffs
 ufs/ufs/ufs_vnops.c		optional ffs
 vm/default_pager.c		standard
 vm/device_pager.c		standard
 vm/phys_pager.c			standard
 vm/redzone.c			optional DEBUG_REDZONE
 vm/sg_pager.c			standard
 vm/swap_pager.c			standard
 vm/uma_core.c			standard
 vm/uma_dbg.c			standard
 vm/memguard.c			optional DEBUG_MEMGUARD
 vm/vm_fault.c			standard
 vm/vm_glue.c			standard
 vm/vm_init.c			standard
 vm/vm_kern.c			standard
 vm/vm_map.c			standard
 vm/vm_meter.c			standard
 vm/vm_mmap.c			standard
 vm/vm_object.c			standard
 vm/vm_page.c			standard
 vm/vm_pageout.c			standard
 vm/vm_pager.c			standard
 vm/vm_phys.c			standard
 vm/vm_radix.c			standard
 vm/vm_reserv.c			standard
 vm/vm_unix.c			standard
 vm/vm_zeroidle.c		standard
 vm/vnode_pager.c		standard
 xen/gnttab.c			optional xen | xenhvm
 xen/features.c			optional xen | xenhvm
 xen/xenbus/xenbus_if.m		optional xen | xenhvm
 xen/xenbus/xenbus.c		optional xen | xenhvm
 xen/xenbus/xenbusb_if.m		optional xen | xenhvm
 xen/xenbus/xenbusb.c		optional xen | xenhvm
 xen/xenbus/xenbusb_front.c	optional xen | xenhvm
 xen/xenbus/xenbusb_back.c	optional xen | xenhvm
 xen/xenstore/xenstore.c		optional xen | xenhvm
 xen/xenstore/xenstore_dev.c	optional xen | xenhvm
 xdr/xdr.c			optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 xdr/xdr_array.c			optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 xdr/xdr_mbuf.c			optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 xdr/xdr_mem.c			optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 xdr/xdr_reference.c		optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
 xdr/xdr_sizeof.c		optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd
Index: stable/10/sys/dev/cxgbe/adapter.h
===================================================================
--- stable/10/sys/dev/cxgbe/adapter.h	(revision 270296)
+++ stable/10/sys/dev/cxgbe/adapter.h	(revision 270297)
@@ -1,924 +1,1071 @@
 /*-
  * Copyright (c) 2011 Chelsio Communications, Inc.
  * All rights reserved.
  * Written by: Navdeep Parhar <np@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $FreeBSD$
  *
  */
 
 #ifndef __T4_ADAPTER_H__
 #define __T4_ADAPTER_H__
 
 #include <sys/kernel.h>
 #include <sys/bus.h>
 #include <sys/rman.h>
 #include <sys/types.h>
 #include <sys/malloc.h>
 #include <dev/pci/pcivar.h>
 #include <dev/pci/pcireg.h>
 #include <machine/bus.h>
 #include <sys/socket.h>
 #include <sys/sysctl.h>
 #include <net/ethernet.h>
 #include <net/if.h>
 #include <net/if_media.h>
 #include <netinet/in.h>
 #include <netinet/tcp_lro.h>
 
 #include "offload.h"
+#include "common/t4_msg.h"
 #include "firmware/t4fw_interface.h"
 
 MALLOC_DECLARE(M_CXGBE);
 #define CXGBE_UNIMPLEMENTED(s) \
     panic("%s (%s, line %d) not implemented yet.", s, __FILE__, __LINE__)
 
 #if defined(__i386__) || defined(__amd64__)
 static __inline void
 prefetch(void *x)
 {
 	__asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
 }
 #else
 #define prefetch(x)
 #endif
 
 #ifndef SYSCTL_ADD_UQUAD
 #define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD
 #define sysctl_handle_64 sysctl_handle_quad
 #define CTLTYPE_U64 CTLTYPE_QUAD
 #endif
 
 #if (__FreeBSD_version >= 900030) || \
     ((__FreeBSD_version >= 802507) && (__FreeBSD_version < 900000))
 #define SBUF_DRAIN 1
 #endif
 
 #ifdef __amd64__
 /* XXX: need systemwide bus_space_read_8/bus_space_write_8 */
 static __inline uint64_t
 t4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle,
     bus_size_t offset)
 {
 	KASSERT(tag == X86_BUS_SPACE_MEM,
 	    ("%s: can only handle mem space", __func__));
 
 	return (*(volatile uint64_t *)(handle + offset));
 }
 
 static __inline void
 t4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh,
     bus_size_t offset, uint64_t value)
 {
 	KASSERT(tag == X86_BUS_SPACE_MEM,
 	    ("%s: can only handle mem space", __func__));
 
 	*(volatile uint64_t *)(bsh + offset) = value;
 }
 #else
 static __inline uint64_t
 t4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle,
     bus_size_t offset)
 {
 	return (uint64_t)bus_space_read_4(tag, handle, offset) +
 	    ((uint64_t)bus_space_read_4(tag, handle, offset + 4) << 32);
 }
 
 static __inline void
 t4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh,
     bus_size_t offset, uint64_t value)
 {
 	bus_space_write_4(tag, bsh, offset, value);
 	bus_space_write_4(tag, bsh, offset + 4, value >> 32);
 }
 #endif
 
 struct adapter;
 typedef struct adapter adapter_t;
 
 enum {
-	FW_IQ_QSIZE = 256,
-	FW_IQ_ESIZE = 64,	/* At least 64 mandated by the firmware spec */
+	/*
+	 * All ingress queues use this entry size.  Note that the firmware event
+	 * queue and any iq expecting CPL_RX_PKT in the descriptor needs this to
+	 * be at least 64.
+	 */
+	IQ_ESIZE = 64,
 
+	/* Default queue sizes for all kinds of ingress queues */
+	FW_IQ_QSIZE = 256,
 	RX_IQ_QSIZE = 1024,
-	RX_IQ_ESIZE = 64,	/* At least 64 so CPL_RX_PKT will fit */
 
-	EQ_ESIZE = 64,		/* All egress queues use this entry size */
+	/* All egress queues use this entry size */
+	EQ_ESIZE = 64,
 
-	RX_FL_ESIZE = EQ_ESIZE,	/* 8 64bit addresses */
+	/* Default queue sizes for all kinds of egress queues */
+	CTRL_EQ_QSIZE = 128,
+	TX_EQ_QSIZE = 1024,
+
 #if MJUMPAGESIZE != MCLBYTES
 	SW_ZONE_SIZES = 4,	/* cluster, jumbop, jumbo9k, jumbo16k */
 #else
 	SW_ZONE_SIZES = 3,	/* cluster, jumbo9k, jumbo16k */
 #endif
 	CL_METADATA_SIZE = CACHE_LINE_SIZE,
 
-	CTRL_EQ_QSIZE = 128,
-
-	TX_EQ_QSIZE = 1024,
+	SGE_MAX_WR_NDESC = SGE_MAX_WR_LEN / EQ_ESIZE, /* max WR size in desc */
 	TX_SGL_SEGS = 36,
 	TX_WR_FLITS = SGE_MAX_WR_LEN / 8
 };
 
 enum {
 	/* adapter intr_type */
 	INTR_INTX	= (1 << 0),
 	INTR_MSI 	= (1 << 1),
 	INTR_MSIX	= (1 << 2)
 };
 
 enum {
+	XGMAC_MTU	= (1 << 0),
+	XGMAC_PROMISC	= (1 << 1),
+	XGMAC_ALLMULTI	= (1 << 2),
+	XGMAC_VLANEX	= (1 << 3),
+	XGMAC_UCADDR	= (1 << 4),
+	XGMAC_MCADDRS	= (1 << 5),
+
+	XGMAC_ALL	= 0xffff
+};
+
+enum {
 	/* flags understood by begin_synchronized_op */
 	HOLD_LOCK	= (1 << 0),
 	SLEEP_OK	= (1 << 1),
 	INTR_OK		= (1 << 2),
 
 	/* flags understood by end_synchronized_op */
 	LOCK_HELD	= HOLD_LOCK,
 };
 
 enum {
 	/* adapter flags */
 	FULL_INIT_DONE	= (1 << 0),
 	FW_OK		= (1 << 1),
-	INTR_DIRECT	= (1 << 2),	/* direct interrupts for everything */
+	/* INTR_DIRECT	= (1 << 2),	No longer used. */
 	MASTER_PF	= (1 << 3),
 	ADAP_SYSCTL_CTX	= (1 << 4),
 	TOM_INIT_DONE	= (1 << 5),
 	BUF_PACKING_OK	= (1 << 6),
 
 	CXGBE_BUSY	= (1 << 9),
 
 	/* port flags */
 	DOOMED		= (1 << 0),
 	PORT_INIT_DONE	= (1 << 1),
 	PORT_SYSCTL_CTX	= (1 << 2),
 	HAS_TRACEQ	= (1 << 3),
+	INTR_RXQ	= (1 << 4),	/* All NIC rxq's take interrupts */
+	INTR_OFLD_RXQ	= (1 << 5),	/* All TOE rxq's take interrupts */
+	INTR_NM_RXQ	= (1 << 6),	/* All netmap rxq's take interrupts */
+	INTR_ALL	= (INTR_RXQ | INTR_OFLD_RXQ | INTR_NM_RXQ),
 };
 
 #define IS_DOOMED(pi)	((pi)->flags & DOOMED)
 #define SET_DOOMED(pi)	do {(pi)->flags |= DOOMED;} while (0)
 #define IS_BUSY(sc)	((sc)->flags & CXGBE_BUSY)
 #define SET_BUSY(sc)	do {(sc)->flags |= CXGBE_BUSY;} while (0)
 #define CLR_BUSY(sc)	do {(sc)->flags &= ~CXGBE_BUSY;} while (0)
 
 struct port_info {
 	device_t dev;
 	struct adapter *adapter;
 
 	struct ifnet *ifp;
 	struct ifmedia media;
 
 	struct mtx pi_lock;
 	char lockname[16];
 	unsigned long flags;
 	int if_flags;
 
 	uint16_t *rss;
 	uint16_t viid;
 	int16_t  xact_addr_filt;/* index of exact MAC address filter */
 	uint16_t rss_size;	/* size of VI's RSS table slice */
 	uint8_t  lport;		/* associated offload logical port */
 	int8_t   mdio_addr;
 	uint8_t  port_type;
 	uint8_t  mod_type;
 	uint8_t  port_id;
 	uint8_t  tx_chan;
 	uint8_t  rx_chan_map;	/* rx MPS channel bitmap */
 
 	/* These need to be int as they are used in sysctl */
 	int ntxq;	/* # of tx queues */
 	int first_txq;	/* index of first tx queue */
 	int rsrv_noflowq; /* Reserve queue 0 for non-flowid packets */
 	int nrxq;	/* # of rx queues */
 	int first_rxq;	/* index of first rx queue */
 #ifdef TCP_OFFLOAD
 	int nofldtxq;		/* # of offload tx queues */
 	int first_ofld_txq;	/* index of first offload tx queue */
 	int nofldrxq;		/* # of offload rx queues */
 	int first_ofld_rxq;	/* index of first offload rx queue */
 #endif
+#ifdef DEV_NETMAP
+	int nnmtxq;		/* # of netmap tx queues */
+	int first_nm_txq;	/* index of first netmap tx queue */
+	int nnmrxq;		/* # of netmap rx queues */
+	int first_nm_rxq;	/* index of first netmap rx queue */
+
+	struct ifnet *nm_ifp;
+	struct ifmedia nm_media;
+	int nmif_flags;
+	uint16_t nm_viid;
+	int16_t nm_xact_addr_filt;
+	uint16_t nm_rss_size;	/* size of netmap VI's RSS table slice */
+#endif
 	int tmr_idx;
 	int pktc_idx;
 	int qsize_rxq;
 	int qsize_txq;
 
 	int linkdnrc;
 	struct link_config link_cfg;
 	struct port_stats stats;
 
 	eventhandler_tag vlan_c;
 
 	struct callout tick;
 	struct sysctl_ctx_list ctx;	/* from ifconfig up to driver detach */
 
 	uint8_t hw_addr[ETHER_ADDR_LEN]; /* factory MAC address, won't change */
 };
 
 /* Where the cluster came from, how it has been carved up. */
 struct cluster_layout {
 	int8_t zidx;
 	int8_t hwidx;
 	uint16_t region1;	/* mbufs laid out within this region */
 				/* region2 is the DMA region */
 	uint16_t region3;	/* cluster_metadata within this region */
 };
 
 struct cluster_metadata {
 	u_int refcount;
 #ifdef INVARIANTS
 	struct fl_sdesc *sd;	/* For debug only.  Could easily be stale */
 #endif
 };
 
 struct fl_sdesc {
 	caddr_t cl;
 	uint16_t nmbuf;	/* # of driver originated mbufs with ref on cluster */
 	struct cluster_layout cll;
 };
 
 struct tx_desc {
 	__be64 flit[8];
 };
 
 struct tx_map {
 	struct mbuf *m;
 	bus_dmamap_t map;
 };
 
 /* DMA maps used for tx */
 struct tx_maps {
 	struct tx_map *maps;
 	uint32_t map_total;	/* # of DMA maps */
 	uint32_t map_pidx;	/* next map to be used */
 	uint32_t map_cidx;	/* reclaimed up to this index */
 	uint32_t map_avail;	/* # of available maps */
 };
 
 struct tx_sdesc {
 	uint8_t desc_used;	/* # of hardware descriptors used by the WR */
 	uint8_t credits;	/* NIC txq: # of frames sent out in the WR */
 };
 
+
+#define IQ_PAD (IQ_ESIZE - sizeof(struct rsp_ctrl) - sizeof(struct rss_header))
+struct iq_desc {
+	struct rss_header rss;
+	uint8_t cpl[IQ_PAD];
+	struct rsp_ctrl rsp;
+};
+#undef IQ_PAD
+CTASSERT(sizeof(struct iq_desc) == IQ_ESIZE);
+
 enum {
 	/* iq flags */
 	IQ_ALLOCATED	= (1 << 0),	/* firmware resources allocated */
 	IQ_HAS_FL	= (1 << 1),	/* iq associated with a freelist */
 	IQ_INTR		= (1 << 2),	/* iq takes direct interrupt */
 	IQ_LRO_ENABLED	= (1 << 3),	/* iq is an eth rxq with LRO enabled */
 
 	/* iq state */
 	IQS_DISABLED	= 0,
 	IQS_BUSY	= 1,
 	IQS_IDLE	= 2,
 };
 
 /*
  * Ingress Queue: T4 is producer, driver is consumer.
  */
 struct sge_iq {
-	bus_dma_tag_t desc_tag;
-	bus_dmamap_t desc_map;
-	bus_addr_t ba;		/* bus address of descriptor ring */
 	uint32_t flags;
-	uint16_t abs_id;	/* absolute SGE id for the iq */
-	int8_t   intr_pktc_idx;	/* packet count threshold index */
-	int8_t   pad0;
-	__be64  *desc;		/* KVA of descriptor ring */
-
 	volatile int state;
 	struct adapter *adapter;
-	const __be64 *cdesc;	/* current descriptor */
+	struct iq_desc  *desc;	/* KVA of descriptor ring */
+	int8_t   intr_pktc_idx;	/* packet count threshold index */
 	uint8_t  gen;		/* generation bit */
 	uint8_t  intr_params;	/* interrupt holdoff parameters */
 	uint8_t  intr_next;	/* XXX: holdoff for next interrupt */
-	uint8_t  esize;		/* size (bytes) of each entry in the queue */
 	uint16_t qsize;		/* size (# of entries) of the queue */
+	uint16_t sidx;		/* index of the entry with the status page */
 	uint16_t cidx;		/* consumer index */
 	uint16_t cntxt_id;	/* SGE context id for the iq */
+	uint16_t abs_id;	/* absolute SGE id for the iq */
 
 	STAILQ_ENTRY(sge_iq) link;
+
+	bus_dma_tag_t desc_tag;
+	bus_dmamap_t desc_map;
+	bus_addr_t ba;		/* bus address of descriptor ring */
 };
 
 enum {
 	EQ_CTRL		= 1,
 	EQ_ETH		= 2,
 #ifdef TCP_OFFLOAD
 	EQ_OFLD		= 3,
 #endif
 
 	/* eq flags */
 	EQ_TYPEMASK	= 7,		/* 3 lsbits hold the type */
 	EQ_ALLOCATED	= (1 << 3),	/* firmware resources allocated */
 	EQ_DOOMED	= (1 << 4),	/* about to be destroyed */
 	EQ_CRFLUSHED	= (1 << 5),	/* expecting an update from SGE */
 	EQ_STALLED	= (1 << 6),	/* out of hw descriptors or dmamaps */
 };
 
 /* Listed in order of preference.  Update t4_sysctls too if you change these */
 enum {DOORBELL_UDB, DOORBELL_WCWR, DOORBELL_UDBWC, DOORBELL_KDB};
 
 /*
  * Egress Queue: driver is producer, T4 is consumer.
  *
  * Note: A free list is an egress queue (driver produces the buffers and T4
  * consumes them) but it's special enough to have its own struct (see sge_fl).
  */
 struct sge_eq {
 	unsigned int flags;	/* MUST be first */
 	unsigned int cntxt_id;	/* SGE context id for the eq */
 	bus_dma_tag_t desc_tag;
 	bus_dmamap_t desc_map;
 	char lockname[16];
 	struct mtx eq_lock;
 
 	struct tx_desc *desc;	/* KVA of descriptor ring */
 	bus_addr_t ba;		/* bus address of descriptor ring */
 	struct sge_qstat *spg;	/* status page, for convenience */
-	int doorbells;
+	uint16_t doorbells;
 	volatile uint32_t *udb;	/* KVA of doorbell (lies within BAR2) */
 	u_int udb_qid;		/* relative qid within the doorbell page */
 	uint16_t cap;		/* max # of desc, for convenience */
 	uint16_t avail;		/* available descriptors, for convenience */
 	uint16_t qsize;		/* size (# of entries) of the queue */
 	uint16_t cidx;		/* consumer idx (desc idx) */
 	uint16_t pidx;		/* producer idx (desc idx) */
 	uint16_t pending;	/* # of descriptors used since last doorbell */
 	uint16_t iqid;		/* iq that gets egr_update for the eq */
 	uint8_t tx_chan;	/* tx channel used by the eq */
 	struct task tx_task;
 	struct callout tx_callout;
 
 	/* stats */
 
 	uint32_t egr_update;	/* # of SGE_EGR_UPDATE notifications for eq */
 	uint32_t unstalled;	/* recovered from stall */
 };
 
 struct sw_zone_info {
 	uma_zone_t zone;	/* zone that this cluster comes from */
 	int size;		/* size of cluster: 2K, 4K, 9K, 16K, etc. */
 	int type;		/* EXT_xxx type of the cluster */
 	int8_t head_hwidx;
 	int8_t tail_hwidx;
 };
 
 struct hw_buf_info {
 	int8_t zidx;		/* backpointer to zone; -ve means unused */
 	int8_t next;		/* next hwidx for this zone; -1 means no more */
 	int size;
 };
 
 enum {
 	FL_STARVING	= (1 << 0), /* on the adapter's list of starving fl's */
 	FL_DOOMED	= (1 << 1), /* about to be destroyed */
 	FL_BUF_PACKING	= (1 << 2), /* buffer packing enabled */
+	FL_BUF_RESUME	= (1 << 3), /* resume from the middle of the frame */
 };
 
-#define FL_RUNNING_LOW(fl)	(fl->cap - fl->needed <= fl->lowat)
-#define FL_NOT_RUNNING_LOW(fl)	(fl->cap - fl->needed >= 2 * fl->lowat)
+#define FL_RUNNING_LOW(fl) \
+    (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) <= fl->lowat)
+#define FL_NOT_RUNNING_LOW(fl) \
+    (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) >= 2 * fl->lowat)
 
 struct sge_fl {
-	bus_dma_tag_t desc_tag;
-	bus_dmamap_t desc_map;
-	struct cluster_layout cll_def;	/* default refill zone, layout */
-	struct cluster_layout cll_alt;	/* alternate refill zone, layout */
 	struct mtx fl_lock;
-	char lockname[16];
-	int flags;
-
 	__be64 *desc;		/* KVA of descriptor ring, ptr to addresses */
-	bus_addr_t ba;		/* bus address of descriptor ring */
 	struct fl_sdesc *sdesc;	/* KVA of software descriptor ring */
-	uint32_t cap;		/* max # of buffers, for convenience */
-	uint16_t qsize;		/* size (# of entries) of the queue */
-	uint16_t cntxt_id;	/* SGE context id for the freelist */
-	uint32_t cidx;		/* consumer idx (buffer idx, NOT hw desc idx) */
-	uint32_t rx_offset;	/* offset in fl buf (when buffer packing) */
-	uint32_t pidx;		/* producer idx (buffer idx, NOT hw desc idx) */
-	uint32_t needed;	/* # of buffers needed to fill up fl. */
-	uint32_t lowat;		/* # of buffers <= this means fl needs help */
-	uint32_t pending;	/* # of bufs allocated since last doorbell */
-	TAILQ_ENTRY(sge_fl) link; /* All starving freelists */
+	struct cluster_layout cll_def;	/* default refill zone, layout */
+	uint16_t lowat;		/* # of buffers <= this means fl needs help */
+	int flags;
+	uint16_t buf_boundary;
 
-	struct mbuf *m0;
-	struct mbuf **pnext;
-	u_int remaining;
+	/* The 16b idx all deal with hw descriptors */
+	uint16_t dbidx;		/* hw pidx after last doorbell */
+	uint16_t sidx;		/* index of status page */
+	volatile uint16_t hw_cidx;
 
+	/* The 32b idx are all buffer idx, not hardware descriptor idx */
+	uint32_t cidx;		/* consumer index */
+	uint32_t pidx;		/* producer index */
+
+	uint32_t dbval;
+	u_int rx_offset;	/* offset in fl buf (when buffer packing) */
+	volatile uint32_t *udb;
+
 	uint64_t mbuf_allocated;/* # of mbuf allocated from zone_mbuf */
 	uint64_t mbuf_inlined;	/* # of mbuf created within clusters */
 	uint64_t cl_allocated;	/* # of clusters allocated */
 	uint64_t cl_recycled;	/* # of clusters recycled */
 	uint64_t cl_fast_recycled; /* # of clusters recycled (fast) */
+
+	/* These 3 are valid when FL_BUF_RESUME is set, stale otherwise. */
+	struct mbuf *m0;
+	struct mbuf **pnext;
+	u_int remaining;
+
+	uint16_t qsize;		/* # of hw descriptors (status page included) */
+	uint16_t cntxt_id;	/* SGE context id for the freelist */
+	TAILQ_ENTRY(sge_fl) link; /* All starving freelists */
+	bus_dma_tag_t desc_tag;
+	bus_dmamap_t desc_map;
+	char lockname[16];
+	bus_addr_t ba;		/* bus address of descriptor ring */
+	struct cluster_layout cll_alt;	/* alternate refill zone, layout */
 };
 
 /* txq: SGE egress queue + what's needed for Ethernet NIC */
 struct sge_txq {
 	struct sge_eq eq;	/* MUST be first */
 
 	struct ifnet *ifp;	/* the interface this txq belongs to */
 	bus_dma_tag_t tx_tag;	/* tag for transmit buffers */
 	struct buf_ring *br;	/* tx buffer ring */
 	struct tx_sdesc *sdesc;	/* KVA of software descriptor ring */
 	struct mbuf *m;		/* held up due to temporary resource shortage */
 
 	struct tx_maps txmaps;
 
 	/* stats for common events first */
 
 	uint64_t txcsum;	/* # of times hardware assisted with checksum */
 	uint64_t tso_wrs;	/* # of TSO work requests */
 	uint64_t vlan_insertion;/* # of times VLAN tag was inserted */
 	uint64_t imm_wrs;	/* # of work requests with immediate data */
 	uint64_t sgl_wrs;	/* # of work requests with direct SGL */
 	uint64_t txpkt_wrs;	/* # of txpkt work requests (not coalesced) */
 	uint64_t txpkts_wrs;	/* # of coalesced tx work requests */
 	uint64_t txpkts_pkts;	/* # of frames in coalesced tx work requests */
 
 	/* stats for not-that-common events */
 
 	uint32_t no_dmamap;	/* no DMA map to load the mbuf */
 	uint32_t no_desc;	/* out of hardware descriptors */
 } __aligned(CACHE_LINE_SIZE);
 
 /* rxq: SGE ingress queue + SGE free list + miscellaneous items */
 struct sge_rxq {
 	struct sge_iq iq;	/* MUST be first */
 	struct sge_fl fl;	/* MUST follow iq */
 
 	struct ifnet *ifp;	/* the interface this rxq belongs to */
 #if defined(INET) || defined(INET6)
 	struct lro_ctrl lro;	/* LRO state */
 #endif
 
 	/* stats for common events first */
 
 	uint64_t rxcsum;	/* # of times hardware assisted with checksum */
 	uint64_t vlan_extraction;/* # of times VLAN tag was extracted */
 
 	/* stats for not-that-common events */
 
 } __aligned(CACHE_LINE_SIZE);
 
 static inline struct sge_rxq *
 iq_to_rxq(struct sge_iq *iq)
 {
 
 	return (__containerof(iq, struct sge_rxq, iq));
 }
 
 
 #ifdef TCP_OFFLOAD
 /* ofld_rxq: SGE ingress queue + SGE free list + miscellaneous items */
 struct sge_ofld_rxq {
 	struct sge_iq iq;	/* MUST be first */
 	struct sge_fl fl;	/* MUST follow iq */
 } __aligned(CACHE_LINE_SIZE);
 
 static inline struct sge_ofld_rxq *
 iq_to_ofld_rxq(struct sge_iq *iq)
 {
 
 	return (__containerof(iq, struct sge_ofld_rxq, iq));
 }
 #endif
 
 struct wrqe {
 	STAILQ_ENTRY(wrqe) link;
 	struct sge_wrq *wrq;
 	int wr_len;
 	uint64_t wr[] __aligned(16);
 };
 
 /*
  * wrq: SGE egress queue that is given prebuilt work requests.  Both the control
  * and offload tx queues are of this type.
  */
 struct sge_wrq {
 	struct sge_eq eq;	/* MUST be first */
 
 	struct adapter *adapter;
 
 	/* List of WRs held up due to lack of tx descriptors */
 	STAILQ_HEAD(, wrqe) wr_list;
 
 	/* stats for common events first */
 
 	uint64_t tx_wrs;	/* # of tx work requests */
 
 	/* stats for not-that-common events */
 
 	uint32_t no_desc;	/* out of hardware descriptors */
 } __aligned(CACHE_LINE_SIZE);
 
+
+#ifdef DEV_NETMAP
+struct sge_nm_rxq {
+	struct port_info *pi;
+
+	struct iq_desc *iq_desc;
+	uint16_t iq_abs_id;
+	uint16_t iq_cntxt_id;
+	uint16_t iq_cidx;
+	uint16_t iq_sidx;
+	uint8_t iq_gen;
+
+	__be64  *fl_desc;
+	uint16_t fl_cntxt_id;
+	uint32_t fl_cidx;
+	uint32_t fl_pidx;
+	uint32_t fl_sidx;
+	uint32_t fl_db_val;
+	u_int fl_hwidx:4;
+
+	u_int nid;		/* netmap ring # for this queue */
+
+	/* infrequently used items after this */
+
+	bus_dma_tag_t iq_desc_tag;
+	bus_dmamap_t iq_desc_map;
+	bus_addr_t iq_ba;
+	int intr_idx;
+
+	bus_dma_tag_t fl_desc_tag;
+	bus_dmamap_t fl_desc_map;
+	bus_addr_t fl_ba;
+} __aligned(CACHE_LINE_SIZE);
+
+struct sge_nm_txq {
+	struct tx_desc *desc;
+	uint16_t cidx;
+	uint16_t pidx;
+	uint16_t sidx;
+	uint16_t equiqidx;	/* EQUIQ last requested at this pidx */
+	uint16_t equeqidx;	/* EQUEQ last requested at this pidx */
+	uint16_t dbidx;		/* pidx of the most recent doorbell */
+	uint16_t doorbells;
+	volatile uint32_t *udb;
+	u_int udb_qid;
+	u_int cntxt_id;
+	__be32 cpl_ctrl0;	/* for convenience */
+	u_int nid;		/* netmap ring # for this queue */
+
+	/* infrequently used items after this */
+
+	bus_dma_tag_t desc_tag;
+	bus_dmamap_t desc_map;
+	bus_addr_t ba;
+	int iqidx;
+} __aligned(CACHE_LINE_SIZE);
+#endif
+
 struct sge {
 	int timer_val[SGE_NTIMERS];
 	int counter_val[SGE_NCOUNTERS];
 	int fl_starve_threshold;
 	int fl_starve_threshold2;
 	int eq_s_qpp;
 	int iq_s_qpp;
 
 	int nrxq;	/* total # of Ethernet rx queues */
 	int ntxq;	/* total # of Ethernet tx tx queues */
 #ifdef TCP_OFFLOAD
 	int nofldrxq;	/* total # of TOE rx queues */
 	int nofldtxq;	/* total # of TOE tx queues */
 #endif
+#ifdef DEV_NETMAP
+	int nnmrxq;	/* total # of netmap rx queues */
+	int nnmtxq;	/* total # of netmap tx queues */
+#endif
 	int niq;	/* total # of ingress queues */
 	int neq;	/* total # of egress queues */
 
 	struct sge_iq fwq;	/* Firmware event queue */
 	struct sge_wrq mgmtq;	/* Management queue (control queue) */
 	struct sge_wrq *ctrlq;	/* Control queues */
 	struct sge_txq *txq;	/* NIC tx queues */
 	struct sge_rxq *rxq;	/* NIC rx queues */
 #ifdef TCP_OFFLOAD
 	struct sge_wrq *ofld_txq;	/* TOE tx queues */
 	struct sge_ofld_rxq *ofld_rxq;	/* TOE rx queues */
 #endif
+#ifdef DEV_NETMAP
+	struct sge_nm_txq *nm_txq;	/* netmap tx queues */
+	struct sge_nm_rxq *nm_rxq;	/* netmap rx queues */
+#endif
 
 	uint16_t iq_start;
 	int eq_start;
 	struct sge_iq **iqmap;	/* iq->cntxt_id to iq mapping */
 	struct sge_eq **eqmap;	/* eq->cntxt_id to eq mapping */
 
 	int pack_boundary;
 	int8_t safe_hwidx1;	/* may not have room for metadata */
 	int8_t safe_hwidx2;	/* with room for metadata and maybe more */
 	struct sw_zone_info sw_zone_info[SW_ZONE_SIZES];
 	struct hw_buf_info hw_buf_info[SGE_FLBUF_SIZES];
 };
 
 struct rss_header;
 typedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *,
     struct mbuf *);
 typedef int (*an_handler_t)(struct sge_iq *, const struct rsp_ctrl *);
 typedef int (*fw_msg_handler_t)(struct adapter *, const __be64 *);
 
 struct adapter {
 	SLIST_ENTRY(adapter) link;
 	device_t dev;
 	struct cdev *cdev;
 
 	/* PCIe register resources */
 	int regs_rid;
 	struct resource *regs_res;
 	int msix_rid;
 	struct resource *msix_res;
 	bus_space_handle_t bh;
 	bus_space_tag_t bt;
 	bus_size_t mmio_len;
 	int udbs_rid;
 	struct resource *udbs_res;
 	volatile uint8_t *udbs_base;
 
 	unsigned int pf;
 	unsigned int mbox;
 
 	/* Interrupt information */
 	int intr_type;
 	int intr_count;
 	struct irq {
 		struct resource *res;
 		int rid;
 		void *tag;
 	} *irq;
 
 	bus_dma_tag_t dmat;	/* Parent DMA tag */
 
 	struct sge sge;
 	int lro_timeout;
 
 	struct taskqueue *tq[NCHAN];	/* taskqueues that flush data out */
 	struct port_info *port[MAX_NPORTS];
 	uint8_t chan_map[NCHAN];
 
 #ifdef TCP_OFFLOAD
 	void *tom_softc;	/* (struct tom_data *) */
 	struct tom_tunables tt;
 	void *iwarp_softc;	/* (struct c4iw_dev *) */
+	void *iscsi_softc;
 #endif
 	struct l2t_data *l2t;	/* L2 table */
 	struct tid_info tids;
 
-	int doorbells;
+	uint16_t doorbells;
 	int open_device_map;
 #ifdef TCP_OFFLOAD
 	int offload_map;
 #endif
 	int flags;
 
 	char ifp_lockname[16];
 	struct mtx ifp_lock;
 	struct ifnet *ifp;	/* tracer ifp */
 	struct ifmedia media;
 	int traceq;		/* iq used by all tracers, -1 if none */
 	int tracer_valid;	/* bitmap of valid tracers */
 	int tracer_enabled;	/* bitmap of enabled tracers */
 
 	char fw_version[32];
 	char cfg_file[32];
 	u_int cfcsum;
 	struct adapter_params params;
 	struct t4_virt_res vres;
 
 	uint16_t linkcaps;
 	uint16_t niccaps;
 	uint16_t toecaps;
 	uint16_t rdmacaps;
 	uint16_t iscsicaps;
 	uint16_t fcoecaps;
 
 	struct sysctl_ctx_list ctx; /* from adapter_full_init to full_uninit */
 
 	struct mtx sc_lock;
 	char lockname[16];
 
 	/* Starving free lists */
 	struct mtx sfl_lock;	/* same cache-line as sc_lock? but that's ok */
 	TAILQ_HEAD(, sge_fl) sfl;
 	struct callout sfl_callout;
 
 	an_handler_t an_handler __aligned(CACHE_LINE_SIZE);
 	fw_msg_handler_t fw_msg_handler[5];	/* NUM_FW6_TYPES */
 	cpl_handler_t cpl_handler[0xef];	/* NUM_CPL_CMDS */
 
 #ifdef INVARIANTS
 	const char *last_op;
 	const void *last_op_thr;
 #endif
 
 	int sc_do_rxcopy;
 };
 
 #define ADAPTER_LOCK(sc)		mtx_lock(&(sc)->sc_lock)
 #define ADAPTER_UNLOCK(sc)		mtx_unlock(&(sc)->sc_lock)
 #define ADAPTER_LOCK_ASSERT_OWNED(sc)	mtx_assert(&(sc)->sc_lock, MA_OWNED)
 #define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) mtx_assert(&(sc)->sc_lock, MA_NOTOWNED)
 
 /* XXX: not bulletproof, but much better than nothing */
 #define ASSERT_SYNCHRONIZED_OP(sc)	\
     KASSERT(IS_BUSY(sc) && \
 	(mtx_owned(&(sc)->sc_lock) || sc->last_op_thr == curthread), \
 	("%s: operation not synchronized.", __func__))
 
 #define PORT_LOCK(pi)			mtx_lock(&(pi)->pi_lock)
 #define PORT_UNLOCK(pi)			mtx_unlock(&(pi)->pi_lock)
 #define PORT_LOCK_ASSERT_OWNED(pi)	mtx_assert(&(pi)->pi_lock, MA_OWNED)
 #define PORT_LOCK_ASSERT_NOTOWNED(pi)	mtx_assert(&(pi)->pi_lock, MA_NOTOWNED)
 
 #define FL_LOCK(fl)			mtx_lock(&(fl)->fl_lock)
 #define FL_TRYLOCK(fl)			mtx_trylock(&(fl)->fl_lock)
 #define FL_UNLOCK(fl)			mtx_unlock(&(fl)->fl_lock)
 #define FL_LOCK_ASSERT_OWNED(fl)	mtx_assert(&(fl)->fl_lock, MA_OWNED)
 #define FL_LOCK_ASSERT_NOTOWNED(fl)	mtx_assert(&(fl)->fl_lock, MA_NOTOWNED)
 
 #define RXQ_FL_LOCK(rxq)		FL_LOCK(&(rxq)->fl)
 #define RXQ_FL_UNLOCK(rxq)		FL_UNLOCK(&(rxq)->fl)
 #define RXQ_FL_LOCK_ASSERT_OWNED(rxq)	FL_LOCK_ASSERT_OWNED(&(rxq)->fl)
 #define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl)
 
 #define EQ_LOCK(eq)			mtx_lock(&(eq)->eq_lock)
 #define EQ_TRYLOCK(eq)			mtx_trylock(&(eq)->eq_lock)
 #define EQ_UNLOCK(eq)			mtx_unlock(&(eq)->eq_lock)
 #define EQ_LOCK_ASSERT_OWNED(eq)	mtx_assert(&(eq)->eq_lock, MA_OWNED)
 #define EQ_LOCK_ASSERT_NOTOWNED(eq)	mtx_assert(&(eq)->eq_lock, MA_NOTOWNED)
 
 #define TXQ_LOCK(txq)			EQ_LOCK(&(txq)->eq)
 #define TXQ_TRYLOCK(txq)		EQ_TRYLOCK(&(txq)->eq)
 #define TXQ_UNLOCK(txq)			EQ_UNLOCK(&(txq)->eq)
 #define TXQ_LOCK_ASSERT_OWNED(txq)	EQ_LOCK_ASSERT_OWNED(&(txq)->eq)
 #define TXQ_LOCK_ASSERT_NOTOWNED(txq)	EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq)
 
 #define for_each_txq(pi, iter, q) \
 	for (q = &pi->adapter->sge.txq[pi->first_txq], iter = 0; \
 	    iter < pi->ntxq; ++iter, ++q)
 #define for_each_rxq(pi, iter, q) \
 	for (q = &pi->adapter->sge.rxq[pi->first_rxq], iter = 0; \
 	    iter < pi->nrxq; ++iter, ++q)
 #define for_each_ofld_txq(pi, iter, q) \
 	for (q = &pi->adapter->sge.ofld_txq[pi->first_ofld_txq], iter = 0; \
 	    iter < pi->nofldtxq; ++iter, ++q)
 #define for_each_ofld_rxq(pi, iter, q) \
 	for (q = &pi->adapter->sge.ofld_rxq[pi->first_ofld_rxq], iter = 0; \
 	    iter < pi->nofldrxq; ++iter, ++q)
+#define for_each_nm_txq(pi, iter, q) \
+	for (q = &pi->adapter->sge.nm_txq[pi->first_nm_txq], iter = 0; \
+	    iter < pi->nnmtxq; ++iter, ++q)
+#define for_each_nm_rxq(pi, iter, q) \
+	for (q = &pi->adapter->sge.nm_rxq[pi->first_nm_rxq], iter = 0; \
+	    iter < pi->nnmrxq; ++iter, ++q)
 
+#define IDXINCR(idx, incr, wrap) do { \
+	idx = wrap - idx > incr ? idx + incr : incr - (wrap - idx); \
+} while (0)
+#define IDXDIFF(head, tail, wrap) \
+	((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head))
+
 /* One for errors, one for firmware events */
 #define T4_EXTRA_INTR 2
 
 static inline uint32_t
 t4_read_reg(struct adapter *sc, uint32_t reg)
 {
 
 	return bus_space_read_4(sc->bt, sc->bh, reg);
 }
 
 static inline void
 t4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val)
 {
 
 	bus_space_write_4(sc->bt, sc->bh, reg, val);
 }
 
 static inline uint64_t
 t4_read_reg64(struct adapter *sc, uint32_t reg)
 {
 
 	return t4_bus_space_read_8(sc->bt, sc->bh, reg);
 }
 
 static inline void
 t4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val)
 {
 
 	t4_bus_space_write_8(sc->bt, sc->bh, reg, val);
 }
 
 static inline void
 t4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val)
 {
 
 	*val = pci_read_config(sc->dev, reg, 1);
 }
 
 static inline void
 t4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val)
 {
 
 	pci_write_config(sc->dev, reg, val, 1);
 }
 
 static inline void
 t4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val)
 {
 
 	*val = pci_read_config(sc->dev, reg, 2);
 }
 
 static inline void
 t4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val)
 {
 
 	pci_write_config(sc->dev, reg, val, 2);
 }
 
 static inline void
 t4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val)
 {
 
 	*val = pci_read_config(sc->dev, reg, 4);
 }
 
 static inline void
 t4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val)
 {
 
 	pci_write_config(sc->dev, reg, val, 4);
 }
 
 static inline struct port_info *
 adap2pinfo(struct adapter *sc, int idx)
 {
 
 	return (sc->port[idx]);
 }
 
 static inline void
 t4_os_set_hw_addr(struct adapter *sc, int idx, uint8_t hw_addr[])
 {
 
 	bcopy(hw_addr, sc->port[idx]->hw_addr, ETHER_ADDR_LEN);
 }
 
 static inline bool
 is_10G_port(const struct port_info *pi)
 {
 
 	return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) != 0);
 }
 
 static inline bool
 is_40G_port(const struct port_info *pi)
 {
 
 	return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G) != 0);
 }
 
 static inline int
 tx_resume_threshold(struct sge_eq *eq)
 {
 
 	return (eq->qsize / 4);
 }
 
 /* t4_main.c */
 void t4_tx_task(void *, int);
 void t4_tx_callout(void *);
 int t4_os_find_pci_capability(struct adapter *, int);
 int t4_os_pci_save_state(struct adapter *);
 int t4_os_pci_restore_state(struct adapter *);
 void t4_os_portmod_changed(const struct adapter *, int);
 void t4_os_link_changed(struct adapter *, int, int, int);
 void t4_iterate(void (*)(struct adapter *, void *), void *);
 int t4_register_cpl_handler(struct adapter *, int, cpl_handler_t);
 int t4_register_an_handler(struct adapter *, an_handler_t);
 int t4_register_fw_msg_handler(struct adapter *, int, fw_msg_handler_t);
 int t4_filter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *);
 int begin_synchronized_op(struct adapter *, struct port_info *, int, char *);
 void end_synchronized_op(struct adapter *, int);
+int update_mac_settings(struct ifnet *, int);
+int adapter_full_init(struct adapter *);
+int adapter_full_uninit(struct adapter *);
+int port_full_init(struct port_info *);
+int port_full_uninit(struct port_info *);
+
+#ifdef DEV_NETMAP
+/* t4_netmap.c */
+int create_netmap_ifnet(struct port_info *);
+int destroy_netmap_ifnet(struct port_info *);
+void t4_nm_intr(void *);
+#endif
 
 /* t4_sge.c */
 void t4_sge_modload(void);
 void t4_sge_modunload(void);
 uint64_t t4_sge_extfree_refs(void);
 void t4_init_sge_cpl_handlers(struct adapter *);
 void t4_tweak_chip_settings(struct adapter *);
 int t4_read_chip_settings(struct adapter *);
 int t4_create_dma_tag(struct adapter *);
 void t4_sge_sysctls(struct adapter *, struct sysctl_ctx_list *,
     struct sysctl_oid_list *);
 int t4_destroy_dma_tag(struct adapter *);
 int t4_setup_adapter_queues(struct adapter *);
 int t4_teardown_adapter_queues(struct adapter *);
 int t4_setup_port_queues(struct port_info *);
 int t4_teardown_port_queues(struct port_info *);
 int t4_alloc_tx_maps(struct tx_maps *, bus_dma_tag_t, int, int);
 void t4_free_tx_maps(struct tx_maps *, bus_dma_tag_t);
 void t4_intr_all(void *);
 void t4_intr(void *);
 void t4_intr_err(void *);
 void t4_intr_evt(void *);
 void t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct wrqe *);
 int t4_eth_tx(struct ifnet *, struct sge_txq *, struct mbuf *);
 void t4_update_fl_bufsize(struct ifnet *);
 int can_resume_tx(struct sge_eq *);
 
 /* t4_tracer.c */
 struct t4_tracer;
 void t4_tracer_modload(void);
 void t4_tracer_modunload(void);
 void t4_tracer_port_detach(struct adapter *);
 int t4_get_tracer(struct adapter *, struct t4_tracer *);
 int t4_set_tracer(struct adapter *, struct t4_tracer *);
 int t4_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *);
 int t5_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *);
 
 static inline struct wrqe *
 alloc_wrqe(int wr_len, struct sge_wrq *wrq)
 {
 	int len = offsetof(struct wrqe, wr) + wr_len;
 	struct wrqe *wr;
 
 	wr = malloc(len, M_CXGBE, M_NOWAIT);
 	if (__predict_false(wr == NULL))
 		return (NULL);
 	wr->wr_len = wr_len;
 	wr->wrq = wrq;
 	return (wr);
 }
 
 static inline void *
 wrtod(struct wrqe *wr)
 {
 	return (&wr->wr[0]);
 }
 
 static inline void
 free_wrqe(struct wrqe *wr)
 {
 	free(wr, M_CXGBE);
 }
 
 static inline void
 t4_wrq_tx(struct adapter *sc, struct wrqe *wr)
 {
 	struct sge_wrq *wrq = wr->wrq;
 
 	TXQ_LOCK(wrq);
 	t4_wrq_tx_locked(sc, wrq, wr);
 	TXQ_UNLOCK(wrq);
 }
 
 #endif
Index: stable/10/sys/dev/cxgbe/common/common.h
===================================================================
--- stable/10/sys/dev/cxgbe/common/common.h	(revision 270296)
+++ stable/10/sys/dev/cxgbe/common/common.h	(revision 270297)
@@ -1,621 +1,623 @@
 /*-
  * Copyright (c) 2011 Chelsio Communications, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $FreeBSD$
  *
  */
 
 #ifndef __CHELSIO_COMMON_H
 #define __CHELSIO_COMMON_H
 
 #include "t4_hw.h"
 
 
 enum {
 	MAX_NPORTS     = 4,     /* max # of ports */
 	SERNUM_LEN     = 24,    /* Serial # length */
 	EC_LEN         = 16,    /* E/C length */
 	ID_LEN         = 16,    /* ID length */
 	PN_LEN         = 16,    /* Part Number length */
 	MACADDR_LEN    = 12,    /* MAC Address length */
 };
 
 enum { MEM_EDC0, MEM_EDC1, MEM_MC, MEM_MC0 = MEM_MC, MEM_MC1 };
 
 enum {
 	MEMWIN0_APERTURE = 2048,
 	MEMWIN0_BASE     = 0x1b800,
 	MEMWIN1_APERTURE = 32768,
 	MEMWIN1_BASE     = 0x28000,
 
 	MEMWIN2_APERTURE_T4 = 65536,
 	MEMWIN2_BASE_T4     = 0x30000,
 
 	MEMWIN2_APERTURE_T5 = 128 * 1024,
 	MEMWIN2_BASE_T5     = 0x60000,
 };
 
 enum dev_master { MASTER_CANT, MASTER_MAY, MASTER_MUST };
 
 enum dev_state { DEV_STATE_UNINIT, DEV_STATE_INIT, DEV_STATE_ERR };
 
 enum {
 	PAUSE_RX      = 1 << 0,
 	PAUSE_TX      = 1 << 1,
 	PAUSE_AUTONEG = 1 << 2
 };
 
 struct memwin {
 	uint32_t base;
 	uint32_t aperture;
 };
 
 struct port_stats {
 	u64 tx_octets;            /* total # of octets in good frames */
 	u64 tx_frames;            /* all good frames */
 	u64 tx_bcast_frames;      /* all broadcast frames */
 	u64 tx_mcast_frames;      /* all multicast frames */
 	u64 tx_ucast_frames;      /* all unicast frames */
 	u64 tx_error_frames;      /* all error frames */
 
 	u64 tx_frames_64;         /* # of Tx frames in a particular range */
 	u64 tx_frames_65_127;
 	u64 tx_frames_128_255;
 	u64 tx_frames_256_511;
 	u64 tx_frames_512_1023;
 	u64 tx_frames_1024_1518;
 	u64 tx_frames_1519_max;
 
 	u64 tx_drop;              /* # of dropped Tx frames */
 	u64 tx_pause;             /* # of transmitted pause frames */
 	u64 tx_ppp0;              /* # of transmitted PPP prio 0 frames */
 	u64 tx_ppp1;              /* # of transmitted PPP prio 1 frames */
 	u64 tx_ppp2;              /* # of transmitted PPP prio 2 frames */
 	u64 tx_ppp3;              /* # of transmitted PPP prio 3 frames */
 	u64 tx_ppp4;              /* # of transmitted PPP prio 4 frames */
 	u64 tx_ppp5;              /* # of transmitted PPP prio 5 frames */
 	u64 tx_ppp6;              /* # of transmitted PPP prio 6 frames */
 	u64 tx_ppp7;              /* # of transmitted PPP prio 7 frames */
 
 	u64 rx_octets;            /* total # of octets in good frames */
 	u64 rx_frames;            /* all good frames */
 	u64 rx_bcast_frames;      /* all broadcast frames */
 	u64 rx_mcast_frames;      /* all multicast frames */
 	u64 rx_ucast_frames;      /* all unicast frames */
 	u64 rx_too_long;          /* # of frames exceeding MTU */
 	u64 rx_jabber;            /* # of jabber frames */
 	u64 rx_fcs_err;           /* # of received frames with bad FCS */
 	u64 rx_len_err;           /* # of received frames with length error */
 	u64 rx_symbol_err;        /* symbol errors */
 	u64 rx_runt;              /* # of short frames */
 
 	u64 rx_frames_64;         /* # of Rx frames in a particular range */
 	u64 rx_frames_65_127;
 	u64 rx_frames_128_255;
 	u64 rx_frames_256_511;
 	u64 rx_frames_512_1023;
 	u64 rx_frames_1024_1518;
 	u64 rx_frames_1519_max;
 
 	u64 rx_pause;             /* # of received pause frames */
 	u64 rx_ppp0;              /* # of received PPP prio 0 frames */
 	u64 rx_ppp1;              /* # of received PPP prio 1 frames */
 	u64 rx_ppp2;              /* # of received PPP prio 2 frames */
 	u64 rx_ppp3;              /* # of received PPP prio 3 frames */
 	u64 rx_ppp4;              /* # of received PPP prio 4 frames */
 	u64 rx_ppp5;              /* # of received PPP prio 5 frames */
 	u64 rx_ppp6;              /* # of received PPP prio 6 frames */
 	u64 rx_ppp7;              /* # of received PPP prio 7 frames */
 
 	u64 rx_ovflow0;           /* drops due to buffer-group 0 overflows */
 	u64 rx_ovflow1;           /* drops due to buffer-group 1 overflows */
 	u64 rx_ovflow2;           /* drops due to buffer-group 2 overflows */
 	u64 rx_ovflow3;           /* drops due to buffer-group 3 overflows */
 	u64 rx_trunc0;            /* buffer-group 0 truncated packets */
 	u64 rx_trunc1;            /* buffer-group 1 truncated packets */
 	u64 rx_trunc2;            /* buffer-group 2 truncated packets */
 	u64 rx_trunc3;            /* buffer-group 3 truncated packets */
 };
 
 struct lb_port_stats {
 	u64 octets;
 	u64 frames;
 	u64 bcast_frames;
 	u64 mcast_frames;
 	u64 ucast_frames;
 	u64 error_frames;
 
 	u64 frames_64;
 	u64 frames_65_127;
 	u64 frames_128_255;
 	u64 frames_256_511;
 	u64 frames_512_1023;
 	u64 frames_1024_1518;
 	u64 frames_1519_max;
 
 	u64 drop;
 
 	u64 ovflow0;
 	u64 ovflow1;
 	u64 ovflow2;
 	u64 ovflow3;
 	u64 trunc0;
 	u64 trunc1;
 	u64 trunc2;
 	u64 trunc3;
 };
 
 struct tp_tcp_stats {
 	u32 tcpOutRsts;
 	u64 tcpInSegs;
 	u64 tcpOutSegs;
 	u64 tcpRetransSegs;
 };
 
 struct tp_usm_stats {
 	u32 frames;
 	u32 drops;
 	u64 octets;
 };
 
 struct tp_fcoe_stats {
 	u32 framesDDP;
 	u32 framesDrop;
 	u64 octetsDDP;
 };
 
 struct tp_err_stats {
 	u32 macInErrs[4];
 	u32 hdrInErrs[4];
 	u32 tcpInErrs[4];
 	u32 tnlCongDrops[4];
 	u32 ofldChanDrops[4];
 	u32 tnlTxDrops[4];
 	u32 ofldVlanDrops[4];
 	u32 tcp6InErrs[4];
 	u32 ofldNoNeigh;
 	u32 ofldCongDefer;
 };
 
 struct tp_proxy_stats {
 	u32 proxy[4];
 };
 
 struct tp_cpl_stats {
 	u32 req[4];
 	u32 rsp[4];
 };
 
 struct tp_rdma_stats {
 	u32 rqe_dfr_mod;
 	u32 rqe_dfr_pkt;
 };
 
 struct tp_params {
 	unsigned int ntxchan;        /* # of Tx channels */
 	unsigned int tre;            /* log2 of core clocks per TP tick */
 	unsigned int dack_re;        /* DACK timer resolution */
 	unsigned int la_mask;        /* what events are recorded by TP LA */
 	unsigned short tx_modq[NCHAN];  /* channel to modulation queue map */
 	uint32_t vlan_pri_map;
 	uint32_t ingress_config;
 	int8_t vlan_shift;
 	int8_t vnic_shift;
 	int8_t port_shift;
 	int8_t protocol_shift;
 };
 
 struct vpd_params {
 	unsigned int cclk;
 	u8 ec[EC_LEN + 1];
 	u8 sn[SERNUM_LEN + 1];
 	u8 id[ID_LEN + 1];
 	u8 pn[PN_LEN + 1];
 	u8 na[MACADDR_LEN + 1];
 };
 
 struct pci_params {
 	unsigned int vpd_cap_addr;
 	unsigned short speed;
 	unsigned short width;
 };
 
 /*
  * Firmware device log.
  */
 struct devlog_params {
 	u32 memtype;			/* which memory (FW_MEMTYPE_* ) */
 	u32 start;			/* start of log in firmware memory */
 	u32 size;			/* size of log */
 };
 
 struct adapter_params {
 	struct tp_params  tp;
 	struct vpd_params vpd;
 	struct pci_params pci;
 	struct devlog_params devlog;
 
 	unsigned int sf_size;             /* serial flash size in bytes */
 	unsigned int sf_nsec;             /* # of flash sectors */
 
 	unsigned int fw_vers;
 	unsigned int tp_vers;
 
 	unsigned short mtus[NMTUS];
 	unsigned short a_wnd[NCCTRL_WIN];
 	unsigned short b_wnd[NCCTRL_WIN];
 
 	u_int ftid_min;
 	u_int ftid_max;
 	u_int etid_min;
 	u_int netids;
 
 	unsigned int cim_la_size;
 
 	uint8_t nports;		/* # of ethernet ports */
 	uint8_t portvec;
 	unsigned int chipid:4;	/* chip ID.  T4 = 4, T5 = 5, ... */
 	unsigned int rev:4;	/* chip revision */
 	unsigned int fpga:1;	/* this is an FPGA */
 	unsigned int offload:1;	/* hw is TOE capable, fw has divvied up card
 				   resources for TOE operation. */
 	unsigned int bypass:1;	/* this is a bypass card */
 	unsigned int ethoffload:1;
 
 	unsigned int ofldq_wr_cred;
 	unsigned int eo_wr_cred;
 };
 
 #define CHELSIO_T4		0x4
 #define CHELSIO_T5		0x5
 
 struct trace_params {
 	u32 data[TRACE_LEN / 4];
 	u32 mask[TRACE_LEN / 4];
 	unsigned short snap_len;
 	unsigned short min_len;
 	unsigned char skip_ofst;
 	unsigned char skip_len;
 	unsigned char invert;
 	unsigned char port;
 };
 
 struct link_config {
 	unsigned short supported;        /* link capabilities */
 	unsigned short advertising;      /* advertised capabilities */
 	unsigned short requested_speed;  /* speed user has requested */
 	unsigned short speed;            /* actual link speed */
 	unsigned char  requested_fc;     /* flow control user has requested */
 	unsigned char  fc;               /* actual link flow control */
 	unsigned char  autoneg;          /* autonegotiating? */
 	unsigned char  link_ok;          /* link up? */
 };
 
 #include "adapter.h"
 
 #ifndef PCI_VENDOR_ID_CHELSIO
 # define PCI_VENDOR_ID_CHELSIO 0x1425
 #endif
 
 #define for_each_port(adapter, iter) \
 	for (iter = 0; iter < (adapter)->params.nports; ++iter)
 
 static inline int is_ftid(const struct adapter *sc, u_int tid)
 {
 
 	return (tid >= sc->params.ftid_min && tid <= sc->params.ftid_max);
 }
 
 static inline int is_etid(const struct adapter *sc, u_int tid)
 {
 
 	return (tid >= sc->params.etid_min);
 }
 
 static inline int is_offload(const struct adapter *adap)
 {
 	return adap->params.offload;
 }
 
 static inline int is_ethoffload(const struct adapter *adap)
 {
 	return adap->params.ethoffload;
 }
 
 static inline int chip_id(struct adapter *adap)
 {
 	return adap->params.chipid;
 }
 
 static inline int chip_rev(struct adapter *adap)
 {
 	return adap->params.rev;
 }
 
 static inline int is_t4(struct adapter *adap)
 {
 	return adap->params.chipid == CHELSIO_T4;
 }
 
 static inline int is_t5(struct adapter *adap)
 {
 	return adap->params.chipid == CHELSIO_T5;
 }
 
 static inline int is_fpga(struct adapter *adap)
 {
 	 return adap->params.fpga;
 }
 
 static inline unsigned int core_ticks_per_usec(const struct adapter *adap)
 {
 	return adap->params.vpd.cclk / 1000;
 }
 
 static inline unsigned int us_to_core_ticks(const struct adapter *adap,
 					    unsigned int us)
 {
 	return (us * adap->params.vpd.cclk) / 1000;
 }
 
 static inline unsigned int dack_ticks_to_usec(const struct adapter *adap,
 					      unsigned int ticks)
 {
 	return (ticks << adap->params.tp.dack_re) / core_ticks_per_usec(adap);
 }
 
 void t4_set_reg_field(struct adapter *adap, unsigned int addr, u32 mask, u32 val);
 int t4_wait_op_done_val(struct adapter *adapter, int reg, u32 mask, int polarity,
 			int attempts, int delay, u32 *valp);
 
 static inline int t4_wait_op_done(struct adapter *adapter, int reg, u32 mask,
 				  int polarity, int attempts, int delay)
 {
 	return t4_wait_op_done_val(adapter, reg, mask, polarity, attempts,
 				   delay, NULL);
 }
 
 int t4_wr_mbox_meat(struct adapter *adap, int mbox, const void *cmd, int size,
 		    void *rpl, bool sleep_ok);
 
 static inline int t4_wr_mbox(struct adapter *adap, int mbox, const void *cmd,
 			     int size, void *rpl)
 {
 	return t4_wr_mbox_meat(adap, mbox, cmd, size, rpl, true);
 }
 
 static inline int t4_wr_mbox_ns(struct adapter *adap, int mbox, const void *cmd,
 				int size, void *rpl)
 {
 	return t4_wr_mbox_meat(adap, mbox, cmd, size, rpl, false);
 }
 
 void t4_read_indirect(struct adapter *adap, unsigned int addr_reg,
 		      unsigned int data_reg, u32 *vals, unsigned int nregs,
 		      unsigned int start_idx);
 void t4_write_indirect(struct adapter *adap, unsigned int addr_reg,
 		       unsigned int data_reg, const u32 *vals,
 		       unsigned int nregs, unsigned int start_idx);
 
 u32 t4_hw_pci_read_cfg4(adapter_t *adapter, int reg);
 
 struct fw_filter_wr;
 
 void t4_intr_enable(struct adapter *adapter);
 void t4_intr_disable(struct adapter *adapter);
 void t4_intr_clear(struct adapter *adapter);
 int t4_slow_intr_handler(struct adapter *adapter);
 
 int t4_hash_mac_addr(const u8 *addr);
 int t4_link_start(struct adapter *adap, unsigned int mbox, unsigned int port,
 		  struct link_config *lc);
 int t4_restart_aneg(struct adapter *adap, unsigned int mbox, unsigned int port);
 int t4_seeprom_read(struct adapter *adapter, u32 addr, u32 *data);
 int t4_seeprom_write(struct adapter *adapter, u32 addr, u32 data);
 int t4_eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz);
 int t4_seeprom_wp(struct adapter *adapter, int enable);
 int t4_read_flash(struct adapter *adapter, unsigned int addr, unsigned int nwords,
 		  u32 *data, int byte_oriented);
 int t4_load_fw(struct adapter *adapter, const u8 *fw_data, unsigned int size);
 int t4_load_boot(struct adapter *adap, u8 *boot_data,
                  unsigned int boot_addr, unsigned int size);
 int t4_flash_cfg_addr(struct adapter *adapter);
 int t4_load_cfg(struct adapter *adapter, const u8 *cfg_data, unsigned int size);
 int t4_get_fw_version(struct adapter *adapter, u32 *vers);
 int t4_get_tp_version(struct adapter *adapter, u32 *vers);
 int t4_check_fw_version(struct adapter *adapter);
 int t4_init_hw(struct adapter *adapter, u32 fw_params);
 int t4_prep_adapter(struct adapter *adapter);
 int t4_init_tp_params(struct adapter *adap);
 int t4_filter_field_shift(const struct adapter *adap, int filter_sel);
 int t4_port_init(struct port_info *p, int mbox, int pf, int vf);
 int t4_reinit_adapter(struct adapter *adap);
 void t4_fatal_err(struct adapter *adapter);
 int t4_set_trace_filter(struct adapter *adapter, const struct trace_params *tp,
 			int filter_index, int enable);
 void t4_get_trace_filter(struct adapter *adapter, struct trace_params *tp,
 			 int filter_index, int *enabled);
 int t4_config_rss_range(struct adapter *adapter, int mbox, unsigned int viid,
 			int start, int n, const u16 *rspq, unsigned int nrspq);
 int t4_config_glbl_rss(struct adapter *adapter, int mbox, unsigned int mode,
 		       unsigned int flags);
 int t4_config_vi_rss(struct adapter *adapter, int mbox, unsigned int viid,
 		     unsigned int flags, unsigned int defq);
 int t4_read_rss(struct adapter *adapter, u16 *entries);
 void t4_read_rss_key(struct adapter *adapter, u32 *key);
 void t4_write_rss_key(struct adapter *adap, const u32 *key, int idx);
 void t4_read_rss_pf_config(struct adapter *adapter, unsigned int index, u32 *valp);
 void t4_write_rss_pf_config(struct adapter *adapter, unsigned int index, u32 val);
 void t4_read_rss_vf_config(struct adapter *adapter, unsigned int index,
 			   u32 *vfl, u32 *vfh);
 void t4_write_rss_vf_config(struct adapter *adapter, unsigned int index,
 			    u32 vfl, u32 vfh);
 u32 t4_read_rss_pf_map(struct adapter *adapter);
 void t4_write_rss_pf_map(struct adapter *adapter, u32 pfmap);
 u32 t4_read_rss_pf_mask(struct adapter *adapter);
 void t4_write_rss_pf_mask(struct adapter *adapter, u32 pfmask);
 int t4_mps_set_active_ports(struct adapter *adap, unsigned int port_mask);
 void t4_pmtx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[]);
 void t4_pmrx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[]);
 void t4_read_cimq_cfg(struct adapter *adap, u16 *base, u16 *size, u16 *thres);
 int t4_read_cim_ibq(struct adapter *adap, unsigned int qid, u32 *data, size_t n);
 int t4_read_cim_obq(struct adapter *adap, unsigned int qid, u32 *data, size_t n);
 int t4_cim_read(struct adapter *adap, unsigned int addr, unsigned int n,
 		unsigned int *valp);
 int t4_cim_write(struct adapter *adap, unsigned int addr, unsigned int n,
 		 const unsigned int *valp);
 int t4_cim_ctl_read(struct adapter *adap, unsigned int addr, unsigned int n,
 		    unsigned int *valp);
 int t4_cim_read_la(struct adapter *adap, u32 *la_buf, unsigned int *wrptr);
 void t4_cim_read_pif_la(struct adapter *adap, u32 *pif_req, u32 *pif_rsp,
 		unsigned int *pif_req_wrptr, unsigned int *pif_rsp_wrptr);
 void t4_cim_read_ma_la(struct adapter *adap, u32 *ma_req, u32 *ma_rsp);
 int t4_mc_read(struct adapter *adap, int idx, u32 addr,
 	       __be32 *data, u64 *parity);
 int t4_edc_read(struct adapter *adap, int idx, u32 addr, __be32 *data, u64 *parity);
 int t4_mem_read(struct adapter *adap, int mtype, u32 addr, u32 size,
 		__be32 *data);
 
 void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p);
 void t4_get_port_stats_offset(struct adapter *adap, int idx,
 		struct port_stats *stats,
 		struct port_stats *offset);
 void t4_get_lb_stats(struct adapter *adap, int idx, struct lb_port_stats *p);
 void t4_clr_port_stats(struct adapter *adap, int idx);
 
 void t4_read_mtu_tbl(struct adapter *adap, u16 *mtus, u8 *mtu_log);
 void t4_read_cong_tbl(struct adapter *adap, u16 incr[NMTUS][NCCTRL_WIN]);
 void t4_read_pace_tbl(struct adapter *adap, unsigned int pace_vals[NTX_SCHED]);
 void t4_get_tx_sched(struct adapter *adap, unsigned int sched, unsigned int *kbps,
 		     unsigned int *ipg);
 void t4_tp_wr_bits_indirect(struct adapter *adap, unsigned int addr,
 			    unsigned int mask, unsigned int val);
 void t4_tp_read_la(struct adapter *adap, u64 *la_buf, unsigned int *wrptr);
 void t4_tp_get_err_stats(struct adapter *adap, struct tp_err_stats *st);
 void t4_tp_get_proxy_stats(struct adapter *adap, struct tp_proxy_stats *st);
 void t4_tp_get_cpl_stats(struct adapter *adap, struct tp_cpl_stats *st);
 void t4_tp_get_rdma_stats(struct adapter *adap, struct tp_rdma_stats *st);
 void t4_get_usm_stats(struct adapter *adap, struct tp_usm_stats *st);
 void t4_tp_get_tcp_stats(struct adapter *adap, struct tp_tcp_stats *v4,
 			 struct tp_tcp_stats *v6);
 void t4_get_fcoe_stats(struct adapter *adap, unsigned int idx,
 		       struct tp_fcoe_stats *st);
 void t4_load_mtus(struct adapter *adap, const unsigned short *mtus,
 		  const unsigned short *alpha, const unsigned short *beta);
 
 void t4_ulprx_read_la(struct adapter *adap, u32 *la_buf);
 
 int t4_set_sched_bps(struct adapter *adap, int sched, unsigned int kbps);
 int t4_set_sched_ipg(struct adapter *adap, int sched, unsigned int ipg);
 int t4_set_pace_tbl(struct adapter *adap, const unsigned int *pace_vals,
 		    unsigned int start, unsigned int n);
 void t4_get_chan_txrate(struct adapter *adap, u64 *nic_rate, u64 *ofld_rate);
 int t4_set_filter_mode(struct adapter *adap, unsigned int mode_map);
 void t4_mk_filtdelwr(unsigned int ftid, struct fw_filter_wr *wr, int qid);
 
 void t4_wol_magic_enable(struct adapter *adap, unsigned int port, const u8 *addr);
 int t4_wol_pat_enable(struct adapter *adap, unsigned int port, unsigned int map,
 		      u64 mask0, u64 mask1, unsigned int crc, bool enable);
 
 int t4_fw_hello(struct adapter *adap, unsigned int mbox, unsigned int evt_mbox,
 		enum dev_master master, enum dev_state *state);
 int t4_fw_bye(struct adapter *adap, unsigned int mbox);
 int t4_fw_reset(struct adapter *adap, unsigned int mbox, int reset);
 int t4_fw_halt(struct adapter *adap, unsigned int mbox, int force);
 int t4_fw_restart(struct adapter *adap, unsigned int mbox, int reset);
 int t4_fw_upgrade(struct adapter *adap, unsigned int mbox,
 		  const u8 *fw_data, unsigned int size, int force);
 int t4_fw_initialize(struct adapter *adap, unsigned int mbox);
 int t4_query_params(struct adapter *adap, unsigned int mbox, unsigned int pf,
 		    unsigned int vf, unsigned int nparams, const u32 *params,
 		    u32 *val);
 int t4_set_params(struct adapter *adap, unsigned int mbox, unsigned int pf,
 		  unsigned int vf, unsigned int nparams, const u32 *params,
 		  const u32 *val);
 int t4_cfg_pfvf(struct adapter *adap, unsigned int mbox, unsigned int pf,
 		unsigned int vf, unsigned int txq, unsigned int txq_eth_ctrl,
 		unsigned int rxqi, unsigned int rxq, unsigned int tc,
 		unsigned int vi, unsigned int cmask, unsigned int pmask,
 		unsigned int exactf, unsigned int rcaps, unsigned int wxcaps);
 int t4_alloc_vi_func(struct adapter *adap, unsigned int mbox,
 		     unsigned int port, unsigned int pf, unsigned int vf,
-		     unsigned int nmac, u8 *mac, unsigned int *rss_size,
+		     unsigned int nmac, u8 *mac, u16 *rss_size,
 		     unsigned int portfunc, unsigned int idstype);
 int t4_alloc_vi(struct adapter *adap, unsigned int mbox, unsigned int port,
 		unsigned int pf, unsigned int vf, unsigned int nmac, u8 *mac,
-		unsigned int *rss_size);
+		u16 *rss_size);
 int t4_free_vi(struct adapter *adap, unsigned int mbox,
 	       unsigned int pf, unsigned int vf,
 	       unsigned int viid);
 int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		  int mtu, int promisc, int all_multi, int bcast, int vlanex,
 		  bool sleep_ok);
 int t4_alloc_mac_filt(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		      bool free, unsigned int naddr, const u8 **addr, u16 *idx,
 		      u64 *hash, bool sleep_ok);
 int t4_change_mac(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		  int idx, const u8 *addr, bool persist, bool add_smt);
 int t4_set_addr_hash(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		     bool ucast, u64 vec, bool sleep_ok);
 int t4_enable_vi(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		 bool rx_en, bool tx_en);
 int t4_identify_port(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		     unsigned int nblinks);
 int t4_mdio_rd(struct adapter *adap, unsigned int mbox, unsigned int phy_addr,
 	       unsigned int mmd, unsigned int reg, unsigned int *valp);
 int t4_mdio_wr(struct adapter *adap, unsigned int mbox, unsigned int phy_addr,
 	       unsigned int mmd, unsigned int reg, unsigned int val);
 int t4_i2c_rd(struct adapter *adap, unsigned int mbox,
 	      int port, unsigned int devid,
 	      unsigned int offset, unsigned int len,
 	      u8 *buf);
 int t4_i2c_wr(struct adapter *adap, unsigned int mbox,
 	      int port, unsigned int devid,
 	      unsigned int offset, unsigned int len,
 	      u8 *buf);
 int t4_iq_start_stop(struct adapter *adap, unsigned int mbox, bool start,
 		     unsigned int pf, unsigned int vf, unsigned int iqid,
 		     unsigned int fl0id, unsigned int fl1id);
 int t4_iq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
 	       unsigned int vf, unsigned int iqtype, unsigned int iqid,
 	       unsigned int fl0id, unsigned int fl1id);
 int t4_eth_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
 		   unsigned int vf, unsigned int eqid);
 int t4_ctrl_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
 		    unsigned int vf, unsigned int eqid);
 int t4_ofld_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
 		    unsigned int vf, unsigned int eqid);
 int t4_sge_ctxt_rd(struct adapter *adap, unsigned int mbox, unsigned int cid,
 		   enum ctxt_type ctype, u32 *data);
 int t4_sge_ctxt_rd_bd(struct adapter *adap, unsigned int cid, enum ctxt_type ctype,
 		      u32 *data);
 int t4_sge_ctxt_flush(struct adapter *adap, unsigned int mbox);
 int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl);
 int t4_fwaddrspace_write(struct adapter *adap, unsigned int mbox, u32 addr, u32 val);
-int t4_sched_config(struct adapter *adapter, int type, int minmaxen);
+int t4_sched_config(struct adapter *adapter, int type, int minmaxen,
+		    int sleep_ok);
 int t4_sched_params(struct adapter *adapter, int type, int level, int mode,
 		    int rateunit, int ratemode, int channel, int cl,
-		    int minrate, int maxrate, int weight, int pktsize);
+		    int minrate, int maxrate, int weight, int pktsize,
+		    int sleep_ok);
 #endif /* __CHELSIO_COMMON_H */
Index: stable/10/sys/dev/cxgbe/common/t4_hw.c
===================================================================
--- stable/10/sys/dev/cxgbe/common/t4_hw.c	(revision 270296)
+++ stable/10/sys/dev/cxgbe/common/t4_hw.c	(revision 270297)
@@ -1,5762 +1,5779 @@
 /*-
  * Copyright (c) 2012 Chelsio Communications, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
 
 #include "common.h"
 #include "t4_regs.h"
 #include "t4_regs_values.h"
 #include "firmware/t4fw_interface.h"
 
 #undef msleep
 #define msleep(x) do { \
 	if (cold) \
 		DELAY((x) * 1000); \
 	else \
 		pause("t4hw", (x) * hz / 1000); \
 } while (0)
 
 /**
  *	t4_wait_op_done_val - wait until an operation is completed
  *	@adapter: the adapter performing the operation
  *	@reg: the register to check for completion
  *	@mask: a single-bit field within @reg that indicates completion
  *	@polarity: the value of the field when the operation is completed
  *	@attempts: number of check iterations
  *	@delay: delay in usecs between iterations
  *	@valp: where to store the value of the register at completion time
  *
  *	Wait until an operation is completed by checking a bit in a register
  *	up to @attempts times.  If @valp is not NULL the value of the register
  *	at the time it indicated completion is stored there.  Returns 0 if the
  *	operation completes and	-EAGAIN	otherwise.
  */
 int t4_wait_op_done_val(struct adapter *adapter, int reg, u32 mask,
 		        int polarity, int attempts, int delay, u32 *valp)
 {
 	while (1) {
 		u32 val = t4_read_reg(adapter, reg);
 
 		if (!!(val & mask) == polarity) {
 			if (valp)
 				*valp = val;
 			return 0;
 		}
 		if (--attempts == 0)
 			return -EAGAIN;
 		if (delay)
 			udelay(delay);
 	}
 }
 
 /**
  *	t4_set_reg_field - set a register field to a value
  *	@adapter: the adapter to program
  *	@addr: the register address
  *	@mask: specifies the portion of the register to modify
  *	@val: the new value for the register field
  *
  *	Sets a register field specified by the supplied mask to the
  *	given value.
  */
 void t4_set_reg_field(struct adapter *adapter, unsigned int addr, u32 mask,
 		      u32 val)
 {
 	u32 v = t4_read_reg(adapter, addr) & ~mask;
 
 	t4_write_reg(adapter, addr, v | val);
 	(void) t4_read_reg(adapter, addr);      /* flush */
 }
 
 /**
  *	t4_read_indirect - read indirectly addressed registers
  *	@adap: the adapter
  *	@addr_reg: register holding the indirect address
  *	@data_reg: register holding the value of the indirect register
  *	@vals: where the read register values are stored
  *	@nregs: how many indirect registers to read
  *	@start_idx: index of first indirect register to read
  *
  *	Reads registers that are accessed indirectly through an address/data
  *	register pair.
  */
 void t4_read_indirect(struct adapter *adap, unsigned int addr_reg,
 		      unsigned int data_reg, u32 *vals, unsigned int nregs,
 		      unsigned int start_idx)
 {
 	while (nregs--) {
 		t4_write_reg(adap, addr_reg, start_idx);
 		*vals++ = t4_read_reg(adap, data_reg);
 		start_idx++;
 	}
 }
 
 /**
  *	t4_write_indirect - write indirectly addressed registers
  *	@adap: the adapter
  *	@addr_reg: register holding the indirect addresses
  *	@data_reg: register holding the value for the indirect registers
  *	@vals: values to write
  *	@nregs: how many indirect registers to write
  *	@start_idx: address of first indirect register to write
  *
  *	Writes a sequential block of registers that are accessed indirectly
  *	through an address/data register pair.
  */
 void t4_write_indirect(struct adapter *adap, unsigned int addr_reg,
 		       unsigned int data_reg, const u32 *vals,
 		       unsigned int nregs, unsigned int start_idx)
 {
 	while (nregs--) {
 		t4_write_reg(adap, addr_reg, start_idx++);
 		t4_write_reg(adap, data_reg, *vals++);
 	}
 }
 
 /*
  * Read a 32-bit PCI Configuration Space register via the PCI-E backdoor
  * mechanism.  This guarantees that we get the real value even if we're
  * operating within a Virtual Machine and the Hypervisor is trapping our
  * Configuration Space accesses.
  */
 u32 t4_hw_pci_read_cfg4(adapter_t *adap, int reg)
 {
 	t4_write_reg(adap, A_PCIE_CFG_SPACE_REQ,
 		     F_ENABLE | F_LOCALCFG | V_FUNCTION(adap->pf) |
 		     V_REGISTER(reg));
 	return t4_read_reg(adap, A_PCIE_CFG_SPACE_DATA);
 }
 
 /*
  *	t4_report_fw_error - report firmware error
  *	@adap: the adapter
  *
  *	The adapter firmware can indicate error conditions to the host.
  *	This routine prints out the reason for the firmware error (as
  *	reported by the firmware).
  */
 static void t4_report_fw_error(struct adapter *adap)
 {
 	static const char *reason[] = {
 		"Crash",			/* PCIE_FW_EVAL_CRASH */
 		"During Device Preparation",	/* PCIE_FW_EVAL_PREP */
 		"During Device Configuration",	/* PCIE_FW_EVAL_CONF */
 		"During Device Initialization",	/* PCIE_FW_EVAL_INIT */
 		"Unexpected Event",		/* PCIE_FW_EVAL_UNEXPECTEDEVENT */
 		"Insufficient Airflow",		/* PCIE_FW_EVAL_OVERHEAT */
 		"Device Shutdown",		/* PCIE_FW_EVAL_DEVICESHUTDOWN */
 		"Reserved",			/* reserved */
 	};
 	u32 pcie_fw;
 
 	pcie_fw = t4_read_reg(adap, A_PCIE_FW);
 	if (pcie_fw & F_PCIE_FW_ERR)
 		CH_ERR(adap, "Firmware reports adapter error: %s\n",
 		       reason[G_PCIE_FW_EVAL(pcie_fw)]);
 }
 
 /*
  * Get the reply to a mailbox command and store it in @rpl in big-endian order.
  */
 static void get_mbox_rpl(struct adapter *adap, __be64 *rpl, int nflit,
 			 u32 mbox_addr)
 {
 	for ( ; nflit; nflit--, mbox_addr += 8)
 		*rpl++ = cpu_to_be64(t4_read_reg64(adap, mbox_addr));
 }
 
 /*
  * Handle a FW assertion reported in a mailbox.
  */
 static void fw_asrt(struct adapter *adap, u32 mbox_addr)
 {
 	struct fw_debug_cmd asrt;
 
 	get_mbox_rpl(adap, (__be64 *)&asrt, sizeof(asrt) / 8, mbox_addr);
 	CH_ALERT(adap, "FW assertion at %.16s:%u, val0 %#x, val1 %#x\n",
 		 asrt.u.assert.filename_0_7, ntohl(asrt.u.assert.line),
 		 ntohl(asrt.u.assert.x), ntohl(asrt.u.assert.y));
 }
 
 #define X_CIM_PF_NOACCESS 0xeeeeeeee
 /**
  *	t4_wr_mbox_meat - send a command to FW through the given mailbox
  *	@adap: the adapter
  *	@mbox: index of the mailbox to use
  *	@cmd: the command to write
  *	@size: command length in bytes
  *	@rpl: where to optionally store the reply
  *	@sleep_ok: if true we may sleep while awaiting command completion
  *
  *	Sends the given command to FW through the selected mailbox and waits
  *	for the FW to execute the command.  If @rpl is not %NULL it is used to
  *	store the FW's reply to the command.  The command and its optional
  *	reply are of the same length.  Some FW commands like RESET and
  *	INITIALIZE can take a considerable amount of time to execute.
  *	@sleep_ok determines whether we may sleep while awaiting the response.
  *	If sleeping is allowed we use progressive backoff otherwise we spin.
  *
  *	The return value is 0 on success or a negative errno on failure.  A
  *	failure can happen either because we are not able to execute the
  *	command or FW executes it but signals an error.  In the latter case
  *	the return value is the error code indicated by FW (negated).
  */
 int t4_wr_mbox_meat(struct adapter *adap, int mbox, const void *cmd, int size,
 		    void *rpl, bool sleep_ok)
 {
 	/*
 	 * We delay in small increments at first in an effort to maintain
 	 * responsiveness for simple, fast executing commands but then back
 	 * off to larger delays to a maximum retry delay.
 	 */
 	static const int delay[] = {
 		1, 1, 3, 5, 10, 10, 20, 50, 100
 	};
 
 	u32 v;
 	u64 res;
 	int i, ms, delay_idx;
 	const __be64 *p = cmd;
 	u32 data_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_DATA);
 	u32 ctl_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_CTRL);
 
 	if ((size & 15) || size > MBOX_LEN)
 		return -EINVAL;
 
 	v = G_MBOWNER(t4_read_reg(adap, ctl_reg));
 	for (i = 0; v == X_MBOWNER_NONE && i < 3; i++)
 		v = G_MBOWNER(t4_read_reg(adap, ctl_reg));
 
 	if (v != X_MBOWNER_PL)
 		return v ? -EBUSY : -ETIMEDOUT;
 
 	for (i = 0; i < size; i += 8, p++)
 		t4_write_reg64(adap, data_reg + i, be64_to_cpu(*p));
 
 	t4_write_reg(adap, ctl_reg, F_MBMSGVALID | V_MBOWNER(X_MBOWNER_FW));
 	t4_read_reg(adap, ctl_reg);          /* flush write */
 
 	delay_idx = 0;
 	ms = delay[0];
 
 	for (i = 0; i < FW_CMD_MAX_TIMEOUT; i += ms) {
 		if (sleep_ok) {
 			ms = delay[delay_idx];  /* last element may repeat */
 			if (delay_idx < ARRAY_SIZE(delay) - 1)
 				delay_idx++;
 			msleep(ms);
 		} else
 			mdelay(ms);
 
 		v = t4_read_reg(adap, ctl_reg);
 		if (v == X_CIM_PF_NOACCESS)
 			continue;
 		if (G_MBOWNER(v) == X_MBOWNER_PL) {
 			if (!(v & F_MBMSGVALID)) {
 				t4_write_reg(adap, ctl_reg,
 					     V_MBOWNER(X_MBOWNER_NONE));
 				continue;
 			}
 
 			res = t4_read_reg64(adap, data_reg);
 			if (G_FW_CMD_OP(res >> 32) == FW_DEBUG_CMD) {
 				fw_asrt(adap, data_reg);
 				res = V_FW_CMD_RETVAL(EIO);
 			} else if (rpl)
 				get_mbox_rpl(adap, rpl, size / 8, data_reg);
 			t4_write_reg(adap, ctl_reg, V_MBOWNER(X_MBOWNER_NONE));
 			return -G_FW_CMD_RETVAL((int)res);
 		}
 	}
 
 	/*
 	 * We timed out waiting for a reply to our mailbox command.  Report
 	 * the error and also check to see if the firmware reported any
 	 * errors ...
 	 */
 	CH_ERR(adap, "command %#x in mailbox %d timed out\n",
 	       *(const u8 *)cmd, mbox);
 	if (t4_read_reg(adap, A_PCIE_FW) & F_PCIE_FW_ERR)
 		t4_report_fw_error(adap);
 	return -ETIMEDOUT;
 }
 
 /**
  *	t4_mc_read - read from MC through backdoor accesses
  *	@adap: the adapter
  *	@idx: which MC to access
  *	@addr: address of first byte requested
  *	@data: 64 bytes of data containing the requested address
  *	@ecc: where to store the corresponding 64-bit ECC word
  *
  *	Read 64 bytes of data from MC starting at a 64-byte-aligned address
  *	that covers the requested address @addr.  If @parity is not %NULL it
  *	is assigned the 64-bit ECC word for the read data.
  */
 int t4_mc_read(struct adapter *adap, int idx, u32 addr, __be32 *data, u64 *ecc)
 {
 	int i;
 	u32 mc_bist_cmd_reg, mc_bist_cmd_addr_reg, mc_bist_cmd_len_reg;
 	u32 mc_bist_status_rdata_reg, mc_bist_data_pattern_reg;
 
 	if (is_t4(adap)) {
 		mc_bist_cmd_reg = A_MC_BIST_CMD;
 		mc_bist_cmd_addr_reg = A_MC_BIST_CMD_ADDR;
 		mc_bist_cmd_len_reg = A_MC_BIST_CMD_LEN;
 		mc_bist_status_rdata_reg = A_MC_BIST_STATUS_RDATA;
 		mc_bist_data_pattern_reg = A_MC_BIST_DATA_PATTERN;
 	} else {
 		mc_bist_cmd_reg = MC_REG(A_MC_P_BIST_CMD, idx);
 		mc_bist_cmd_addr_reg = MC_REG(A_MC_P_BIST_CMD_ADDR, idx);
 		mc_bist_cmd_len_reg = MC_REG(A_MC_P_BIST_CMD_LEN, idx);
 		mc_bist_status_rdata_reg = MC_REG(A_MC_P_BIST_STATUS_RDATA,
 						  idx);
 		mc_bist_data_pattern_reg = MC_REG(A_MC_P_BIST_DATA_PATTERN,
 						  idx);
 	}
 
 	if (t4_read_reg(adap, mc_bist_cmd_reg) & F_START_BIST)
 		return -EBUSY;
 	t4_write_reg(adap, mc_bist_cmd_addr_reg, addr & ~0x3fU);
 	t4_write_reg(adap, mc_bist_cmd_len_reg, 64);
 	t4_write_reg(adap, mc_bist_data_pattern_reg, 0xc);
 	t4_write_reg(adap, mc_bist_cmd_reg, V_BIST_OPCODE(1) |
 		     F_START_BIST | V_BIST_CMD_GAP(1));
 	i = t4_wait_op_done(adap, mc_bist_cmd_reg, F_START_BIST, 0, 10, 1);
 	if (i)
 		return i;
 
 #define MC_DATA(i) MC_BIST_STATUS_REG(mc_bist_status_rdata_reg, i)
 
 	for (i = 15; i >= 0; i--)
 		*data++ = ntohl(t4_read_reg(adap, MC_DATA(i)));
 	if (ecc)
 		*ecc = t4_read_reg64(adap, MC_DATA(16));
 #undef MC_DATA
 	return 0;
 }
 
 /**
  *	t4_edc_read - read from EDC through backdoor accesses
  *	@adap: the adapter
  *	@idx: which EDC to access
  *	@addr: address of first byte requested
  *	@data: 64 bytes of data containing the requested address
  *	@ecc: where to store the corresponding 64-bit ECC word
  *
  *	Read 64 bytes of data from EDC starting at a 64-byte-aligned address
  *	that covers the requested address @addr.  If @parity is not %NULL it
  *	is assigned the 64-bit ECC word for the read data.
  */
 int t4_edc_read(struct adapter *adap, int idx, u32 addr, __be32 *data, u64 *ecc)
 {
 	int i;
 	u32 edc_bist_cmd_reg, edc_bist_cmd_addr_reg, edc_bist_cmd_len_reg;
 	u32 edc_bist_cmd_data_pattern, edc_bist_status_rdata_reg;
 
 	if (is_t4(adap)) {
 		edc_bist_cmd_reg = EDC_REG(A_EDC_BIST_CMD, idx);
 		edc_bist_cmd_addr_reg = EDC_REG(A_EDC_BIST_CMD_ADDR, idx);
 		edc_bist_cmd_len_reg = EDC_REG(A_EDC_BIST_CMD_LEN, idx);
 		edc_bist_cmd_data_pattern = EDC_REG(A_EDC_BIST_DATA_PATTERN,
 						    idx);
 		edc_bist_status_rdata_reg = EDC_REG(A_EDC_BIST_STATUS_RDATA,
 						    idx);
 	} else {
 /*
  * These macro are missing in t4_regs.h file.
  * Added temporarily for testing.
  */
 #define EDC_STRIDE_T5 (EDC_T51_BASE_ADDR - EDC_T50_BASE_ADDR)
 #define EDC_REG_T5(reg, idx) (reg + EDC_STRIDE_T5 * idx)
 		edc_bist_cmd_reg = EDC_REG_T5(A_EDC_H_BIST_CMD, idx);
 		edc_bist_cmd_addr_reg = EDC_REG_T5(A_EDC_H_BIST_CMD_ADDR, idx);
 		edc_bist_cmd_len_reg = EDC_REG_T5(A_EDC_H_BIST_CMD_LEN, idx);
 		edc_bist_cmd_data_pattern = EDC_REG_T5(A_EDC_H_BIST_DATA_PATTERN,
 						    idx);
 		edc_bist_status_rdata_reg = EDC_REG_T5(A_EDC_H_BIST_STATUS_RDATA,
 						    idx);
 #undef EDC_REG_T5
 #undef EDC_STRIDE_T5
 	}
 
 	if (t4_read_reg(adap, edc_bist_cmd_reg) & F_START_BIST)
 		return -EBUSY;
 	t4_write_reg(adap, edc_bist_cmd_addr_reg, addr & ~0x3fU);
 	t4_write_reg(adap, edc_bist_cmd_len_reg, 64);
 	t4_write_reg(adap, edc_bist_cmd_data_pattern, 0xc);
 	t4_write_reg(adap, edc_bist_cmd_reg,
 		     V_BIST_OPCODE(1) | V_BIST_CMD_GAP(1) | F_START_BIST);
 	i = t4_wait_op_done(adap, edc_bist_cmd_reg, F_START_BIST, 0, 10, 1);
 	if (i)
 		return i;
 
 #define EDC_DATA(i) EDC_BIST_STATUS_REG(edc_bist_status_rdata_reg, i)
 
 	for (i = 15; i >= 0; i--)
 		*data++ = ntohl(t4_read_reg(adap, EDC_DATA(i)));
 	if (ecc)
 		*ecc = t4_read_reg64(adap, EDC_DATA(16));
 #undef EDC_DATA
 	return 0;
 }
 
 /**
  *	t4_mem_read - read EDC 0, EDC 1 or MC into buffer
  *	@adap: the adapter
  *	@mtype: memory type: MEM_EDC0, MEM_EDC1 or MEM_MC
  *	@addr: address within indicated memory type
  *	@len: amount of memory to read
  *	@buf: host memory buffer
  *
  *	Reads an [almost] arbitrary memory region in the firmware: the
  *	firmware memory address, length and host buffer must be aligned on
  *	32-bit boudaries.  The memory is returned as a raw byte sequence from
  *	the firmware's memory.  If this memory contains data structures which
  *	contain multi-byte integers, it's the callers responsibility to
  *	perform appropriate byte order conversions.
  */
 int t4_mem_read(struct adapter *adap, int mtype, u32 addr, u32 len,
 		__be32 *buf)
 {
 	u32 pos, start, end, offset;
 	int ret;
 
 	/*
 	 * Argument sanity checks ...
 	 */
 	if ((addr & 0x3) || (len & 0x3))
 		return -EINVAL;
 
 	/*
 	 * The underlaying EDC/MC read routines read 64 bytes at a time so we
 	 * need to round down the start and round up the end.  We'll start
 	 * copying out of the first line at (addr - start) a word at a time.
 	 */
 	start = addr & ~(64-1);
 	end = (addr + len + 64-1) & ~(64-1);
 	offset = (addr - start)/sizeof(__be32);
 
 	for (pos = start; pos < end; pos += 64, offset = 0) {
 		__be32 data[16];
 
 		/*
 		 * Read the chip's memory block and bail if there's an error.
 		 */
 		if ((mtype == MEM_MC) || (mtype == MEM_MC1))
 			ret = t4_mc_read(adap, mtype - MEM_MC, pos, data, NULL);
 		else
 			ret = t4_edc_read(adap, mtype, pos, data, NULL);
 		if (ret)
 			return ret;
 
 		/*
 		 * Copy the data into the caller's memory buffer.
 		 */
 		while (offset < 16 && len > 0) {
 			*buf++ = data[offset++];
 			len -= sizeof(__be32);
 		}
 	}
 
 	return 0;
 }
 
 /*
  * Partial EEPROM Vital Product Data structure.  Includes only the ID and
  * VPD-R header.
  */
 struct t4_vpd_hdr {
 	u8  id_tag;
 	u8  id_len[2];
 	u8  id_data[ID_LEN];
 	u8  vpdr_tag;
 	u8  vpdr_len[2];
 };
 
 /*
  * EEPROM reads take a few tens of us while writes can take a bit over 5 ms.
  */
 #define EEPROM_MAX_RD_POLL 40
 #define EEPROM_MAX_WR_POLL 6
 #define EEPROM_STAT_ADDR   0x7bfc
 #define VPD_BASE           0x400
 #define VPD_BASE_OLD       0
 #define VPD_LEN            1024
 #define VPD_INFO_FLD_HDR_SIZE	3
 #define CHELSIO_VPD_UNIQUE_ID 0x82
 
 /**
  *	t4_seeprom_read - read a serial EEPROM location
  *	@adapter: adapter to read
  *	@addr: EEPROM virtual address
  *	@data: where to store the read data
  *
  *	Read a 32-bit word from a location in serial EEPROM using the card's PCI
  *	VPD capability.  Note that this function must be called with a virtual
  *	address.
  */
 int t4_seeprom_read(struct adapter *adapter, u32 addr, u32 *data)
 {
 	u16 val;
 	int attempts = EEPROM_MAX_RD_POLL;
 	unsigned int base = adapter->params.pci.vpd_cap_addr;
 
 	if (addr >= EEPROMVSIZE || (addr & 3))
 		return -EINVAL;
 
 	t4_os_pci_write_cfg2(adapter, base + PCI_VPD_ADDR, (u16)addr);
 	do {
 		udelay(10);
 		t4_os_pci_read_cfg2(adapter, base + PCI_VPD_ADDR, &val);
 	} while (!(val & PCI_VPD_ADDR_F) && --attempts);
 
 	if (!(val & PCI_VPD_ADDR_F)) {
 		CH_ERR(adapter, "reading EEPROM address 0x%x failed\n", addr);
 		return -EIO;
 	}
 	t4_os_pci_read_cfg4(adapter, base + PCI_VPD_DATA, data);
 	*data = le32_to_cpu(*data);
 	return 0;
 }
 
 /**
  *	t4_seeprom_write - write a serial EEPROM location
  *	@adapter: adapter to write
  *	@addr: virtual EEPROM address
  *	@data: value to write
  *
  *	Write a 32-bit word to a location in serial EEPROM using the card's PCI
  *	VPD capability.  Note that this function must be called with a virtual
  *	address.
  */
 int t4_seeprom_write(struct adapter *adapter, u32 addr, u32 data)
 {
 	u16 val;
 	int attempts = EEPROM_MAX_WR_POLL;
 	unsigned int base = adapter->params.pci.vpd_cap_addr;
 
 	if (addr >= EEPROMVSIZE || (addr & 3))
 		return -EINVAL;
 
 	t4_os_pci_write_cfg4(adapter, base + PCI_VPD_DATA,
 				 cpu_to_le32(data));
 	t4_os_pci_write_cfg2(adapter, base + PCI_VPD_ADDR,
 				 (u16)addr | PCI_VPD_ADDR_F);
 	do {
 		msleep(1);
 		t4_os_pci_read_cfg2(adapter, base + PCI_VPD_ADDR, &val);
 	} while ((val & PCI_VPD_ADDR_F) && --attempts);
 
 	if (val & PCI_VPD_ADDR_F) {
 		CH_ERR(adapter, "write to EEPROM address 0x%x failed\n", addr);
 		return -EIO;
 	}
 	return 0;
 }
 
 /**
  *	t4_eeprom_ptov - translate a physical EEPROM address to virtual
  *	@phys_addr: the physical EEPROM address
  *	@fn: the PCI function number
  *	@sz: size of function-specific area
  *
  *	Translate a physical EEPROM address to virtual.  The first 1K is
  *	accessed through virtual addresses starting at 31K, the rest is
  *	accessed through virtual addresses starting at 0.
  *
  *	The mapping is as follows:
  *	[0..1K) -> [31K..32K)
  *	[1K..1K+A) -> [ES-A..ES)
  *	[1K+A..ES) -> [0..ES-A-1K)
  *
  *	where A = @fn * @sz, and ES = EEPROM size.
  */
 int t4_eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
 {
 	fn *= sz;
 	if (phys_addr < 1024)
 		return phys_addr + (31 << 10);
 	if (phys_addr < 1024 + fn)
 		return EEPROMSIZE - fn + phys_addr - 1024;
 	if (phys_addr < EEPROMSIZE)
 		return phys_addr - 1024 - fn;
 	return -EINVAL;
 }
 
 /**
  *	t4_seeprom_wp - enable/disable EEPROM write protection
  *	@adapter: the adapter
  *	@enable: whether to enable or disable write protection
  *
  *	Enables or disables write protection on the serial EEPROM.
  */
 int t4_seeprom_wp(struct adapter *adapter, int enable)
 {
 	return t4_seeprom_write(adapter, EEPROM_STAT_ADDR, enable ? 0xc : 0);
 }
 
 /**
  *	get_vpd_keyword_val - Locates an information field keyword in the VPD
  *	@v: Pointer to buffered vpd data structure
  *	@kw: The keyword to search for
  *	
  *	Returns the value of the information field keyword or
  *	-ENOENT otherwise.
  */
 static int get_vpd_keyword_val(const struct t4_vpd_hdr *v, const char *kw)
 {
          int i;
 	 unsigned int offset , len;
 	 const u8 *buf = &v->id_tag;
 	 const u8 *vpdr_len = &v->vpdr_tag; 
 	 offset = sizeof(struct t4_vpd_hdr);
 	 len =  (u16)vpdr_len[1] + ((u16)vpdr_len[2] << 8);
 	 
 	 if (len + sizeof(struct t4_vpd_hdr) > VPD_LEN) {
 		 return -ENOENT;
 	 }
 
          for (i = offset; i + VPD_INFO_FLD_HDR_SIZE <= offset + len;) {
 		 if(memcmp(buf + i , kw , 2) == 0){
 			 i += VPD_INFO_FLD_HDR_SIZE;
                          return i;
 		  }
 
                  i += VPD_INFO_FLD_HDR_SIZE + buf[i+2];
          }
 
          return -ENOENT;
 }
 
 
 /**
  *	get_vpd_params - read VPD parameters from VPD EEPROM
  *	@adapter: adapter to read
  *	@p: where to store the parameters
  *
  *	Reads card parameters stored in VPD EEPROM.
  */
 static int get_vpd_params(struct adapter *adapter, struct vpd_params *p)
 {
 	int i, ret, addr;
 	int ec, sn, pn, na;
 	u8 vpd[VPD_LEN], csum;
 	const struct t4_vpd_hdr *v;
 
 	/*
 	 * Card information normally starts at VPD_BASE but early cards had
 	 * it at 0.
 	 */
 	ret = t4_seeprom_read(adapter, VPD_BASE, (u32 *)(vpd));
 	addr = *vpd == CHELSIO_VPD_UNIQUE_ID ? VPD_BASE : VPD_BASE_OLD;
 
 	for (i = 0; i < sizeof(vpd); i += 4) {
 		ret = t4_seeprom_read(adapter, addr + i, (u32 *)(vpd + i));
 		if (ret)
 			return ret;
 	}
  	v = (const struct t4_vpd_hdr *)vpd;
 	
 #define FIND_VPD_KW(var,name) do { \
 	var = get_vpd_keyword_val(v , name); \
 	if (var < 0) { \
 		CH_ERR(adapter, "missing VPD keyword " name "\n"); \
 		return -EINVAL; \
 	} \
 } while (0)	
 
 	FIND_VPD_KW(i, "RV");
 	for (csum = 0; i >= 0; i--)
 		csum += vpd[i];
 
 	if (csum) {
 		CH_ERR(adapter, "corrupted VPD EEPROM, actual csum %u\n", csum);
 		return -EINVAL;
 	}
 	FIND_VPD_KW(ec, "EC");
 	FIND_VPD_KW(sn, "SN");
 	FIND_VPD_KW(pn, "PN");
 	FIND_VPD_KW(na, "NA");
 #undef FIND_VPD_KW
 
 	memcpy(p->id, v->id_data, ID_LEN);
 	strstrip(p->id);
 	memcpy(p->ec, vpd + ec, EC_LEN);
 	strstrip(p->ec);
 	i = vpd[sn - VPD_INFO_FLD_HDR_SIZE + 2];
 	memcpy(p->sn, vpd + sn, min(i, SERNUM_LEN));
 	strstrip(p->sn);
 	i = vpd[pn - VPD_INFO_FLD_HDR_SIZE + 2];
 	memcpy(p->pn, vpd + pn, min(i, PN_LEN));
 	strstrip((char *)p->pn);
 	i = vpd[na - VPD_INFO_FLD_HDR_SIZE + 2];
 	memcpy(p->na, vpd + na, min(i, MACADDR_LEN));
 	strstrip((char *)p->na);
 
 	return 0;
 }
 
 /* serial flash and firmware constants and flash config file constants */
 enum {
 	SF_ATTEMPTS = 10,             /* max retries for SF operations */
 
 	/* flash command opcodes */
 	SF_PROG_PAGE    = 2,          /* program page */
 	SF_WR_DISABLE   = 4,          /* disable writes */
 	SF_RD_STATUS    = 5,          /* read status register */
 	SF_WR_ENABLE    = 6,          /* enable writes */
 	SF_RD_DATA_FAST = 0xb,        /* read flash */
 	SF_RD_ID        = 0x9f,       /* read ID */
 	SF_ERASE_SECTOR = 0xd8,       /* erase sector */
 };
 
 /**
  *	sf1_read - read data from the serial flash
  *	@adapter: the adapter
  *	@byte_cnt: number of bytes to read
  *	@cont: whether another operation will be chained
  *	@lock: whether to lock SF for PL access only
  *	@valp: where to store the read data
  *
  *	Reads up to 4 bytes of data from the serial flash.  The location of
  *	the read needs to be specified prior to calling this by issuing the
  *	appropriate commands to the serial flash.
  */
 static int sf1_read(struct adapter *adapter, unsigned int byte_cnt, int cont,
 		    int lock, u32 *valp)
 {
 	int ret;
 
 	if (!byte_cnt || byte_cnt > 4)
 		return -EINVAL;
 	if (t4_read_reg(adapter, A_SF_OP) & F_BUSY)
 		return -EBUSY;
 	t4_write_reg(adapter, A_SF_OP,
 		     V_SF_LOCK(lock) | V_CONT(cont) | V_BYTECNT(byte_cnt - 1));
 	ret = t4_wait_op_done(adapter, A_SF_OP, F_BUSY, 0, SF_ATTEMPTS, 5);
 	if (!ret)
 		*valp = t4_read_reg(adapter, A_SF_DATA);
 	return ret;
 }
 
 /**
  *	sf1_write - write data to the serial flash
  *	@adapter: the adapter
  *	@byte_cnt: number of bytes to write
  *	@cont: whether another operation will be chained
  *	@lock: whether to lock SF for PL access only
  *	@val: value to write
  *
  *	Writes up to 4 bytes of data to the serial flash.  The location of
  *	the write needs to be specified prior to calling this by issuing the
  *	appropriate commands to the serial flash.
  */
 static int sf1_write(struct adapter *adapter, unsigned int byte_cnt, int cont,
 		     int lock, u32 val)
 {
 	if (!byte_cnt || byte_cnt > 4)
 		return -EINVAL;
 	if (t4_read_reg(adapter, A_SF_OP) & F_BUSY)
 		return -EBUSY;
 	t4_write_reg(adapter, A_SF_DATA, val);
 	t4_write_reg(adapter, A_SF_OP, V_SF_LOCK(lock) |
 		     V_CONT(cont) | V_BYTECNT(byte_cnt - 1) | V_OP(1));
 	return t4_wait_op_done(adapter, A_SF_OP, F_BUSY, 0, SF_ATTEMPTS, 5);
 }
 
 /**
  *	flash_wait_op - wait for a flash operation to complete
  *	@adapter: the adapter
  *	@attempts: max number of polls of the status register
  *	@delay: delay between polls in ms
  *
  *	Wait for a flash operation to complete by polling the status register.
  */
 static int flash_wait_op(struct adapter *adapter, int attempts, int delay)
 {
 	int ret;
 	u32 status;
 
 	while (1) {
 		if ((ret = sf1_write(adapter, 1, 1, 1, SF_RD_STATUS)) != 0 ||
 		    (ret = sf1_read(adapter, 1, 0, 1, &status)) != 0)
 			return ret;
 		if (!(status & 1))
 			return 0;
 		if (--attempts == 0)
 			return -EAGAIN;
 		if (delay)
 			msleep(delay);
 	}
 }
 
 /**
  *	t4_read_flash - read words from serial flash
  *	@adapter: the adapter
  *	@addr: the start address for the read
  *	@nwords: how many 32-bit words to read
  *	@data: where to store the read data
  *	@byte_oriented: whether to store data as bytes or as words
  *
  *	Read the specified number of 32-bit words from the serial flash.
  *	If @byte_oriented is set the read data is stored as a byte array
  *	(i.e., big-endian), otherwise as 32-bit words in the platform's
  *	natural endianess.
  */
 int t4_read_flash(struct adapter *adapter, unsigned int addr,
 		  unsigned int nwords, u32 *data, int byte_oriented)
 {
 	int ret;
 
 	if (addr + nwords * sizeof(u32) > adapter->params.sf_size || (addr & 3))
 		return -EINVAL;
 
 	addr = swab32(addr) | SF_RD_DATA_FAST;
 
 	if ((ret = sf1_write(adapter, 4, 1, 0, addr)) != 0 ||
 	    (ret = sf1_read(adapter, 1, 1, 0, data)) != 0)
 		return ret;
 
 	for ( ; nwords; nwords--, data++) {
 		ret = sf1_read(adapter, 4, nwords > 1, nwords == 1, data);
 		if (nwords == 1)
 			t4_write_reg(adapter, A_SF_OP, 0);    /* unlock SF */
 		if (ret)
 			return ret;
 		if (byte_oriented)
 			*data = htonl(*data);
 	}
 	return 0;
 }
 
 /**
  *	t4_write_flash - write up to a page of data to the serial flash
  *	@adapter: the adapter
  *	@addr: the start address to write
  *	@n: length of data to write in bytes
  *	@data: the data to write
  *	@byte_oriented: whether to store data as bytes or as words
  *
  *	Writes up to a page of data (256 bytes) to the serial flash starting
  *	at the given address.  All the data must be written to the same page.
  *	If @byte_oriented is set the write data is stored as byte stream 
  *	(i.e. matches what on disk), otherwise in big-endian.
  */
 static int t4_write_flash(struct adapter *adapter, unsigned int addr,
 			  unsigned int n, const u8 *data, int byte_oriented)
 {
 	int ret;
 	u32 buf[SF_PAGE_SIZE / 4];
 	unsigned int i, c, left, val, offset = addr & 0xff;
 
 	if (addr >= adapter->params.sf_size || offset + n > SF_PAGE_SIZE)
 		return -EINVAL;
 
 	val = swab32(addr) | SF_PROG_PAGE;
 
 	if ((ret = sf1_write(adapter, 1, 0, 1, SF_WR_ENABLE)) != 0 ||
 	    (ret = sf1_write(adapter, 4, 1, 1, val)) != 0)
 		goto unlock;
 
 	for (left = n; left; left -= c) {
 		c = min(left, 4U);
 		for (val = 0, i = 0; i < c; ++i)
 			val = (val << 8) + *data++;
 
 		if (!byte_oriented)
 			val = htonl(val);
 
 		ret = sf1_write(adapter, c, c != left, 1, val);
 		if (ret)
 			goto unlock;
 	}
 	ret = flash_wait_op(adapter, 8, 1);
 	if (ret)
 		goto unlock;
 
 	t4_write_reg(adapter, A_SF_OP, 0);    /* unlock SF */
 
 	/* Read the page to verify the write succeeded */
 	ret = t4_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf,
 			    byte_oriented);
 	if (ret)
 		return ret;
 
 	if (memcmp(data - n, (u8 *)buf + offset, n)) {
 		CH_ERR(adapter, "failed to correctly write the flash page "
 		       "at %#x\n", addr);
 		return -EIO;
 	}
 	return 0;
 
 unlock:
 	t4_write_reg(adapter, A_SF_OP, 0);    /* unlock SF */
 	return ret;
 }
 
 /**
  *	t4_get_fw_version - read the firmware version
  *	@adapter: the adapter
  *	@vers: where to place the version
  *
  *	Reads the FW version from flash.
  */
 int t4_get_fw_version(struct adapter *adapter, u32 *vers)
 {
 	return t4_read_flash(adapter,
 			     FLASH_FW_START + offsetof(struct fw_hdr, fw_ver), 1,
 			     vers, 0);
 }
 
 /**
  *	t4_get_tp_version - read the TP microcode version
  *	@adapter: the adapter
  *	@vers: where to place the version
  *
  *	Reads the TP microcode version from flash.
  */
 int t4_get_tp_version(struct adapter *adapter, u32 *vers)
 {
 	return t4_read_flash(adapter, FLASH_FW_START + offsetof(struct fw_hdr,
 							      tp_microcode_ver),
 			     1, vers, 0);
 }
 
 /**
  *	t4_check_fw_version - check if the FW is compatible with this driver
  *	@adapter: the adapter
  *
  *	Checks if an adapter's FW is compatible with the driver.  Returns 0
  *	if there's exact match, a negative error if the version could not be
  *	read or there's a major version mismatch, and a positive value if the
  *	expected major version is found but there's a minor version mismatch.
  */
 int t4_check_fw_version(struct adapter *adapter)
 {
 	int ret, major, minor, micro;
 	int exp_major, exp_minor, exp_micro;
 
 	ret = t4_get_fw_version(adapter, &adapter->params.fw_vers);
 	if (!ret)
 		ret = t4_get_tp_version(adapter, &adapter->params.tp_vers);
 	if (ret)
 		return ret;
 
 	major = G_FW_HDR_FW_VER_MAJOR(adapter->params.fw_vers);
 	minor = G_FW_HDR_FW_VER_MINOR(adapter->params.fw_vers);
 	micro = G_FW_HDR_FW_VER_MICRO(adapter->params.fw_vers);
 
 	switch (chip_id(adapter)) {
 	case CHELSIO_T4:
 		exp_major = T4FW_VERSION_MAJOR;
 		exp_minor = T4FW_VERSION_MINOR;
 		exp_micro = T4FW_VERSION_MICRO;
 		break;
 	case CHELSIO_T5:
 		exp_major = T5FW_VERSION_MAJOR;
 		exp_minor = T5FW_VERSION_MINOR;
 		exp_micro = T5FW_VERSION_MICRO;
 		break;
 	default:
 		CH_ERR(adapter, "Unsupported chip type, %x\n",
 		    chip_id(adapter));
 		return -EINVAL;
 	}
 
 	if (major != exp_major) {            /* major mismatch - fail */
 		CH_ERR(adapter, "card FW has major version %u, driver wants "
 		       "%u\n", major, exp_major);
 		return -EINVAL;
 	}
 
 	if (minor == exp_minor && micro == exp_micro)
 		return 0;                                   /* perfect match */
 
 	/* Minor/micro version mismatch.  Report it but often it's OK. */
 	return 1;
 }
 
 /**
  *	t4_flash_erase_sectors - erase a range of flash sectors
  *	@adapter: the adapter
  *	@start: the first sector to erase
  *	@end: the last sector to erase
  *
  *	Erases the sectors in the given inclusive range.
  */
 static int t4_flash_erase_sectors(struct adapter *adapter, int start, int end)
 {
 	int ret = 0;
 
 	while (start <= end) {
 		if ((ret = sf1_write(adapter, 1, 0, 1, SF_WR_ENABLE)) != 0 ||
 		    (ret = sf1_write(adapter, 4, 0, 1,
 				     SF_ERASE_SECTOR | (start << 8))) != 0 ||
 		    (ret = flash_wait_op(adapter, 14, 500)) != 0) {
 			CH_ERR(adapter, "erase of flash sector %d failed, "
 			       "error %d\n", start, ret);
 			break;
 		}
 		start++;
 	}
 	t4_write_reg(adapter, A_SF_OP, 0);    /* unlock SF */
 	return ret;
 }
 
 /**
  *	t4_flash_cfg_addr - return the address of the flash configuration file
  *	@adapter: the adapter
  *
  *	Return the address within the flash where the Firmware Configuration
  *	File is stored, or an error if the device FLASH is too small to contain
  *	a Firmware Configuration File.
  */
 int t4_flash_cfg_addr(struct adapter *adapter)
 {
 	/*
 	 * If the device FLASH isn't large enough to hold a Firmware
 	 * Configuration File, return an error.
 	 */
 	if (adapter->params.sf_size < FLASH_CFG_START + FLASH_CFG_MAX_SIZE)
 		return -ENOSPC;
 
 	return FLASH_CFG_START;
 }
 
 /**
  *	t4_load_cfg - download config file
  *	@adap: the adapter
  *	@cfg_data: the cfg text file to write
  *	@size: text file size
  *
  *	Write the supplied config text file to the card's serial flash.
  */
 int t4_load_cfg(struct adapter *adap, const u8 *cfg_data, unsigned int size)
 {
 	int ret, i, n, cfg_addr;
 	unsigned int addr;
 	unsigned int flash_cfg_start_sec;
 	unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec;
 
 	cfg_addr = t4_flash_cfg_addr(adap);
 	if (cfg_addr < 0)
 		return cfg_addr;
 
 	addr = cfg_addr;
 	flash_cfg_start_sec = addr / SF_SEC_SIZE;
 
 	if (size > FLASH_CFG_MAX_SIZE) {
 		CH_ERR(adap, "cfg file too large, max is %u bytes\n",
 		       FLASH_CFG_MAX_SIZE);
 		return -EFBIG;
 	}
 
 	i = DIV_ROUND_UP(FLASH_CFG_MAX_SIZE,	/* # of sectors spanned */
 			 sf_sec_size);
 	ret = t4_flash_erase_sectors(adap, flash_cfg_start_sec,
 				     flash_cfg_start_sec + i - 1);
 	/*
 	 * If size == 0 then we're simply erasing the FLASH sectors associated
 	 * with the on-adapter Firmware Configuration File.
 	 */
 	if (ret || size == 0)
 		goto out;
 
 	/* this will write to the flash up to SF_PAGE_SIZE at a time */
 	for (i = 0; i< size; i+= SF_PAGE_SIZE) {
 		if ( (size - i) <  SF_PAGE_SIZE) 
 			n = size - i;
 		else 
 			n = SF_PAGE_SIZE;
 		ret = t4_write_flash(adap, addr, n, cfg_data, 1);
 		if (ret)
 			goto out;
 		
 		addr += SF_PAGE_SIZE;
 		cfg_data += SF_PAGE_SIZE;
 	} 
                 
 out:
 	if (ret)
 		CH_ERR(adap, "config file %s failed %d\n",
 		       (size == 0 ? "clear" : "download"), ret);
 	return ret;
 }
 
 
 /**
  *	t4_load_fw - download firmware
  *	@adap: the adapter
  *	@fw_data: the firmware image to write
  *	@size: image size
  *
  *	Write the supplied firmware image to the card's serial flash.
  */
 int t4_load_fw(struct adapter *adap, const u8 *fw_data, unsigned int size)
 {
 	u32 csum;
 	int ret, addr;
 	unsigned int i;
 	u8 first_page[SF_PAGE_SIZE];
 	const u32 *p = (const u32 *)fw_data;
 	const struct fw_hdr *hdr = (const struct fw_hdr *)fw_data;
 	unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec;
 	unsigned int fw_start_sec;
 	unsigned int fw_start;
 	unsigned int fw_size;
 
 	if (ntohl(hdr->magic) == FW_HDR_MAGIC_BOOTSTRAP) {
 		fw_start_sec = FLASH_FWBOOTSTRAP_START_SEC;
 		fw_start = FLASH_FWBOOTSTRAP_START;
 		fw_size = FLASH_FWBOOTSTRAP_MAX_SIZE;
 	} else {
 		fw_start_sec = FLASH_FW_START_SEC;
  		fw_start = FLASH_FW_START;
 		fw_size = FLASH_FW_MAX_SIZE;
 	}
 	if (!size) {
 		CH_ERR(adap, "FW image has no data\n");
 		return -EINVAL;
 	}
 	if (size & 511) {
 		CH_ERR(adap, "FW image size not multiple of 512 bytes\n");
 		return -EINVAL;
 	}
 	if (ntohs(hdr->len512) * 512 != size) {
 		CH_ERR(adap, "FW image size differs from size in FW header\n");
 		return -EINVAL;
 	}
 	if (size > fw_size) {
 		CH_ERR(adap, "FW image too large, max is %u bytes\n", fw_size);
 		return -EFBIG;
 	}
 	if ((is_t4(adap) && hdr->chip != FW_HDR_CHIP_T4) ||
 	    (is_t5(adap) && hdr->chip != FW_HDR_CHIP_T5)) {
 		CH_ERR(adap,
 		    "FW image (%d) is not suitable for this adapter (%d)\n",
 		    hdr->chip, chip_id(adap));
 		return -EINVAL;
 	}
 
 	for (csum = 0, i = 0; i < size / sizeof(csum); i++)
 		csum += ntohl(p[i]);
 
 	if (csum != 0xffffffff) {
 		CH_ERR(adap, "corrupted firmware image, checksum %#x\n",
 		       csum);
 		return -EINVAL;
 	}
 
 	i = DIV_ROUND_UP(size, sf_sec_size);        /* # of sectors spanned */
 	ret = t4_flash_erase_sectors(adap, fw_start_sec, fw_start_sec + i - 1);
 	if (ret)
 		goto out;
 
 	/*
 	 * We write the correct version at the end so the driver can see a bad
 	 * version if the FW write fails.  Start by writing a copy of the
 	 * first page with a bad version.
 	 */
 	memcpy(first_page, fw_data, SF_PAGE_SIZE);
 	((struct fw_hdr *)first_page)->fw_ver = htonl(0xffffffff);
 	ret = t4_write_flash(adap, fw_start, SF_PAGE_SIZE, first_page, 1);
 	if (ret)
 		goto out;
 
 	addr = fw_start;
 	for (size -= SF_PAGE_SIZE; size; size -= SF_PAGE_SIZE) {
 		addr += SF_PAGE_SIZE;
 		fw_data += SF_PAGE_SIZE;
 		ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, fw_data, 1);
 		if (ret)
 			goto out;
 	}
 
 	ret = t4_write_flash(adap,
 			     fw_start + offsetof(struct fw_hdr, fw_ver),
 			     sizeof(hdr->fw_ver), (const u8 *)&hdr->fw_ver, 1);
 out:
 	if (ret)
 		CH_ERR(adap, "firmware download failed, error %d\n", ret);
 	return ret;
 }
 
 /* BIOS boot headers */
 typedef struct pci_expansion_rom_header {
 	u8	signature[2]; /* ROM Signature. Should be 0xaa55 */
 	u8	reserved[22]; /* Reserved per processor Architecture data */
 	u8	pcir_offset[2]; /* Offset to PCI Data Structure */
 } pci_exp_rom_header_t; /* PCI_EXPANSION_ROM_HEADER */
 
 /* Legacy PCI Expansion ROM Header */
 typedef struct legacy_pci_expansion_rom_header {
 	u8	signature[2]; /* ROM Signature. Should be 0xaa55 */
 	u8	size512; /* Current Image Size in units of 512 bytes */
 	u8	initentry_point[4];
 	u8	cksum; /* Checksum computed on the entire Image */
 	u8	reserved[16]; /* Reserved */
 	u8	pcir_offset[2]; /* Offset to PCI Data Struture */
 } legacy_pci_exp_rom_header_t; /* LEGACY_PCI_EXPANSION_ROM_HEADER */
 
 /* EFI PCI Expansion ROM Header */
 typedef struct efi_pci_expansion_rom_header {
 	u8	signature[2]; // ROM signature. The value 0xaa55
 	u8	initialization_size[2]; /* Units 512. Includes this header */
 	u8	efi_signature[4]; /* Signature from EFI image header. 0x0EF1 */
 	u8	efi_subsystem[2]; /* Subsystem value for EFI image header */
 	u8	efi_machine_type[2]; /* Machine type from EFI image header */
 	u8	compression_type[2]; /* Compression type. */
 		/* 
 		 * Compression type definition
 		 * 0x0: uncompressed
 		 * 0x1: Compressed
 		 * 0x2-0xFFFF: Reserved
 		 */
 	u8	reserved[8]; /* Reserved */
 	u8	efi_image_header_offset[2]; /* Offset to EFI Image */
 	u8	pcir_offset[2]; /* Offset to PCI Data Structure */
 } efi_pci_exp_rom_header_t; /* EFI PCI Expansion ROM Header */
 
 /* PCI Data Structure Format */
 typedef struct pcir_data_structure { /* PCI Data Structure */
 	u8	signature[4]; /* Signature. The string "PCIR" */
 	u8	vendor_id[2]; /* Vendor Identification */
 	u8	device_id[2]; /* Device Identification */
 	u8	vital_product[2]; /* Pointer to Vital Product Data */
 	u8	length[2]; /* PCIR Data Structure Length */
 	u8	revision; /* PCIR Data Structure Revision */
 	u8	class_code[3]; /* Class Code */
 	u8	image_length[2]; /* Image Length. Multiple of 512B */
 	u8	code_revision[2]; /* Revision Level of Code/Data */
 	u8	code_type; /* Code Type. */
 		/*
 		 * PCI Expansion ROM Code Types
 		 * 0x00: Intel IA-32, PC-AT compatible. Legacy
 		 * 0x01: Open Firmware standard for PCI. FCODE
 		 * 0x02: Hewlett-Packard PA RISC. HP reserved
 		 * 0x03: EFI Image. EFI
 		 * 0x04-0xFF: Reserved.
 		 */
 	u8	indicator; /* Indicator. Identifies the last image in the ROM */
 	u8	reserved[2]; /* Reserved */
 } pcir_data_t; /* PCI__DATA_STRUCTURE */
 
 /* BOOT constants */
 enum {
 	BOOT_FLASH_BOOT_ADDR = 0x0,/* start address of boot image in flash */
 	BOOT_SIGNATURE = 0xaa55,   /* signature of BIOS boot ROM */
 	BOOT_SIZE_INC = 512,       /* image size measured in 512B chunks */
 	BOOT_MIN_SIZE = sizeof(pci_exp_rom_header_t), /* basic header */
 	BOOT_MAX_SIZE = 1024*BOOT_SIZE_INC, /* 1 byte * length increment  */
 	VENDOR_ID = 0x1425, /* Vendor ID */
 	PCIR_SIGNATURE = 0x52494350 /* PCIR signature */
 };
 
 /*
  *	modify_device_id - Modifies the device ID of the Boot BIOS image 
  *	@adatper: the device ID to write.
  *	@boot_data: the boot image to modify.
  *
  *	Write the supplied device ID to the boot BIOS image.
  */
 static void modify_device_id(int device_id, u8 *boot_data)
 {
 	legacy_pci_exp_rom_header_t *header;
 	pcir_data_t *pcir_header;
 	u32 cur_header = 0;
 
 	/*
 	 * Loop through all chained images and change the device ID's
 	 */
 	while (1) {
 		header = (legacy_pci_exp_rom_header_t *) &boot_data[cur_header];
 		pcir_header = (pcir_data_t *) &boot_data[cur_header +
 		    le16_to_cpu(*(u16*)header->pcir_offset)];
 
 		/*
 		 * Only modify the Device ID if code type is Legacy or HP.
 		 * 0x00: Okay to modify
 		 * 0x01: FCODE. Do not be modify
 		 * 0x03: Okay to modify
 		 * 0x04-0xFF: Do not modify
 		 */
 		if (pcir_header->code_type == 0x00) {
 			u8 csum = 0;
 			int i;
 
 			/*
 			 * Modify Device ID to match current adatper
 			 */
 			*(u16*) pcir_header->device_id = device_id;
 
 			/*
 			 * Set checksum temporarily to 0.
 			 * We will recalculate it later.
 			 */
 			header->cksum = 0x0;
 
 			/*
 			 * Calculate and update checksum
 			 */
 			for (i = 0; i < (header->size512 * 512); i++)
 				csum += (u8)boot_data[cur_header + i];
 
 			/*
 			 * Invert summed value to create the checksum
 			 * Writing new checksum value directly to the boot data
 			 */
 			boot_data[cur_header + 7] = -csum;
 
 		} else if (pcir_header->code_type == 0x03) {
 
 			/*
 			 * Modify Device ID to match current adatper
 			 */
 			*(u16*) pcir_header->device_id = device_id;
 
 		}
 
 
 		/*
 		 * Check indicator element to identify if this is the last
 		 * image in the ROM.
 		 */
 		if (pcir_header->indicator & 0x80)
 			break;
 
 		/*
 		 * Move header pointer up to the next image in the ROM.
 		 */
 		cur_header += header->size512 * 512;
 	}
 }
 
 /*
  *	t4_load_boot - download boot flash
  *	@adapter: the adapter
  *	@boot_data: the boot image to write
  *	@boot_addr: offset in flash to write boot_data
  *	@size: image size
  *
  *	Write the supplied boot image to the card's serial flash.
  *	The boot image has the following sections: a 28-byte header and the
  *	boot image.
  */
 int t4_load_boot(struct adapter *adap, u8 *boot_data, 
 		 unsigned int boot_addr, unsigned int size)
 {
 	pci_exp_rom_header_t *header;
 	int pcir_offset ;
 	pcir_data_t *pcir_header;
 	int ret, addr;
 	uint16_t device_id;
 	unsigned int i;
 	unsigned int boot_sector = boot_addr * 1024;
 	unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec;
 
 	/*
 	 * Make sure the boot image does not encroach on the firmware region
 	 */
 	if ((boot_sector + size) >> 16 > FLASH_FW_START_SEC) {
 		CH_ERR(adap, "boot image encroaching on firmware region\n");
 		return -EFBIG;
 	}
 
 	/*
 	 * Number of sectors spanned
 	 */
 	i = DIV_ROUND_UP(size ? size : FLASH_BOOTCFG_MAX_SIZE,
 			sf_sec_size);
 	ret = t4_flash_erase_sectors(adap, boot_sector >> 16,
 				     (boot_sector >> 16) + i - 1);
 
 	/*
 	 * If size == 0 then we're simply erasing the FLASH sectors associated
 	 * with the on-adapter option ROM file
 	 */
 	if (ret || (size == 0))
 		goto out;
 
 	/* Get boot header */
 	header = (pci_exp_rom_header_t *)boot_data;
 	pcir_offset = le16_to_cpu(*(u16 *)header->pcir_offset);
 	/* PCIR Data Structure */
 	pcir_header = (pcir_data_t *) &boot_data[pcir_offset];
 
 	/*
 	 * Perform some primitive sanity testing to avoid accidentally
 	 * writing garbage over the boot sectors.  We ought to check for
 	 * more but it's not worth it for now ...
 	 */
 	if (size < BOOT_MIN_SIZE || size > BOOT_MAX_SIZE) {
 		CH_ERR(adap, "boot image too small/large\n");
 		return -EFBIG;
 	}
 
 	/*
 	 * Check BOOT ROM header signature
 	 */
 	if (le16_to_cpu(*(u16*)header->signature) != BOOT_SIGNATURE ) {
 		CH_ERR(adap, "Boot image missing signature\n");
 		return -EINVAL;
 	}
 
 	/*
 	 * Check PCI header signature
 	 */
 	if (le32_to_cpu(*(u32*)pcir_header->signature) != PCIR_SIGNATURE) {
 		CH_ERR(adap, "PCI header missing signature\n");
 		return -EINVAL;
 	}
 
 	/*
 	 * Check Vendor ID matches Chelsio ID
 	 */
 	if (le16_to_cpu(*(u16*)pcir_header->vendor_id) != VENDOR_ID) {
 		CH_ERR(adap, "Vendor ID missing signature\n");
 		return -EINVAL;
 	}
 
 	/*
 	 * Retrieve adapter's device ID
 	 */
 	t4_os_pci_read_cfg2(adap, PCI_DEVICE_ID, &device_id);
 	/* Want to deal with PF 0 so I strip off PF 4 indicator */
 	device_id = (device_id & 0xff) | 0x4000;
 
 	/*
 	 * Check PCIE Device ID
 	 */
 	if (le16_to_cpu(*(u16*)pcir_header->device_id) != device_id) {
 		/*
 		 * Change the device ID in the Boot BIOS image to match
 		 * the Device ID of the current adapter.
 		 */
 		modify_device_id(device_id, boot_data);
 	}
 
 	/*
 	 * Skip over the first SF_PAGE_SIZE worth of data and write it after
 	 * we finish copying the rest of the boot image. This will ensure
 	 * that the BIOS boot header will only be written if the boot image
 	 * was written in full.
 	 */
 	addr = boot_sector;
 	for (size -= SF_PAGE_SIZE; size; size -= SF_PAGE_SIZE) {
 		addr += SF_PAGE_SIZE; 
 		boot_data += SF_PAGE_SIZE;
 		ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, boot_data, 0);
 		if (ret)
 			goto out;
 	}
 
 	ret = t4_write_flash(adap, boot_sector, SF_PAGE_SIZE, boot_data, 0);
 
 out:
 	if (ret)
 		CH_ERR(adap, "boot image download failed, error %d\n", ret);
 	return ret;
 }
 
 /**
  *	t4_read_cimq_cfg - read CIM queue configuration
  *	@adap: the adapter
  *	@base: holds the queue base addresses in bytes
  *	@size: holds the queue sizes in bytes
  *	@thres: holds the queue full thresholds in bytes
  *
  *	Returns the current configuration of the CIM queues, starting with
  *	the IBQs, then the OBQs.
  */
 void t4_read_cimq_cfg(struct adapter *adap, u16 *base, u16 *size, u16 *thres)
 {
 	unsigned int i, v;
 	int cim_num_obq = is_t4(adap) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
 
 	for (i = 0; i < CIM_NUM_IBQ; i++) {
 		t4_write_reg(adap, A_CIM_QUEUE_CONFIG_REF, F_IBQSELECT |
 			     V_QUENUMSELECT(i));
 		v = t4_read_reg(adap, A_CIM_QUEUE_CONFIG_CTRL);
 		*base++ = G_CIMQBASE(v) * 256; /* value is in 256-byte units */
 		*size++ = G_CIMQSIZE(v) * 256; /* value is in 256-byte units */
 		*thres++ = G_QUEFULLTHRSH(v) * 8;   /* 8-byte unit */
 	}
 	for (i = 0; i < cim_num_obq; i++) {
 		t4_write_reg(adap, A_CIM_QUEUE_CONFIG_REF, F_OBQSELECT |
 			     V_QUENUMSELECT(i));
 		v = t4_read_reg(adap, A_CIM_QUEUE_CONFIG_CTRL);
 		*base++ = G_CIMQBASE(v) * 256; /* value is in 256-byte units */
 		*size++ = G_CIMQSIZE(v) * 256; /* value is in 256-byte units */
 	}
 }
 
 /**
  *	t4_read_cim_ibq - read the contents of a CIM inbound queue
  *	@adap: the adapter
  *	@qid: the queue index
  *	@data: where to store the queue contents
  *	@n: capacity of @data in 32-bit words
  *
  *	Reads the contents of the selected CIM queue starting at address 0 up
  *	to the capacity of @data.  @n must be a multiple of 4.  Returns < 0 on
  *	error and the number of 32-bit words actually read on success.
  */
 int t4_read_cim_ibq(struct adapter *adap, unsigned int qid, u32 *data, size_t n)
 {
 	int i, err;
 	unsigned int addr;
 	const unsigned int nwords = CIM_IBQ_SIZE * 4;
 
 	if (qid > 5 || (n & 3))
 		return -EINVAL;
 
 	addr = qid * nwords;
 	if (n > nwords)
 		n = nwords;
 
 	for (i = 0; i < n; i++, addr++) {
 		t4_write_reg(adap, A_CIM_IBQ_DBG_CFG, V_IBQDBGADDR(addr) |
 			     F_IBQDBGEN);
 		/*
 		 * It might take 3-10ms before the IBQ debug read access is
 		 * allowed.  Wait for 1 Sec with a delay of 1 usec.
 		 */
 		err = t4_wait_op_done(adap, A_CIM_IBQ_DBG_CFG, F_IBQDBGBUSY, 0,
 				      1000000, 1);
 		if (err)
 			return err;
 		*data++ = t4_read_reg(adap, A_CIM_IBQ_DBG_DATA);
 	}
 	t4_write_reg(adap, A_CIM_IBQ_DBG_CFG, 0);
 	return i;
 }
 
 /**
  *	t4_read_cim_obq - read the contents of a CIM outbound queue
  *	@adap: the adapter
  *	@qid: the queue index
  *	@data: where to store the queue contents
  *	@n: capacity of @data in 32-bit words
  *
  *	Reads the contents of the selected CIM queue starting at address 0 up
  *	to the capacity of @data.  @n must be a multiple of 4.  Returns < 0 on
  *	error and the number of 32-bit words actually read on success.
  */
 int t4_read_cim_obq(struct adapter *adap, unsigned int qid, u32 *data, size_t n)
 {
 	int i, err;
 	unsigned int addr, v, nwords;
 	int cim_num_obq = is_t4(adap) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
 
 	if (qid >= cim_num_obq || (n & 3))
 		return -EINVAL;
 
 	t4_write_reg(adap, A_CIM_QUEUE_CONFIG_REF, F_OBQSELECT |
 		     V_QUENUMSELECT(qid));
 	v = t4_read_reg(adap, A_CIM_QUEUE_CONFIG_CTRL);
 
 	addr = G_CIMQBASE(v) * 64;    /* muliple of 256 -> muliple of 4 */
 	nwords = G_CIMQSIZE(v) * 64;  /* same */
 	if (n > nwords)
 		n = nwords;
 
 	for (i = 0; i < n; i++, addr++) {
 		t4_write_reg(adap, A_CIM_OBQ_DBG_CFG, V_OBQDBGADDR(addr) |
 			     F_OBQDBGEN);
 		err = t4_wait_op_done(adap, A_CIM_OBQ_DBG_CFG, F_OBQDBGBUSY, 0,
 				      2, 1);
 		if (err)
 			return err;
 		*data++ = t4_read_reg(adap, A_CIM_OBQ_DBG_DATA);
 	}
 	t4_write_reg(adap, A_CIM_OBQ_DBG_CFG, 0);
 	return i;
 }
 
 enum {
 	CIM_QCTL_BASE     = 0,
 	CIM_CTL_BASE      = 0x2000,
 	CIM_PBT_ADDR_BASE = 0x2800,
 	CIM_PBT_LRF_BASE  = 0x3000,
 	CIM_PBT_DATA_BASE = 0x3800
 };
 
 /**
  *	t4_cim_read - read a block from CIM internal address space
  *	@adap: the adapter
  *	@addr: the start address within the CIM address space
  *	@n: number of words to read
  *	@valp: where to store the result
  *
  *	Reads a block of 4-byte words from the CIM intenal address space.
  */
 int t4_cim_read(struct adapter *adap, unsigned int addr, unsigned int n,
 		unsigned int *valp)
 {
 	int ret = 0;
 
 	if (t4_read_reg(adap, A_CIM_HOST_ACC_CTRL) & F_HOSTBUSY)
 		return -EBUSY;
 
 	for ( ; !ret && n--; addr += 4) {
 		t4_write_reg(adap, A_CIM_HOST_ACC_CTRL, addr);
 		ret = t4_wait_op_done(adap, A_CIM_HOST_ACC_CTRL, F_HOSTBUSY,
 				      0, 5, 2);
 		if (!ret)
 			*valp++ = t4_read_reg(adap, A_CIM_HOST_ACC_DATA);
 	}
 	return ret;
 }
 
 /**
  *	t4_cim_write - write a block into CIM internal address space
  *	@adap: the adapter
  *	@addr: the start address within the CIM address space
  *	@n: number of words to write
  *	@valp: set of values to write
  *
  *	Writes a block of 4-byte words into the CIM intenal address space.
  */
 int t4_cim_write(struct adapter *adap, unsigned int addr, unsigned int n,
 		 const unsigned int *valp)
 {
 	int ret = 0;
 
 	if (t4_read_reg(adap, A_CIM_HOST_ACC_CTRL) & F_HOSTBUSY)
 		return -EBUSY;
 
 	for ( ; !ret && n--; addr += 4) {
 		t4_write_reg(adap, A_CIM_HOST_ACC_DATA, *valp++);
 		t4_write_reg(adap, A_CIM_HOST_ACC_CTRL, addr | F_HOSTWRITE);
 		ret = t4_wait_op_done(adap, A_CIM_HOST_ACC_CTRL, F_HOSTBUSY,
 				      0, 5, 2);
 	}
 	return ret;
 }
 
 static int t4_cim_write1(struct adapter *adap, unsigned int addr, unsigned int val)
 {
 	return t4_cim_write(adap, addr, 1, &val);
 }
 
 /**
  *	t4_cim_ctl_read - read a block from CIM control region
  *	@adap: the adapter
  *	@addr: the start address within the CIM control region
  *	@n: number of words to read
  *	@valp: where to store the result
  *
  *	Reads a block of 4-byte words from the CIM control region.
  */
 int t4_cim_ctl_read(struct adapter *adap, unsigned int addr, unsigned int n,
 		    unsigned int *valp)
 {
 	return t4_cim_read(adap, addr + CIM_CTL_BASE, n, valp);
 }
 
 /**
  *	t4_cim_read_la - read CIM LA capture buffer
  *	@adap: the adapter
  *	@la_buf: where to store the LA data
  *	@wrptr: the HW write pointer within the capture buffer
  *
  *	Reads the contents of the CIM LA buffer with the most recent entry at
  *	the end	of the returned data and with the entry at @wrptr first.
  *	We try to leave the LA in the running state we find it in.
  */
 int t4_cim_read_la(struct adapter *adap, u32 *la_buf, unsigned int *wrptr)
 {
 	int i, ret;
 	unsigned int cfg, val, idx;
 
 	ret = t4_cim_read(adap, A_UP_UP_DBG_LA_CFG, 1, &cfg);
 	if (ret)
 		return ret;
 
 	if (cfg & F_UPDBGLAEN) {                /* LA is running, freeze it */
 		ret = t4_cim_write1(adap, A_UP_UP_DBG_LA_CFG, 0);
 		if (ret)
 			return ret;
 	}
 
 	ret = t4_cim_read(adap, A_UP_UP_DBG_LA_CFG, 1, &val);
 	if (ret)
 		goto restart;
 
 	idx = G_UPDBGLAWRPTR(val);
 	if (wrptr)
 		*wrptr = idx;
 
 	for (i = 0; i < adap->params.cim_la_size; i++) {
 		ret = t4_cim_write1(adap, A_UP_UP_DBG_LA_CFG,
 				    V_UPDBGLARDPTR(idx) | F_UPDBGLARDEN);
 		if (ret)
 			break;
 		ret = t4_cim_read(adap, A_UP_UP_DBG_LA_CFG, 1, &val);
 		if (ret)
 			break;
 		if (val & F_UPDBGLARDEN) {
 			ret = -ETIMEDOUT;
 			break;
 		}
 		ret = t4_cim_read(adap, A_UP_UP_DBG_LA_DATA, 1, &la_buf[i]);
 		if (ret)
 			break;
 		idx = (idx + 1) & M_UPDBGLARDPTR;
 	}
 restart:
 	if (cfg & F_UPDBGLAEN) {
 		int r = t4_cim_write1(adap, A_UP_UP_DBG_LA_CFG,
 				      cfg & ~F_UPDBGLARDEN);
 		if (!ret)
 			ret = r;
 	}
 	return ret;
 }
 
 void t4_cim_read_pif_la(struct adapter *adap, u32 *pif_req, u32 *pif_rsp,
 			unsigned int *pif_req_wrptr,
 			unsigned int *pif_rsp_wrptr)
 {
 	int i, j;
 	u32 cfg, val, req, rsp;
 
 	cfg = t4_read_reg(adap, A_CIM_DEBUGCFG);
 	if (cfg & F_LADBGEN)
 		t4_write_reg(adap, A_CIM_DEBUGCFG, cfg ^ F_LADBGEN);
 
 	val = t4_read_reg(adap, A_CIM_DEBUGSTS);
 	req = G_POLADBGWRPTR(val);
 	rsp = G_PILADBGWRPTR(val);
 	if (pif_req_wrptr)
 		*pif_req_wrptr = req;
 	if (pif_rsp_wrptr)
 		*pif_rsp_wrptr = rsp;
 
 	for (i = 0; i < CIM_PIFLA_SIZE; i++) {
 		for (j = 0; j < 6; j++) {
 			t4_write_reg(adap, A_CIM_DEBUGCFG, V_POLADBGRDPTR(req) |
 				     V_PILADBGRDPTR(rsp));
 			*pif_req++ = t4_read_reg(adap, A_CIM_PO_LA_DEBUGDATA);
 			*pif_rsp++ = t4_read_reg(adap, A_CIM_PI_LA_DEBUGDATA);
 			req++;
 			rsp++;
 		}
 		req = (req + 2) & M_POLADBGRDPTR;
 		rsp = (rsp + 2) & M_PILADBGRDPTR;
 	}
 	t4_write_reg(adap, A_CIM_DEBUGCFG, cfg);
 }
 
 void t4_cim_read_ma_la(struct adapter *adap, u32 *ma_req, u32 *ma_rsp)
 {
 	u32 cfg;
 	int i, j, idx;
 
 	cfg = t4_read_reg(adap, A_CIM_DEBUGCFG);
 	if (cfg & F_LADBGEN)
 		t4_write_reg(adap, A_CIM_DEBUGCFG, cfg ^ F_LADBGEN);
 
 	for (i = 0; i < CIM_MALA_SIZE; i++) {
 		for (j = 0; j < 5; j++) {
 			idx = 8 * i + j;
 			t4_write_reg(adap, A_CIM_DEBUGCFG, V_POLADBGRDPTR(idx) |
 				     V_PILADBGRDPTR(idx));
 			*ma_req++ = t4_read_reg(adap, A_CIM_PO_LA_MADEBUGDATA);
 			*ma_rsp++ = t4_read_reg(adap, A_CIM_PI_LA_MADEBUGDATA);
 		}
 	}
 	t4_write_reg(adap, A_CIM_DEBUGCFG, cfg);
 }
 
 /**
  *	t4_tp_read_la - read TP LA capture buffer
  *	@adap: the adapter
  *	@la_buf: where to store the LA data
  *	@wrptr: the HW write pointer within the capture buffer
  *
  *	Reads the contents of the TP LA buffer with the most recent entry at
  *	the end	of the returned data and with the entry at @wrptr first.
  *	We leave the LA in the running state we find it in.
  */
 void t4_tp_read_la(struct adapter *adap, u64 *la_buf, unsigned int *wrptr)
 {
 	bool last_incomplete;
 	unsigned int i, cfg, val, idx;
 
 	cfg = t4_read_reg(adap, A_TP_DBG_LA_CONFIG) & 0xffff;
 	if (cfg & F_DBGLAENABLE)                    /* freeze LA */
 		t4_write_reg(adap, A_TP_DBG_LA_CONFIG,
 			     adap->params.tp.la_mask | (cfg ^ F_DBGLAENABLE));
 
 	val = t4_read_reg(adap, A_TP_DBG_LA_CONFIG);
 	idx = G_DBGLAWPTR(val);
 	last_incomplete = G_DBGLAMODE(val) >= 2 && (val & F_DBGLAWHLF) == 0;
 	if (last_incomplete)
 		idx = (idx + 1) & M_DBGLARPTR;
 	if (wrptr)
 		*wrptr = idx;
 
 	val &= 0xffff;
 	val &= ~V_DBGLARPTR(M_DBGLARPTR);
 	val |= adap->params.tp.la_mask;
 
 	for (i = 0; i < TPLA_SIZE; i++) {
 		t4_write_reg(adap, A_TP_DBG_LA_CONFIG, V_DBGLARPTR(idx) | val);
 		la_buf[i] = t4_read_reg64(adap, A_TP_DBG_LA_DATAL);
 		idx = (idx + 1) & M_DBGLARPTR;
 	}
 
 	/* Wipe out last entry if it isn't valid */
 	if (last_incomplete)
 		la_buf[TPLA_SIZE - 1] = ~0ULL;
 
 	if (cfg & F_DBGLAENABLE)                    /* restore running state */
 		t4_write_reg(adap, A_TP_DBG_LA_CONFIG,
 			     cfg | adap->params.tp.la_mask);
 }
 
 void t4_ulprx_read_la(struct adapter *adap, u32 *la_buf)
 {
 	unsigned int i, j;
 
 	for (i = 0; i < 8; i++) {
 		u32 *p = la_buf + i;
 
 		t4_write_reg(adap, A_ULP_RX_LA_CTL, i);
 		j = t4_read_reg(adap, A_ULP_RX_LA_WRPTR);
 		t4_write_reg(adap, A_ULP_RX_LA_RDPTR, j);
 		for (j = 0; j < ULPRX_LA_SIZE; j++, p += 8)
 			*p = t4_read_reg(adap, A_ULP_RX_LA_RDDATA);
 	}
 }
 
 #define ADVERT_MASK (FW_PORT_CAP_SPEED_100M | FW_PORT_CAP_SPEED_1G |\
 		     FW_PORT_CAP_SPEED_10G | FW_PORT_CAP_SPEED_40G | \
 		     FW_PORT_CAP_SPEED_100G | FW_PORT_CAP_ANEG)
 
 /**
  *	t4_link_start - apply link configuration to MAC/PHY
  *	@phy: the PHY to setup
  *	@mac: the MAC to setup
  *	@lc: the requested link configuration
  *
  *	Set up a port's MAC and PHY according to a desired link configuration.
  *	- If the PHY can auto-negotiate first decide what to advertise, then
  *	  enable/disable auto-negotiation as desired, and reset.
  *	- If the PHY does not auto-negotiate just reset it.
  *	- If auto-negotiation is off set the MAC to the proper speed/duplex/FC,
  *	  otherwise do it later based on the outcome of auto-negotiation.
  */
 int t4_link_start(struct adapter *adap, unsigned int mbox, unsigned int port,
 		  struct link_config *lc)
 {
 	struct fw_port_cmd c;
 	unsigned int fc = 0, mdi = V_FW_PORT_CAP_MDI(FW_PORT_CAP_MDI_AUTO);
 
 	lc->link_ok = 0;
 	if (lc->requested_fc & PAUSE_RX)
 		fc |= FW_PORT_CAP_FC_RX;
 	if (lc->requested_fc & PAUSE_TX)
 		fc |= FW_PORT_CAP_FC_TX;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_portid = htonl(V_FW_CMD_OP(FW_PORT_CMD) | F_FW_CMD_REQUEST |
 			       F_FW_CMD_EXEC | V_FW_PORT_CMD_PORTID(port));
 	c.action_to_len16 = htonl(V_FW_PORT_CMD_ACTION(FW_PORT_ACTION_L1_CFG) |
 				  FW_LEN16(c));
 
 	if (!(lc->supported & FW_PORT_CAP_ANEG)) {
 		c.u.l1cfg.rcap = htonl((lc->supported & ADVERT_MASK) | fc);
 		lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
 	} else if (lc->autoneg == AUTONEG_DISABLE) {
 		c.u.l1cfg.rcap = htonl(lc->requested_speed | fc | mdi);
 		lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
 	} else
 		c.u.l1cfg.rcap = htonl(lc->advertising | fc | mdi);
 
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
 /**
  *	t4_restart_aneg - restart autonegotiation
  *	@adap: the adapter
  *	@mbox: mbox to use for the FW command
  *	@port: the port id
  *
  *	Restarts autonegotiation for the selected port.
  */
 int t4_restart_aneg(struct adapter *adap, unsigned int mbox, unsigned int port)
 {
 	struct fw_port_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_portid = htonl(V_FW_CMD_OP(FW_PORT_CMD) | F_FW_CMD_REQUEST |
 			       F_FW_CMD_EXEC | V_FW_PORT_CMD_PORTID(port));
 	c.action_to_len16 = htonl(V_FW_PORT_CMD_ACTION(FW_PORT_ACTION_L1_CFG) |
 				  FW_LEN16(c));
 	c.u.l1cfg.rcap = htonl(FW_PORT_CAP_ANEG);
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
 struct intr_info {
 	unsigned int mask;       /* bits to check in interrupt status */
 	const char *msg;         /* message to print or NULL */
 	short stat_idx;          /* stat counter to increment or -1 */
 	unsigned short fatal;    /* whether the condition reported is fatal */
 };
 
 /**
  *	t4_handle_intr_status - table driven interrupt handler
  *	@adapter: the adapter that generated the interrupt
  *	@reg: the interrupt status register to process
  *	@acts: table of interrupt actions
  *
  *	A table driven interrupt handler that applies a set of masks to an
  *	interrupt status word and performs the corresponding actions if the
  *	interrupts described by the mask have occured.  The actions include
  *	optionally emitting a warning or alert message.  The table is terminated
  *	by an entry specifying mask 0.  Returns the number of fatal interrupt
  *	conditions.
  */
 static int t4_handle_intr_status(struct adapter *adapter, unsigned int reg,
 				 const struct intr_info *acts)
 {
 	int fatal = 0;
 	unsigned int mask = 0;
 	unsigned int status = t4_read_reg(adapter, reg);
 
 	for ( ; acts->mask; ++acts) {
 		if (!(status & acts->mask))
 			continue;
 		if (acts->fatal) {
 			fatal++;
 			CH_ALERT(adapter, "%s (0x%x)\n",
 				 acts->msg, status & acts->mask);
 		} else if (acts->msg)
 			CH_WARN_RATELIMIT(adapter, "%s (0x%x)\n",
 					  acts->msg, status & acts->mask);
 		mask |= acts->mask;
 	}
 	status &= mask;
 	if (status)                           /* clear processed interrupts */
 		t4_write_reg(adapter, reg, status);
 	return fatal;
 }
 
 /*
  * Interrupt handler for the PCIE module.
  */
 static void pcie_intr_handler(struct adapter *adapter)
 {
 	static struct intr_info sysbus_intr_info[] = {
 		{ F_RNPP, "RXNP array parity error", -1, 1 },
 		{ F_RPCP, "RXPC array parity error", -1, 1 },
 		{ F_RCIP, "RXCIF array parity error", -1, 1 },
 		{ F_RCCP, "Rx completions control array parity error", -1, 1 },
 		{ F_RFTP, "RXFT array parity error", -1, 1 },
 		{ 0 }
 	};
 	static struct intr_info pcie_port_intr_info[] = {
 		{ F_TPCP, "TXPC array parity error", -1, 1 },
 		{ F_TNPP, "TXNP array parity error", -1, 1 },
 		{ F_TFTP, "TXFT array parity error", -1, 1 },
 		{ F_TCAP, "TXCA array parity error", -1, 1 },
 		{ F_TCIP, "TXCIF array parity error", -1, 1 },
 		{ F_RCAP, "RXCA array parity error", -1, 1 },
 		{ F_OTDD, "outbound request TLP discarded", -1, 1 },
 		{ F_RDPE, "Rx data parity error", -1, 1 },
 		{ F_TDUE, "Tx uncorrectable data error", -1, 1 },
 		{ 0 }
 	};
 	static struct intr_info pcie_intr_info[] = {
 		{ F_MSIADDRLPERR, "MSI AddrL parity error", -1, 1 },
 		{ F_MSIADDRHPERR, "MSI AddrH parity error", -1, 1 },
 		{ F_MSIDATAPERR, "MSI data parity error", -1, 1 },
 		{ F_MSIXADDRLPERR, "MSI-X AddrL parity error", -1, 1 },
 		{ F_MSIXADDRHPERR, "MSI-X AddrH parity error", -1, 1 },
 		{ F_MSIXDATAPERR, "MSI-X data parity error", -1, 1 },
 		{ F_MSIXDIPERR, "MSI-X DI parity error", -1, 1 },
 		{ F_PIOCPLPERR, "PCI PIO completion FIFO parity error", -1, 1 },
 		{ F_PIOREQPERR, "PCI PIO request FIFO parity error", -1, 1 },
 		{ F_TARTAGPERR, "PCI PCI target tag FIFO parity error", -1, 1 },
 		{ F_CCNTPERR, "PCI CMD channel count parity error", -1, 1 },
 		{ F_CREQPERR, "PCI CMD channel request parity error", -1, 1 },
 		{ F_CRSPPERR, "PCI CMD channel response parity error", -1, 1 },
 		{ F_DCNTPERR, "PCI DMA channel count parity error", -1, 1 },
 		{ F_DREQPERR, "PCI DMA channel request parity error", -1, 1 },
 		{ F_DRSPPERR, "PCI DMA channel response parity error", -1, 1 },
 		{ F_HCNTPERR, "PCI HMA channel count parity error", -1, 1 },
 		{ F_HREQPERR, "PCI HMA channel request parity error", -1, 1 },
 		{ F_HRSPPERR, "PCI HMA channel response parity error", -1, 1 },
 		{ F_CFGSNPPERR, "PCI config snoop FIFO parity error", -1, 1 },
 		{ F_FIDPERR, "PCI FID parity error", -1, 1 },
 		{ F_INTXCLRPERR, "PCI INTx clear parity error", -1, 1 },
 		{ F_MATAGPERR, "PCI MA tag parity error", -1, 1 },
 		{ F_PIOTAGPERR, "PCI PIO tag parity error", -1, 1 },
 		{ F_RXCPLPERR, "PCI Rx completion parity error", -1, 1 },
 		{ F_RXWRPERR, "PCI Rx write parity error", -1, 1 },
 		{ F_RPLPERR, "PCI replay buffer parity error", -1, 1 },
 		{ F_PCIESINT, "PCI core secondary fault", -1, 1 },
 		{ F_PCIEPINT, "PCI core primary fault", -1, 1 },
 		{ F_UNXSPLCPLERR, "PCI unexpected split completion error", -1,
 		  0 },
 		{ 0 }
 	};
 
 	static struct intr_info t5_pcie_intr_info[] = {
 		{ F_MSTGRPPERR, "Master Response Read Queue parity error",
 		  -1, 1 },
 		{ F_MSTTIMEOUTPERR, "Master Timeout FIFO parity error", -1, 1 },
 		{ F_MSIXSTIPERR, "MSI-X STI SRAM parity error", -1, 1 },
 		{ F_MSIXADDRLPERR, "MSI-X AddrL parity error", -1, 1 },
 		{ F_MSIXADDRHPERR, "MSI-X AddrH parity error", -1, 1 },
 		{ F_MSIXDATAPERR, "MSI-X data parity error", -1, 1 },
 		{ F_MSIXDIPERR, "MSI-X DI parity error", -1, 1 },
 		{ F_PIOCPLGRPPERR, "PCI PIO completion Group FIFO parity error",
 		  -1, 1 },
 		{ F_PIOREQGRPPERR, "PCI PIO request Group FIFO parity error",
 		  -1, 1 },
 		{ F_TARTAGPERR, "PCI PCI target tag FIFO parity error", -1, 1 },
 		{ F_MSTTAGQPERR, "PCI master tag queue parity error", -1, 1 },
 		{ F_CREQPERR, "PCI CMD channel request parity error", -1, 1 },
 		{ F_CRSPPERR, "PCI CMD channel response parity error", -1, 1 },
 		{ F_DREQWRPERR, "PCI DMA channel write request parity error",
 		  -1, 1 },
 		{ F_DREQPERR, "PCI DMA channel request parity error", -1, 1 },
 		{ F_DRSPPERR, "PCI DMA channel response parity error", -1, 1 },
 		{ F_HREQWRPERR, "PCI HMA channel count parity error", -1, 1 },
 		{ F_HREQPERR, "PCI HMA channel request parity error", -1, 1 },
 		{ F_HRSPPERR, "PCI HMA channel response parity error", -1, 1 },
 		{ F_CFGSNPPERR, "PCI config snoop FIFO parity error", -1, 1 },
 		{ F_FIDPERR, "PCI FID parity error", -1, 1 },
 		{ F_VFIDPERR, "PCI INTx clear parity error", -1, 1 },
 		{ F_MAGRPPERR, "PCI MA group FIFO parity error", -1, 1 },
 		{ F_PIOTAGPERR, "PCI PIO tag parity error", -1, 1 },
 		{ F_IPRXHDRGRPPERR, "PCI IP Rx header group parity error",
 		  -1, 1 },
 		{ F_IPRXDATAGRPPERR, "PCI IP Rx data group parity error",
 		  -1, 1 },
 		{ F_RPLPERR, "PCI IP replay buffer parity error", -1, 1 },
 		{ F_IPSOTPERR, "PCI IP SOT buffer parity error", -1, 1 },
 		{ F_TRGT1GRPPERR, "PCI TRGT1 group FIFOs parity error", -1, 1 },
 		{ F_READRSPERR, "Outbound read error", -1,
 		  0 },
 		{ 0 }
 	};
 
 	int fat;
 
-	fat = t4_handle_intr_status(adapter,
-				    A_PCIE_CORE_UTL_SYSTEM_BUS_AGENT_STATUS,
-				    sysbus_intr_info) +
-	      t4_handle_intr_status(adapter,
-				    A_PCIE_CORE_UTL_PCI_EXPRESS_PORT_STATUS,
-				    pcie_port_intr_info) +
-	      t4_handle_intr_status(adapter, A_PCIE_INT_CAUSE,
-				    is_t4(adapter) ?
-				    pcie_intr_info : t5_pcie_intr_info);
+	if (is_t4(adapter))
+		fat = t4_handle_intr_status(adapter,
+					    A_PCIE_CORE_UTL_SYSTEM_BUS_AGENT_STATUS,
+					    sysbus_intr_info) +
+		      t4_handle_intr_status(adapter,
+					    A_PCIE_CORE_UTL_PCI_EXPRESS_PORT_STATUS,
+					    pcie_port_intr_info) +
+		      t4_handle_intr_status(adapter, A_PCIE_INT_CAUSE,
+					    pcie_intr_info);
+	else
+		fat = t4_handle_intr_status(adapter, A_PCIE_INT_CAUSE,
+					    t5_pcie_intr_info);
 	if (fat)
 		t4_fatal_err(adapter);
 }
 
 /*
  * TP interrupt handler.
  */
 static void tp_intr_handler(struct adapter *adapter)
 {
 	static struct intr_info tp_intr_info[] = {
 		{ 0x3fffffff, "TP parity error", -1, 1 },
 		{ F_FLMTXFLSTEMPTY, "TP out of Tx pages", -1, 1 },
 		{ 0 }
 	};
 
 	if (t4_handle_intr_status(adapter, A_TP_INT_CAUSE, tp_intr_info))
 		t4_fatal_err(adapter);
 }
 
 /*
  * SGE interrupt handler.
  */
 static void sge_intr_handler(struct adapter *adapter)
 {
 	u64 v;
 	u32 err;
 
 	static struct intr_info sge_intr_info[] = {
 		{ F_ERR_CPL_EXCEED_IQE_SIZE,
 		  "SGE received CPL exceeding IQE size", -1, 1 },
 		{ F_ERR_INVALID_CIDX_INC,
 		  "SGE GTS CIDX increment too large", -1, 0 },
 		{ F_ERR_CPL_OPCODE_0, "SGE received 0-length CPL", -1, 0 },
 		{ F_ERR_DROPPED_DB, "SGE doorbell dropped", -1, 0 },
 		{ F_ERR_DATA_CPL_ON_HIGH_QID1 | F_ERR_DATA_CPL_ON_HIGH_QID0,
 		  "SGE IQID > 1023 received CPL for FL", -1, 0 },
 		{ F_ERR_BAD_DB_PIDX3, "SGE DBP 3 pidx increment too large", -1,
 		  0 },
 		{ F_ERR_BAD_DB_PIDX2, "SGE DBP 2 pidx increment too large", -1,
 		  0 },
 		{ F_ERR_BAD_DB_PIDX1, "SGE DBP 1 pidx increment too large", -1,
 		  0 },
 		{ F_ERR_BAD_DB_PIDX0, "SGE DBP 0 pidx increment too large", -1,
 		  0 },
 		{ F_ERR_ING_CTXT_PRIO,
 		  "SGE too many priority ingress contexts", -1, 0 },
 		{ F_ERR_EGR_CTXT_PRIO,
 		  "SGE too many priority egress contexts", -1, 0 },
 		{ F_INGRESS_SIZE_ERR, "SGE illegal ingress QID", -1, 0 },
 		{ F_EGRESS_SIZE_ERR, "SGE illegal egress QID", -1, 0 },
 		{ 0 }
 	};
 
 	v = (u64)t4_read_reg(adapter, A_SGE_INT_CAUSE1) |
 	    ((u64)t4_read_reg(adapter, A_SGE_INT_CAUSE2) << 32);
 	if (v) {
 		CH_ALERT(adapter, "SGE parity error (%#llx)\n",
 			 (unsigned long long)v);
 		t4_write_reg(adapter, A_SGE_INT_CAUSE1, v);
 		t4_write_reg(adapter, A_SGE_INT_CAUSE2, v >> 32);
 	}
 
 	v |= t4_handle_intr_status(adapter, A_SGE_INT_CAUSE3, sge_intr_info);
 
 	err = t4_read_reg(adapter, A_SGE_ERROR_STATS);
 	if (err & F_ERROR_QID_VALID) {
 		CH_ERR(adapter, "SGE error for queue %u\n", G_ERROR_QID(err));
 		if (err & F_UNCAPTURED_ERROR)
 			CH_ERR(adapter, "SGE UNCAPTURED_ERROR set (clearing)\n");
 		t4_write_reg(adapter, A_SGE_ERROR_STATS, F_ERROR_QID_VALID |
 			     F_UNCAPTURED_ERROR);
 	}
 
 	if (v != 0)
 		t4_fatal_err(adapter);
 }
 
 #define CIM_OBQ_INTR (F_OBQULP0PARERR | F_OBQULP1PARERR | F_OBQULP2PARERR |\
 		      F_OBQULP3PARERR | F_OBQSGEPARERR | F_OBQNCSIPARERR)
 #define CIM_IBQ_INTR (F_IBQTP0PARERR | F_IBQTP1PARERR | F_IBQULPPARERR |\
 		      F_IBQSGEHIPARERR | F_IBQSGELOPARERR | F_IBQNCSIPARERR)
 
 /*
  * CIM interrupt handler.
  */
 static void cim_intr_handler(struct adapter *adapter)
 {
 	static struct intr_info cim_intr_info[] = {
 		{ F_PREFDROPINT, "CIM control register prefetch drop", -1, 1 },
 		{ CIM_OBQ_INTR, "CIM OBQ parity error", -1, 1 },
 		{ CIM_IBQ_INTR, "CIM IBQ parity error", -1, 1 },
 		{ F_MBUPPARERR, "CIM mailbox uP parity error", -1, 1 },
 		{ F_MBHOSTPARERR, "CIM mailbox host parity error", -1, 1 },
 		{ F_TIEQINPARERRINT, "CIM TIEQ outgoing parity error", -1, 1 },
 		{ F_TIEQOUTPARERRINT, "CIM TIEQ incoming parity error", -1, 1 },
 		{ 0 }
 	};
 	static struct intr_info cim_upintr_info[] = {
 		{ F_RSVDSPACEINT, "CIM reserved space access", -1, 1 },
 		{ F_ILLTRANSINT, "CIM illegal transaction", -1, 1 },
 		{ F_ILLWRINT, "CIM illegal write", -1, 1 },
 		{ F_ILLRDINT, "CIM illegal read", -1, 1 },
 		{ F_ILLRDBEINT, "CIM illegal read BE", -1, 1 },
 		{ F_ILLWRBEINT, "CIM illegal write BE", -1, 1 },
 		{ F_SGLRDBOOTINT, "CIM single read from boot space", -1, 1 },
 		{ F_SGLWRBOOTINT, "CIM single write to boot space", -1, 1 },
 		{ F_BLKWRBOOTINT, "CIM block write to boot space", -1, 1 },
 		{ F_SGLRDFLASHINT, "CIM single read from flash space", -1, 1 },
 		{ F_SGLWRFLASHINT, "CIM single write to flash space", -1, 1 },
 		{ F_BLKWRFLASHINT, "CIM block write to flash space", -1, 1 },
 		{ F_SGLRDEEPROMINT, "CIM single EEPROM read", -1, 1 },
 		{ F_SGLWREEPROMINT, "CIM single EEPROM write", -1, 1 },
 		{ F_BLKRDEEPROMINT, "CIM block EEPROM read", -1, 1 },
 		{ F_BLKWREEPROMINT, "CIM block EEPROM write", -1, 1 },
 		{ F_SGLRDCTLINT , "CIM single read from CTL space", -1, 1 },
 		{ F_SGLWRCTLINT , "CIM single write to CTL space", -1, 1 },
 		{ F_BLKRDCTLINT , "CIM block read from CTL space", -1, 1 },
 		{ F_BLKWRCTLINT , "CIM block write to CTL space", -1, 1 },
 		{ F_SGLRDPLINT , "CIM single read from PL space", -1, 1 },
 		{ F_SGLWRPLINT , "CIM single write to PL space", -1, 1 },
 		{ F_BLKRDPLINT , "CIM block read from PL space", -1, 1 },
 		{ F_BLKWRPLINT , "CIM block write to PL space", -1, 1 },
 		{ F_REQOVRLOOKUPINT , "CIM request FIFO overwrite", -1, 1 },
 		{ F_RSPOVRLOOKUPINT , "CIM response FIFO overwrite", -1, 1 },
 		{ F_TIMEOUTINT , "CIM PIF timeout", -1, 1 },
 		{ F_TIMEOUTMAINT , "CIM PIF MA timeout", -1, 1 },
 		{ 0 }
 	};
 	int fat;
 
 	if (t4_read_reg(adapter, A_PCIE_FW) & F_PCIE_FW_ERR)
 		t4_report_fw_error(adapter);
 
 	fat = t4_handle_intr_status(adapter, A_CIM_HOST_INT_CAUSE,
 				    cim_intr_info) +
 	      t4_handle_intr_status(adapter, A_CIM_HOST_UPACC_INT_CAUSE,
 				    cim_upintr_info);
 	if (fat)
 		t4_fatal_err(adapter);
 }
 
 /*
  * ULP RX interrupt handler.
  */
 static void ulprx_intr_handler(struct adapter *adapter)
 {
 	static struct intr_info ulprx_intr_info[] = {
 		{ F_CAUSE_CTX_1, "ULPRX channel 1 context error", -1, 1 },
 		{ F_CAUSE_CTX_0, "ULPRX channel 0 context error", -1, 1 },
 		{ 0x7fffff, "ULPRX parity error", -1, 1 },
 		{ 0 }
 	};
 
 	if (t4_handle_intr_status(adapter, A_ULP_RX_INT_CAUSE, ulprx_intr_info))
 		t4_fatal_err(adapter);
 }
 
 /*
  * ULP TX interrupt handler.
  */
 static void ulptx_intr_handler(struct adapter *adapter)
 {
 	static struct intr_info ulptx_intr_info[] = {
 		{ F_PBL_BOUND_ERR_CH3, "ULPTX channel 3 PBL out of bounds", -1,
 		  0 },
 		{ F_PBL_BOUND_ERR_CH2, "ULPTX channel 2 PBL out of bounds", -1,
 		  0 },
 		{ F_PBL_BOUND_ERR_CH1, "ULPTX channel 1 PBL out of bounds", -1,
 		  0 },
 		{ F_PBL_BOUND_ERR_CH0, "ULPTX channel 0 PBL out of bounds", -1,
 		  0 },
 		{ 0xfffffff, "ULPTX parity error", -1, 1 },
 		{ 0 }
 	};
 
 	if (t4_handle_intr_status(adapter, A_ULP_TX_INT_CAUSE, ulptx_intr_info))
 		t4_fatal_err(adapter);
 }
 
 /*
  * PM TX interrupt handler.
  */
 static void pmtx_intr_handler(struct adapter *adapter)
 {
 	static struct intr_info pmtx_intr_info[] = {
 		{ F_PCMD_LEN_OVFL0, "PMTX channel 0 pcmd too large", -1, 1 },
 		{ F_PCMD_LEN_OVFL1, "PMTX channel 1 pcmd too large", -1, 1 },
 		{ F_PCMD_LEN_OVFL2, "PMTX channel 2 pcmd too large", -1, 1 },
 		{ F_ZERO_C_CMD_ERROR, "PMTX 0-length pcmd", -1, 1 },
 		{ 0xffffff0, "PMTX framing error", -1, 1 },
 		{ F_OESPI_PAR_ERROR, "PMTX oespi parity error", -1, 1 },
 		{ F_DB_OPTIONS_PAR_ERROR, "PMTX db_options parity error", -1,
 		  1 },
 		{ F_ICSPI_PAR_ERROR, "PMTX icspi parity error", -1, 1 },
 		{ F_C_PCMD_PAR_ERROR, "PMTX c_pcmd parity error", -1, 1},
 		{ 0 }
 	};
 
 	if (t4_handle_intr_status(adapter, A_PM_TX_INT_CAUSE, pmtx_intr_info))
 		t4_fatal_err(adapter);
 }
 
 /*
  * PM RX interrupt handler.
  */
 static void pmrx_intr_handler(struct adapter *adapter)
 {
 	static struct intr_info pmrx_intr_info[] = {
 		{ F_ZERO_E_CMD_ERROR, "PMRX 0-length pcmd", -1, 1 },
 		{ 0x3ffff0, "PMRX framing error", -1, 1 },
 		{ F_OCSPI_PAR_ERROR, "PMRX ocspi parity error", -1, 1 },
 		{ F_DB_OPTIONS_PAR_ERROR, "PMRX db_options parity error", -1,
 		  1 },
 		{ F_IESPI_PAR_ERROR, "PMRX iespi parity error", -1, 1 },
 		{ F_E_PCMD_PAR_ERROR, "PMRX e_pcmd parity error", -1, 1},
 		{ 0 }
 	};
 
 	if (t4_handle_intr_status(adapter, A_PM_RX_INT_CAUSE, pmrx_intr_info))
 		t4_fatal_err(adapter);
 }
 
 /*
  * CPL switch interrupt handler.
  */
 static void cplsw_intr_handler(struct adapter *adapter)
 {
 	static struct intr_info cplsw_intr_info[] = {
 		{ F_CIM_OP_MAP_PERR, "CPLSW CIM op_map parity error", -1, 1 },
 		{ F_CIM_OVFL_ERROR, "CPLSW CIM overflow", -1, 1 },
 		{ F_TP_FRAMING_ERROR, "CPLSW TP framing error", -1, 1 },
 		{ F_SGE_FRAMING_ERROR, "CPLSW SGE framing error", -1, 1 },
 		{ F_CIM_FRAMING_ERROR, "CPLSW CIM framing error", -1, 1 },
 		{ F_ZERO_SWITCH_ERROR, "CPLSW no-switch error", -1, 1 },
 		{ 0 }
 	};
 
 	if (t4_handle_intr_status(adapter, A_CPL_INTR_CAUSE, cplsw_intr_info))
 		t4_fatal_err(adapter);
 }
 
 /*
  * LE interrupt handler.
  */
 static void le_intr_handler(struct adapter *adap)
 {
 	static struct intr_info le_intr_info[] = {
 		{ F_LIPMISS, "LE LIP miss", -1, 0 },
 		{ F_LIP0, "LE 0 LIP error", -1, 0 },
 		{ F_PARITYERR, "LE parity error", -1, 1 },
 		{ F_UNKNOWNCMD, "LE unknown command", -1, 1 },
 		{ F_REQQPARERR, "LE request queue parity error", -1, 1 },
 		{ 0 }
 	};
 
 	if (t4_handle_intr_status(adap, A_LE_DB_INT_CAUSE, le_intr_info))
 		t4_fatal_err(adap);
 }
 
 /*
  * MPS interrupt handler.
  */
 static void mps_intr_handler(struct adapter *adapter)
 {
 	static struct intr_info mps_rx_intr_info[] = {
 		{ 0xffffff, "MPS Rx parity error", -1, 1 },
 		{ 0 }
 	};
 	static struct intr_info mps_tx_intr_info[] = {
 		{ V_TPFIFO(M_TPFIFO), "MPS Tx TP FIFO parity error", -1, 1 },
 		{ F_NCSIFIFO, "MPS Tx NC-SI FIFO parity error", -1, 1 },
 		{ V_TXDATAFIFO(M_TXDATAFIFO), "MPS Tx data FIFO parity error",
 		  -1, 1 },
 		{ V_TXDESCFIFO(M_TXDESCFIFO), "MPS Tx desc FIFO parity error",
 		  -1, 1 },
 		{ F_BUBBLE, "MPS Tx underflow", -1, 1 },
 		{ F_SECNTERR, "MPS Tx SOP/EOP error", -1, 1 },
 		{ F_FRMERR, "MPS Tx framing error", -1, 1 },
 		{ 0 }
 	};
 	static struct intr_info mps_trc_intr_info[] = {
 		{ V_FILTMEM(M_FILTMEM), "MPS TRC filter parity error", -1, 1 },
 		{ V_PKTFIFO(M_PKTFIFO), "MPS TRC packet FIFO parity error", -1,
 		  1 },
 		{ F_MISCPERR, "MPS TRC misc parity error", -1, 1 },
 		{ 0 }
 	};
 	static struct intr_info mps_stat_sram_intr_info[] = {
 		{ 0x1fffff, "MPS statistics SRAM parity error", -1, 1 },
 		{ 0 }
 	};
 	static struct intr_info mps_stat_tx_intr_info[] = {
 		{ 0xfffff, "MPS statistics Tx FIFO parity error", -1, 1 },
 		{ 0 }
 	};
 	static struct intr_info mps_stat_rx_intr_info[] = {
 		{ 0xffffff, "MPS statistics Rx FIFO parity error", -1, 1 },
 		{ 0 }
 	};
 	static struct intr_info mps_cls_intr_info[] = {
 		{ F_MATCHSRAM, "MPS match SRAM parity error", -1, 1 },
 		{ F_MATCHTCAM, "MPS match TCAM parity error", -1, 1 },
 		{ F_HASHSRAM, "MPS hash SRAM parity error", -1, 1 },
 		{ 0 }
 	};
 
 	int fat;
 
 	fat = t4_handle_intr_status(adapter, A_MPS_RX_PERR_INT_CAUSE,
 				    mps_rx_intr_info) +
 	      t4_handle_intr_status(adapter, A_MPS_TX_INT_CAUSE,
 				    mps_tx_intr_info) +
 	      t4_handle_intr_status(adapter, A_MPS_TRC_INT_CAUSE,
 				    mps_trc_intr_info) +
 	      t4_handle_intr_status(adapter, A_MPS_STAT_PERR_INT_CAUSE_SRAM,
 				    mps_stat_sram_intr_info) +
 	      t4_handle_intr_status(adapter, A_MPS_STAT_PERR_INT_CAUSE_TX_FIFO,
 				    mps_stat_tx_intr_info) +
 	      t4_handle_intr_status(adapter, A_MPS_STAT_PERR_INT_CAUSE_RX_FIFO,
 				    mps_stat_rx_intr_info) +
 	      t4_handle_intr_status(adapter, A_MPS_CLS_INT_CAUSE,
 				    mps_cls_intr_info);
 
 	t4_write_reg(adapter, A_MPS_INT_CAUSE, 0);
 	t4_read_reg(adapter, A_MPS_INT_CAUSE);                    /* flush */
 	if (fat)
 		t4_fatal_err(adapter);
 }
 
 #define MEM_INT_MASK (F_PERR_INT_CAUSE | F_ECC_CE_INT_CAUSE | F_ECC_UE_INT_CAUSE)
 
 /*
  * EDC/MC interrupt handler.
  */
 static void mem_intr_handler(struct adapter *adapter, int idx)
 {
 	static const char name[3][5] = { "EDC0", "EDC1", "MC" };
 
 	unsigned int addr, cnt_addr, v;
 
 	if (idx <= MEM_EDC1) {
 		addr = EDC_REG(A_EDC_INT_CAUSE, idx);
 		cnt_addr = EDC_REG(A_EDC_ECC_STATUS, idx);
 	} else {
 		if (is_t4(adapter)) {
 			addr = A_MC_INT_CAUSE;
 			cnt_addr = A_MC_ECC_STATUS;
 		} else {
 			addr = A_MC_P_INT_CAUSE;
 			cnt_addr = A_MC_P_ECC_STATUS;
 		}
 	}
 
 	v = t4_read_reg(adapter, addr) & MEM_INT_MASK;
 	if (v & F_PERR_INT_CAUSE)
 		CH_ALERT(adapter, "%s FIFO parity error\n", name[idx]);
 	if (v & F_ECC_CE_INT_CAUSE) {
 		u32 cnt = G_ECC_CECNT(t4_read_reg(adapter, cnt_addr));
 
 		t4_write_reg(adapter, cnt_addr, V_ECC_CECNT(M_ECC_CECNT));
 		CH_WARN_RATELIMIT(adapter,
 				  "%u %s correctable ECC data error%s\n",
 				  cnt, name[idx], cnt > 1 ? "s" : "");
 	}
 	if (v & F_ECC_UE_INT_CAUSE)
 		CH_ALERT(adapter, "%s uncorrectable ECC data error\n",
 			 name[idx]);
 
 	t4_write_reg(adapter, addr, v);
 	if (v & (F_PERR_INT_CAUSE | F_ECC_UE_INT_CAUSE))
 		t4_fatal_err(adapter);
 }
 
 /*
  * MA interrupt handler.
  */
 static void ma_intr_handler(struct adapter *adapter)
 {
 	u32 v, status = t4_read_reg(adapter, A_MA_INT_CAUSE);
 
-	if (status & F_MEM_PERR_INT_CAUSE)
+	if (status & F_MEM_PERR_INT_CAUSE) {
 		CH_ALERT(adapter, "MA parity error, parity status %#x\n",
-			 t4_read_reg(adapter, A_MA_PARITY_ERROR_STATUS));
+			 t4_read_reg(adapter, A_MA_PARITY_ERROR_STATUS1));
+		if (is_t5(adapter))
+			CH_ALERT(adapter,
+				 "MA parity error, parity status %#x\n",
+				 t4_read_reg(adapter,
+				 	     A_MA_PARITY_ERROR_STATUS2));
+	}
 	if (status & F_MEM_WRAP_INT_CAUSE) {
 		v = t4_read_reg(adapter, A_MA_INT_WRAP_STATUS);
 		CH_ALERT(adapter, "MA address wrap-around error by client %u to"
 			 " address %#x\n", G_MEM_WRAP_CLIENT_NUM(v),
 			 G_MEM_WRAP_ADDRESS(v) << 4);
 	}
 	t4_write_reg(adapter, A_MA_INT_CAUSE, status);
 	t4_fatal_err(adapter);
 }
 
 /*
  * SMB interrupt handler.
  */
 static void smb_intr_handler(struct adapter *adap)
 {
 	static struct intr_info smb_intr_info[] = {
 		{ F_MSTTXFIFOPARINT, "SMB master Tx FIFO parity error", -1, 1 },
 		{ F_MSTRXFIFOPARINT, "SMB master Rx FIFO parity error", -1, 1 },
 		{ F_SLVFIFOPARINT, "SMB slave FIFO parity error", -1, 1 },
 		{ 0 }
 	};
 
 	if (t4_handle_intr_status(adap, A_SMB_INT_CAUSE, smb_intr_info))
 		t4_fatal_err(adap);
 }
 
 /*
  * NC-SI interrupt handler.
  */
 static void ncsi_intr_handler(struct adapter *adap)
 {
 	static struct intr_info ncsi_intr_info[] = {
 		{ F_CIM_DM_PRTY_ERR, "NC-SI CIM parity error", -1, 1 },
 		{ F_MPS_DM_PRTY_ERR, "NC-SI MPS parity error", -1, 1 },
 		{ F_TXFIFO_PRTY_ERR, "NC-SI Tx FIFO parity error", -1, 1 },
 		{ F_RXFIFO_PRTY_ERR, "NC-SI Rx FIFO parity error", -1, 1 },
 		{ 0 }
 	};
 
 	if (t4_handle_intr_status(adap, A_NCSI_INT_CAUSE, ncsi_intr_info))
 		t4_fatal_err(adap);
 }
 
 /*
  * XGMAC interrupt handler.
  */
 static void xgmac_intr_handler(struct adapter *adap, int port)
 {
 	u32 v, int_cause_reg;
 
 	if (is_t4(adap))
 		int_cause_reg = PORT_REG(port, A_XGMAC_PORT_INT_CAUSE);
 	else
 		int_cause_reg = T5_PORT_REG(port, A_MAC_PORT_INT_CAUSE);
 
 	v = t4_read_reg(adap, int_cause_reg);
 	v &= (F_TXFIFO_PRTY_ERR | F_RXFIFO_PRTY_ERR);
 	if (!v)
 		return;
 
 	if (v & F_TXFIFO_PRTY_ERR)
 		CH_ALERT(adap, "XGMAC %d Tx FIFO parity error\n", port);
 	if (v & F_RXFIFO_PRTY_ERR)
 		CH_ALERT(adap, "XGMAC %d Rx FIFO parity error\n", port);
 	t4_write_reg(adap, int_cause_reg, v);
 	t4_fatal_err(adap);
 }
 
 /*
  * PL interrupt handler.
  */
 static void pl_intr_handler(struct adapter *adap)
 {
 	static struct intr_info pl_intr_info[] = {
 		{ F_FATALPERR, "Fatal parity error", -1, 1 },
 		{ F_PERRVFID, "PL VFID_MAP parity error", -1, 1 },
 		{ 0 }
 	};
 
 	static struct intr_info t5_pl_intr_info[] = {
 		{ F_PL_BUSPERR, "PL bus parity error", -1, 1 },
 		{ F_FATALPERR, "Fatal parity error", -1, 1 },
 		{ 0 }
 	};
 
 	if (t4_handle_intr_status(adap, A_PL_PL_INT_CAUSE,
 	    is_t4(adap) ?  pl_intr_info : t5_pl_intr_info))
 		t4_fatal_err(adap);
 }
 
 #define PF_INTR_MASK (F_PFSW | F_PFCIM)
 #define GLBL_INTR_MASK (F_CIM | F_MPS | F_PL | F_PCIE | F_MC | F_EDC0 | \
 		F_EDC1 | F_LE | F_TP | F_MA | F_PM_TX | F_PM_RX | F_ULP_RX | \
 		F_CPL_SWITCH | F_SGE | F_ULP_TX)
 
 /**
  *	t4_slow_intr_handler - control path interrupt handler
  *	@adapter: the adapter
  *
  *	T4 interrupt handler for non-data global interrupt events, e.g., errors.
  *	The designation 'slow' is because it involves register reads, while
  *	data interrupts typically don't involve any MMIOs.
  */
 int t4_slow_intr_handler(struct adapter *adapter)
 {
 	u32 cause = t4_read_reg(adapter, A_PL_INT_CAUSE);
 
 	if (!(cause & GLBL_INTR_MASK))
 		return 0;
 	if (cause & F_CIM)
 		cim_intr_handler(adapter);
 	if (cause & F_MPS)
 		mps_intr_handler(adapter);
 	if (cause & F_NCSI)
 		ncsi_intr_handler(adapter);
 	if (cause & F_PL)
 		pl_intr_handler(adapter);
 	if (cause & F_SMB)
 		smb_intr_handler(adapter);
 	if (cause & F_XGMAC0)
 		xgmac_intr_handler(adapter, 0);
 	if (cause & F_XGMAC1)
 		xgmac_intr_handler(adapter, 1);
 	if (cause & F_XGMAC_KR0)
 		xgmac_intr_handler(adapter, 2);
 	if (cause & F_XGMAC_KR1)
 		xgmac_intr_handler(adapter, 3);
 	if (cause & F_PCIE)
 		pcie_intr_handler(adapter);
 	if (cause & F_MC)
 		mem_intr_handler(adapter, MEM_MC);
 	if (cause & F_EDC0)
 		mem_intr_handler(adapter, MEM_EDC0);
 	if (cause & F_EDC1)
 		mem_intr_handler(adapter, MEM_EDC1);
 	if (cause & F_LE)
 		le_intr_handler(adapter);
 	if (cause & F_TP)
 		tp_intr_handler(adapter);
 	if (cause & F_MA)
 		ma_intr_handler(adapter);
 	if (cause & F_PM_TX)
 		pmtx_intr_handler(adapter);
 	if (cause & F_PM_RX)
 		pmrx_intr_handler(adapter);
 	if (cause & F_ULP_RX)
 		ulprx_intr_handler(adapter);
 	if (cause & F_CPL_SWITCH)
 		cplsw_intr_handler(adapter);
 	if (cause & F_SGE)
 		sge_intr_handler(adapter);
 	if (cause & F_ULP_TX)
 		ulptx_intr_handler(adapter);
 
 	/* Clear the interrupts just processed for which we are the master. */
 	t4_write_reg(adapter, A_PL_INT_CAUSE, cause & GLBL_INTR_MASK);
 	(void) t4_read_reg(adapter, A_PL_INT_CAUSE); /* flush */
 	return 1;
 }
 
 /**
  *	t4_intr_enable - enable interrupts
  *	@adapter: the adapter whose interrupts should be enabled
  *
  *	Enable PF-specific interrupts for the calling function and the top-level
  *	interrupt concentrator for global interrupts.  Interrupts are already
  *	enabled at each module,	here we just enable the roots of the interrupt
  *	hierarchies.
  *
  *	Note: this function should be called only when the driver manages
  *	non PF-specific interrupts from the various HW modules.  Only one PCI
  *	function at a time should be doing this.
  */
 void t4_intr_enable(struct adapter *adapter)
 {
 	u32 pf = G_SOURCEPF(t4_read_reg(adapter, A_PL_WHOAMI));
 
 	t4_write_reg(adapter, A_SGE_INT_ENABLE3, F_ERR_CPL_EXCEED_IQE_SIZE |
 		     F_ERR_INVALID_CIDX_INC | F_ERR_CPL_OPCODE_0 |
 		     F_ERR_DROPPED_DB | F_ERR_DATA_CPL_ON_HIGH_QID1 |
 		     F_ERR_DATA_CPL_ON_HIGH_QID0 | F_ERR_BAD_DB_PIDX3 |
 		     F_ERR_BAD_DB_PIDX2 | F_ERR_BAD_DB_PIDX1 |
 		     F_ERR_BAD_DB_PIDX0 | F_ERR_ING_CTXT_PRIO |
 		     F_ERR_EGR_CTXT_PRIO | F_INGRESS_SIZE_ERR |
 		     F_EGRESS_SIZE_ERR);
 	t4_write_reg(adapter, MYPF_REG(A_PL_PF_INT_ENABLE), PF_INTR_MASK);
 	t4_set_reg_field(adapter, A_PL_INT_MAP0, 0, 1 << pf);
 }
 
 /**
  *	t4_intr_disable - disable interrupts
  *	@adapter: the adapter whose interrupts should be disabled
  *
  *	Disable interrupts.  We only disable the top-level interrupt
  *	concentrators.  The caller must be a PCI function managing global
  *	interrupts.
  */
 void t4_intr_disable(struct adapter *adapter)
 {
 	u32 pf = G_SOURCEPF(t4_read_reg(adapter, A_PL_WHOAMI));
 
 	t4_write_reg(adapter, MYPF_REG(A_PL_PF_INT_ENABLE), 0);
 	t4_set_reg_field(adapter, A_PL_INT_MAP0, 1 << pf, 0);
 }
 
 /**
  *	t4_intr_clear - clear all interrupts
  *	@adapter: the adapter whose interrupts should be cleared
  *
  *	Clears all interrupts.  The caller must be a PCI function managing
  *	global interrupts.
  */
 void t4_intr_clear(struct adapter *adapter)
 {
 	static const unsigned int cause_reg[] = {
 		A_SGE_INT_CAUSE1, A_SGE_INT_CAUSE2, A_SGE_INT_CAUSE3,
-		A_PCIE_CORE_UTL_SYSTEM_BUS_AGENT_STATUS,
-		A_PCIE_CORE_UTL_PCI_EXPRESS_PORT_STATUS,
 		A_PCIE_NONFAT_ERR, A_PCIE_INT_CAUSE,
-		A_MA_INT_WRAP_STATUS, A_MA_PARITY_ERROR_STATUS, A_MA_INT_CAUSE,
+		A_MA_INT_WRAP_STATUS, A_MA_PARITY_ERROR_STATUS1, A_MA_INT_CAUSE,
 		A_EDC_INT_CAUSE, EDC_REG(A_EDC_INT_CAUSE, 1),
 		A_CIM_HOST_INT_CAUSE, A_CIM_HOST_UPACC_INT_CAUSE,
 		MYPF_REG(A_CIM_PF_HOST_INT_CAUSE),
 		A_TP_INT_CAUSE,
 		A_ULP_RX_INT_CAUSE, A_ULP_TX_INT_CAUSE,
 		A_PM_RX_INT_CAUSE, A_PM_TX_INT_CAUSE,
 		A_MPS_RX_PERR_INT_CAUSE,
 		A_CPL_INTR_CAUSE,
 		MYPF_REG(A_PL_PF_INT_CAUSE),
 		A_PL_PL_INT_CAUSE,
 		A_LE_DB_INT_CAUSE,
 	};
 
 	unsigned int i;
 
 	for (i = 0; i < ARRAY_SIZE(cause_reg); ++i)
 		t4_write_reg(adapter, cause_reg[i], 0xffffffff);
 
 	t4_write_reg(adapter, is_t4(adapter) ? A_MC_INT_CAUSE :
 				A_MC_P_INT_CAUSE, 0xffffffff);
 
+	if (is_t4(adapter)) {
+		t4_write_reg(adapter, A_PCIE_CORE_UTL_SYSTEM_BUS_AGENT_STATUS,
+				0xffffffff);
+		t4_write_reg(adapter, A_PCIE_CORE_UTL_PCI_EXPRESS_PORT_STATUS,
+				0xffffffff);
+	} else
+		t4_write_reg(adapter, A_MA_PARITY_ERROR_STATUS2, 0xffffffff);
+
 	t4_write_reg(adapter, A_PL_INT_CAUSE, GLBL_INTR_MASK);
 	(void) t4_read_reg(adapter, A_PL_INT_CAUSE);          /* flush */
 }
 
 /**
  *	hash_mac_addr - return the hash value of a MAC address
  *	@addr: the 48-bit Ethernet MAC address
  *
  *	Hashes a MAC address according to the hash function used by HW inexact
  *	(hash) address matching.
  */
 static int hash_mac_addr(const u8 *addr)
 {
 	u32 a = ((u32)addr[0] << 16) | ((u32)addr[1] << 8) | addr[2];
 	u32 b = ((u32)addr[3] << 16) | ((u32)addr[4] << 8) | addr[5];
 	a ^= b;
 	a ^= (a >> 12);
 	a ^= (a >> 6);
 	return a & 0x3f;
 }
 
 /**
  *	t4_config_rss_range - configure a portion of the RSS mapping table
  *	@adapter: the adapter
  *	@mbox: mbox to use for the FW command
  *	@viid: virtual interface whose RSS subtable is to be written
  *	@start: start entry in the table to write
  *	@n: how many table entries to write
  *	@rspq: values for the "response queue" (Ingress Queue) lookup table
  *	@nrspq: number of values in @rspq
  *
  *	Programs the selected part of the VI's RSS mapping table with the
  *	provided values.  If @nrspq < @n the supplied values are used repeatedly
  *	until the full table range is populated.
  *
  *	The caller must ensure the values in @rspq are in the range allowed for
  *	@viid.
  */
 int t4_config_rss_range(struct adapter *adapter, int mbox, unsigned int viid,
 			int start, int n, const u16 *rspq, unsigned int nrspq)
 {
 	int ret;
 	const u16 *rsp = rspq;
 	const u16 *rsp_end = rspq + nrspq;
 	struct fw_rss_ind_tbl_cmd cmd;
 
 	memset(&cmd, 0, sizeof(cmd));
 	cmd.op_to_viid = htonl(V_FW_CMD_OP(FW_RSS_IND_TBL_CMD) |
 			       F_FW_CMD_REQUEST | F_FW_CMD_WRITE |
 			       V_FW_RSS_IND_TBL_CMD_VIID(viid));
 	cmd.retval_len16 = htonl(FW_LEN16(cmd));
 
 
 	/*
 	 * Each firmware RSS command can accommodate up to 32 RSS Ingress
 	 * Queue Identifiers.  These Ingress Queue IDs are packed three to
 	 * a 32-bit word as 10-bit values with the upper remaining 2 bits
 	 * reserved.
 	 */
 	while (n > 0) {
 		int nq = min(n, 32);
 		int nq_packed = 0;
 		__be32 *qp = &cmd.iq0_to_iq2;
 
 		/*
 		 * Set up the firmware RSS command header to send the next
 		 * "nq" Ingress Queue IDs to the firmware.
 		 */
 		cmd.niqid = htons(nq);
 		cmd.startidx = htons(start);
 
 		/*
 		 * "nq" more done for the start of the next loop.
 		 */
 		start += nq;
 		n -= nq;
 
 		/*
 		 * While there are still Ingress Queue IDs to stuff into the
 		 * current firmware RSS command, retrieve them from the
 		 * Ingress Queue ID array and insert them into the command.
 		 */
 		while (nq > 0) {
 			/*
 			 * Grab up to the next 3 Ingress Queue IDs (wrapping
 			 * around the Ingress Queue ID array if necessary) and
 			 * insert them into the firmware RSS command at the
 			 * current 3-tuple position within the commad.
 			 */
 			u16 qbuf[3];
 			u16 *qbp = qbuf;
 			int nqbuf = min(3, nq);
 
 			nq -= nqbuf;
 			qbuf[0] = qbuf[1] = qbuf[2] = 0;
 			while (nqbuf && nq_packed < 32) {
 				nqbuf--;
 				nq_packed++;
 				*qbp++ = *rsp++;
 				if (rsp >= rsp_end)
 					rsp = rspq;
 			}
 			*qp++ = cpu_to_be32(V_FW_RSS_IND_TBL_CMD_IQ0(qbuf[0]) |
 					    V_FW_RSS_IND_TBL_CMD_IQ1(qbuf[1]) |
 					    V_FW_RSS_IND_TBL_CMD_IQ2(qbuf[2]));
 		}
 
 		/*
 		 * Send this portion of the RRS table update to the firmware;
 		 * bail out on any errors.
 		 */
 		ret = t4_wr_mbox(adapter, mbox, &cmd, sizeof(cmd), NULL);
 		if (ret)
 			return ret;
 	}
 
 	return 0;
 }
 
 /**
  *	t4_config_glbl_rss - configure the global RSS mode
  *	@adapter: the adapter
  *	@mbox: mbox to use for the FW command
  *	@mode: global RSS mode
  *	@flags: mode-specific flags
  *
  *	Sets the global RSS mode.
  */
 int t4_config_glbl_rss(struct adapter *adapter, int mbox, unsigned int mode,
 		       unsigned int flags)
 {
 	struct fw_rss_glb_config_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_write = htonl(V_FW_CMD_OP(FW_RSS_GLB_CONFIG_CMD) |
 			      F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
 	c.retval_len16 = htonl(FW_LEN16(c));
 	if (mode == FW_RSS_GLB_CONFIG_CMD_MODE_MANUAL) {
 		c.u.manual.mode_pkd = htonl(V_FW_RSS_GLB_CONFIG_CMD_MODE(mode));
 	} else if (mode == FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) {
 		c.u.basicvirtual.mode_pkd =
 			htonl(V_FW_RSS_GLB_CONFIG_CMD_MODE(mode));
 		c.u.basicvirtual.synmapen_to_hashtoeplitz = htonl(flags);
 	} else
 		return -EINVAL;
 	return t4_wr_mbox(adapter, mbox, &c, sizeof(c), NULL);
 }
 
 /**
  *	t4_config_vi_rss - configure per VI RSS settings
  *	@adapter: the adapter
  *	@mbox: mbox to use for the FW command
  *	@viid: the VI id
  *	@flags: RSS flags
  *	@defq: id of the default RSS queue for the VI.
  *
  *	Configures VI-specific RSS properties.
  */
 int t4_config_vi_rss(struct adapter *adapter, int mbox, unsigned int viid,
 		     unsigned int flags, unsigned int defq)
 {
 	struct fw_rss_vi_config_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_viid = htonl(V_FW_CMD_OP(FW_RSS_VI_CONFIG_CMD) |
 			     F_FW_CMD_REQUEST | F_FW_CMD_WRITE |
 			     V_FW_RSS_VI_CONFIG_CMD_VIID(viid));
 	c.retval_len16 = htonl(FW_LEN16(c));
 	c.u.basicvirtual.defaultq_to_udpen = htonl(flags |
 					V_FW_RSS_VI_CONFIG_CMD_DEFAULTQ(defq));
 	return t4_wr_mbox(adapter, mbox, &c, sizeof(c), NULL);
 }
 
 /* Read an RSS table row */
 static int rd_rss_row(struct adapter *adap, int row, u32 *val)
 {
 	t4_write_reg(adap, A_TP_RSS_LKP_TABLE, 0xfff00000 | row);
 	return t4_wait_op_done_val(adap, A_TP_RSS_LKP_TABLE, F_LKPTBLROWVLD, 1,
 				   5, 0, val);
 }
 	
 /**
  *	t4_read_rss - read the contents of the RSS mapping table
  *	@adapter: the adapter
  *	@map: holds the contents of the RSS mapping table
  *
  *	Reads the contents of the RSS hash->queue mapping table.
  */
 int t4_read_rss(struct adapter *adapter, u16 *map)
 {
 	u32 val;
 	int i, ret;
 
 	for (i = 0; i < RSS_NENTRIES / 2; ++i) {
 		ret = rd_rss_row(adapter, i, &val);
 		if (ret)
 			return ret;
 		*map++ = G_LKPTBLQUEUE0(val);
 		*map++ = G_LKPTBLQUEUE1(val);
 	}
 	return 0;
 }
 
 /**
  *	t4_read_rss_key - read the global RSS key
  *	@adap: the adapter
  *	@key: 10-entry array holding the 320-bit RSS key
  *
  *	Reads the global 320-bit RSS key.
  */
 void t4_read_rss_key(struct adapter *adap, u32 *key)
 {
 	t4_read_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, key, 10,
 			 A_TP_RSS_SECRET_KEY0);
 }
 
 /**
  *	t4_write_rss_key - program one of the RSS keys
  *	@adap: the adapter
  *	@key: 10-entry array holding the 320-bit RSS key
  *	@idx: which RSS key to write
  *
  *	Writes one of the RSS keys with the given 320-bit value.  If @idx is
  *	0..15 the corresponding entry in the RSS key table is written,
  *	otherwise the global RSS key is written.
  */
 void t4_write_rss_key(struct adapter *adap, const u32 *key, int idx)
 {
 	t4_write_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, key, 10,
 			  A_TP_RSS_SECRET_KEY0);
 	if (idx >= 0 && idx < 16)
 		t4_write_reg(adap, A_TP_RSS_CONFIG_VRT,
 			     V_KEYWRADDR(idx) | F_KEYWREN);
 }
 
 /**
  *	t4_read_rss_pf_config - read PF RSS Configuration Table
  *	@adapter: the adapter
  *	@index: the entry in the PF RSS table to read
  *	@valp: where to store the returned value
  *
  *	Reads the PF RSS Configuration Table at the specified index and returns
  *	the value found there.
  */
 void t4_read_rss_pf_config(struct adapter *adapter, unsigned int index, u32 *valp)
 {
 	t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
 			 valp, 1, A_TP_RSS_PF0_CONFIG + index);
 }
 
 /**
  *	t4_write_rss_pf_config - write PF RSS Configuration Table
  *	@adapter: the adapter
  *	@index: the entry in the VF RSS table to read
  *	@val: the value to store
  *
  *	Writes the PF RSS Configuration Table at the specified index with the
  *	specified value.
  */
 void t4_write_rss_pf_config(struct adapter *adapter, unsigned int index, u32 val)
 {
 	t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
 			  &val, 1, A_TP_RSS_PF0_CONFIG + index);
 }
 
 /**
  *	t4_read_rss_vf_config - read VF RSS Configuration Table
  *	@adapter: the adapter
  *	@index: the entry in the VF RSS table to read
  *	@vfl: where to store the returned VFL
  *	@vfh: where to store the returned VFH
  *
  *	Reads the VF RSS Configuration Table at the specified index and returns
  *	the (VFL, VFH) values found there.
  */
 void t4_read_rss_vf_config(struct adapter *adapter, unsigned int index,
 			   u32 *vfl, u32 *vfh)
 {
 	u32 vrt;
 
 	/*
 	 * Request that the index'th VF Table values be read into VFL/VFH.
 	 */
 	vrt = t4_read_reg(adapter, A_TP_RSS_CONFIG_VRT);
 	vrt &= ~(F_VFRDRG | V_VFWRADDR(M_VFWRADDR) | F_VFWREN | F_KEYWREN);
 	vrt |= V_VFWRADDR(index) | F_VFRDEN;
 	t4_write_reg(adapter, A_TP_RSS_CONFIG_VRT, vrt);
 
 	/*
 	 * Grab the VFL/VFH values ...
 	 */
 	t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
 			 vfl, 1, A_TP_RSS_VFL_CONFIG);
 	t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
 			 vfh, 1, A_TP_RSS_VFH_CONFIG);
 }
 
 /**
  *	t4_write_rss_vf_config - write VF RSS Configuration Table
  *	
  *	@adapter: the adapter
  *	@index: the entry in the VF RSS table to write
  *	@vfl: the VFL to store
  *	@vfh: the VFH to store
  *
  *	Writes the VF RSS Configuration Table at the specified index with the
  *	specified (VFL, VFH) values.
  */
 void t4_write_rss_vf_config(struct adapter *adapter, unsigned int index,
 			    u32 vfl, u32 vfh)
 {
 	u32 vrt;
 
 	/*
 	 * Load up VFL/VFH with the values to be written ...
 	 */
 	t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
 			  &vfl, 1, A_TP_RSS_VFL_CONFIG);
 	t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
 			  &vfh, 1, A_TP_RSS_VFH_CONFIG);
 
 	/*
 	 * Write the VFL/VFH into the VF Table at index'th location.
 	 */
 	vrt = t4_read_reg(adapter, A_TP_RSS_CONFIG_VRT);
 	vrt &= ~(F_VFRDRG | F_VFRDEN | V_VFWRADDR(M_VFWRADDR) | F_KEYWREN);
 	vrt |= V_VFWRADDR(index) | F_VFWREN;
 	t4_write_reg(adapter, A_TP_RSS_CONFIG_VRT, vrt);
 }
 
 /**
  *	t4_read_rss_pf_map - read PF RSS Map
  *	@adapter: the adapter
  *
  *	Reads the PF RSS Map register and returns its value.
  */
 u32 t4_read_rss_pf_map(struct adapter *adapter)
 {
 	u32 pfmap;
 
 	t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
 			 &pfmap, 1, A_TP_RSS_PF_MAP);
 	return pfmap;
 }
 
 /**
  *	t4_write_rss_pf_map - write PF RSS Map
  *	@adapter: the adapter
  *	@pfmap: PF RSS Map value
  *
  *	Writes the specified value to the PF RSS Map register.
  */
 void t4_write_rss_pf_map(struct adapter *adapter, u32 pfmap)
 {
 	t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
 			  &pfmap, 1, A_TP_RSS_PF_MAP);
 }
 
 /**
  *	t4_read_rss_pf_mask - read PF RSS Mask
  *	@adapter: the adapter
  *
  *	Reads the PF RSS Mask register and returns its value.
  */
 u32 t4_read_rss_pf_mask(struct adapter *adapter)
 {
 	u32 pfmask;
 
 	t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
 			 &pfmask, 1, A_TP_RSS_PF_MSK);
 	return pfmask;
 }
 
 /**
  *	t4_write_rss_pf_mask - write PF RSS Mask
  *	@adapter: the adapter
  *	@pfmask: PF RSS Mask value
  *
  *	Writes the specified value to the PF RSS Mask register.
  */
 void t4_write_rss_pf_mask(struct adapter *adapter, u32 pfmask)
 {
 	t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
 			  &pfmask, 1, A_TP_RSS_PF_MSK);
 }
 
 /**
  *	t4_set_filter_mode - configure the optional components of filter tuples
  *	@adap: the adapter
  *	@mode_map: a bitmap selcting which optional filter components to enable
  *
  *	Sets the filter mode by selecting the optional components to enable
  *	in filter tuples.  Returns 0 on success and a negative error if the
  *	requested mode needs more bits than are available for optional
  *	components.
  */
 int t4_set_filter_mode(struct adapter *adap, unsigned int mode_map)
 {
 	static u8 width[] = { 1, 3, 17, 17, 8, 8, 16, 9, 3, 1 };
 
 	int i, nbits = 0;
 
 	for (i = S_FCOE; i <= S_FRAGMENTATION; i++)
 		if (mode_map & (1 << i))
 			nbits += width[i];
 	if (nbits > FILTER_OPT_LEN)
 		return -EINVAL;
 	t4_write_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, &mode_map, 1,
 			  A_TP_VLAN_PRI_MAP);
 	return 0;
 }
 
 /**
  *	t4_tp_get_tcp_stats - read TP's TCP MIB counters
  *	@adap: the adapter
  *	@v4: holds the TCP/IP counter values
  *	@v6: holds the TCP/IPv6 counter values
  *
  *	Returns the values of TP's TCP/IP and TCP/IPv6 MIB counters.
  *	Either @v4 or @v6 may be %NULL to skip the corresponding stats.
  */
 void t4_tp_get_tcp_stats(struct adapter *adap, struct tp_tcp_stats *v4,
 			 struct tp_tcp_stats *v6)
 {
 	u32 val[A_TP_MIB_TCP_RXT_SEG_LO - A_TP_MIB_TCP_OUT_RST + 1];
 
 #define STAT_IDX(x) ((A_TP_MIB_TCP_##x) - A_TP_MIB_TCP_OUT_RST)
 #define STAT(x)     val[STAT_IDX(x)]
 #define STAT64(x)   (((u64)STAT(x##_HI) << 32) | STAT(x##_LO))
 
 	if (v4) {
 		t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, val,
 				 ARRAY_SIZE(val), A_TP_MIB_TCP_OUT_RST);
 		v4->tcpOutRsts = STAT(OUT_RST);
 		v4->tcpInSegs  = STAT64(IN_SEG);
 		v4->tcpOutSegs = STAT64(OUT_SEG);
 		v4->tcpRetransSegs = STAT64(RXT_SEG);
 	}
 	if (v6) {
 		t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, val,
 				 ARRAY_SIZE(val), A_TP_MIB_TCP_V6OUT_RST);
 		v6->tcpOutRsts = STAT(OUT_RST);
 		v6->tcpInSegs  = STAT64(IN_SEG);
 		v6->tcpOutSegs = STAT64(OUT_SEG);
 		v6->tcpRetransSegs = STAT64(RXT_SEG);
 	}
 #undef STAT64
 #undef STAT
 #undef STAT_IDX
 }
 
 /**
  *	t4_tp_get_err_stats - read TP's error MIB counters
  *	@adap: the adapter
  *	@st: holds the counter values
  *
  *	Returns the values of TP's error counters.
  */
 void t4_tp_get_err_stats(struct adapter *adap, struct tp_err_stats *st)
 {
 	t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, st->macInErrs,
 			 12, A_TP_MIB_MAC_IN_ERR_0);
 	t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, st->tnlCongDrops,
 			 8, A_TP_MIB_TNL_CNG_DROP_0);
 	t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, st->tnlTxDrops,
 			 4, A_TP_MIB_TNL_DROP_0);
 	t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, st->ofldVlanDrops,
 			 4, A_TP_MIB_OFD_VLN_DROP_0);
 	t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, st->tcp6InErrs,
 			 4, A_TP_MIB_TCP_V6IN_ERR_0);
 	t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, &st->ofldNoNeigh,
 			 2, A_TP_MIB_OFD_ARP_DROP);
 }
 
 /**
  *	t4_tp_get_proxy_stats - read TP's proxy MIB counters
  *	@adap: the adapter
  *	@st: holds the counter values
  *
  *	Returns the values of TP's proxy counters.
  */
 void t4_tp_get_proxy_stats(struct adapter *adap, struct tp_proxy_stats *st)
 {
 	t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, st->proxy,
 			 4, A_TP_MIB_TNL_LPBK_0);
 }
 
 /**
  *	t4_tp_get_cpl_stats - read TP's CPL MIB counters
  *	@adap: the adapter
  *	@st: holds the counter values
  *
  *	Returns the values of TP's CPL counters.
  */
 void t4_tp_get_cpl_stats(struct adapter *adap, struct tp_cpl_stats *st)
 {
 	t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, st->req,
 			 8, A_TP_MIB_CPL_IN_REQ_0);
 }
 
 /**
  *	t4_tp_get_rdma_stats - read TP's RDMA MIB counters
  *	@adap: the adapter
  *	@st: holds the counter values
  *
  *	Returns the values of TP's RDMA counters.
  */
 void t4_tp_get_rdma_stats(struct adapter *adap, struct tp_rdma_stats *st)
 {
 	t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, &st->rqe_dfr_mod,
 			 2, A_TP_MIB_RQE_DFR_MOD);
 }
 
 /**
  *	t4_get_fcoe_stats - read TP's FCoE MIB counters for a port
  *	@adap: the adapter
  *	@idx: the port index
  *	@st: holds the counter values
  *
  *	Returns the values of TP's FCoE counters for the selected port.
  */
 void t4_get_fcoe_stats(struct adapter *adap, unsigned int idx,
 		       struct tp_fcoe_stats *st)
 {
 	u32 val[2];
 
 	t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, &st->framesDDP,
 			 1, A_TP_MIB_FCOE_DDP_0 + idx);
 	t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, &st->framesDrop,
 			 1, A_TP_MIB_FCOE_DROP_0 + idx);
 	t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, val,
 			 2, A_TP_MIB_FCOE_BYTE_0_HI + 2 * idx);
 	st->octetsDDP = ((u64)val[0] << 32) | val[1];
 }
 
 /**
  *	t4_get_usm_stats - read TP's non-TCP DDP MIB counters
  *	@adap: the adapter
  *	@st: holds the counter values
  *
  *	Returns the values of TP's counters for non-TCP directly-placed packets.
  */
 void t4_get_usm_stats(struct adapter *adap, struct tp_usm_stats *st)
 {
 	u32 val[4];
 
 	t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, val, 4,
 			 A_TP_MIB_USM_PKTS);
 	st->frames = val[0];
 	st->drops = val[1];
 	st->octets = ((u64)val[2] << 32) | val[3];
 }
 
 /**
  *	t4_read_mtu_tbl - returns the values in the HW path MTU table
  *	@adap: the adapter
  *	@mtus: where to store the MTU values
  *	@mtu_log: where to store the MTU base-2 log (may be %NULL)
  *
  *	Reads the HW path MTU table.
  */
 void t4_read_mtu_tbl(struct adapter *adap, u16 *mtus, u8 *mtu_log)
 {
 	u32 v;
 	int i;
 
 	for (i = 0; i < NMTUS; ++i) {
 		t4_write_reg(adap, A_TP_MTU_TABLE,
 			     V_MTUINDEX(0xff) | V_MTUVALUE(i));
 		v = t4_read_reg(adap, A_TP_MTU_TABLE);
 		mtus[i] = G_MTUVALUE(v);
 		if (mtu_log)
 			mtu_log[i] = G_MTUWIDTH(v);
 	}
 }
 
 /**
  *	t4_read_cong_tbl - reads the congestion control table
  *	@adap: the adapter
  *	@incr: where to store the alpha values
  *
  *	Reads the additive increments programmed into the HW congestion
  *	control table.
  */
 void t4_read_cong_tbl(struct adapter *adap, u16 incr[NMTUS][NCCTRL_WIN])
 {
 	unsigned int mtu, w;
 
 	for (mtu = 0; mtu < NMTUS; ++mtu)
 		for (w = 0; w < NCCTRL_WIN; ++w) {
 			t4_write_reg(adap, A_TP_CCTRL_TABLE,
 				     V_ROWINDEX(0xffff) | (mtu << 5) | w);
 			incr[mtu][w] = (u16)t4_read_reg(adap,
 						A_TP_CCTRL_TABLE) & 0x1fff;
 		}
 }
 
 /**
  *	t4_read_pace_tbl - read the pace table
  *	@adap: the adapter
  *	@pace_vals: holds the returned values
  *
  *	Returns the values of TP's pace table in microseconds.
  */
 void t4_read_pace_tbl(struct adapter *adap, unsigned int pace_vals[NTX_SCHED])
 {
 	unsigned int i, v;
 
 	for (i = 0; i < NTX_SCHED; i++) {
 		t4_write_reg(adap, A_TP_PACE_TABLE, 0xffff0000 + i);
 		v = t4_read_reg(adap, A_TP_PACE_TABLE);
 		pace_vals[i] = dack_ticks_to_usec(adap, v);
 	}
 }
 
 /**
  *	t4_tp_wr_bits_indirect - set/clear bits in an indirect TP register
  *	@adap: the adapter
  *	@addr: the indirect TP register address
  *	@mask: specifies the field within the register to modify
  *	@val: new value for the field
  *
  *	Sets a field of an indirect TP register to the given value.
  */
 void t4_tp_wr_bits_indirect(struct adapter *adap, unsigned int addr,
 			    unsigned int mask, unsigned int val)
 {
 	t4_write_reg(adap, A_TP_PIO_ADDR, addr);
 	val |= t4_read_reg(adap, A_TP_PIO_DATA) & ~mask;
 	t4_write_reg(adap, A_TP_PIO_DATA, val);
 }
 
 /**
  *	init_cong_ctrl - initialize congestion control parameters
  *	@a: the alpha values for congestion control
  *	@b: the beta values for congestion control
  *
  *	Initialize the congestion control parameters.
  */
 static void __devinit init_cong_ctrl(unsigned short *a, unsigned short *b)
 {
 	a[0] = a[1] = a[2] = a[3] = a[4] = a[5] = a[6] = a[7] = a[8] = 1;
 	a[9] = 2;
 	a[10] = 3;
 	a[11] = 4;
 	a[12] = 5;
 	a[13] = 6;
 	a[14] = 7;
 	a[15] = 8;
 	a[16] = 9;
 	a[17] = 10;
 	a[18] = 14;
 	a[19] = 17;
 	a[20] = 21;
 	a[21] = 25;
 	a[22] = 30;
 	a[23] = 35;
 	a[24] = 45;
 	a[25] = 60;
 	a[26] = 80;
 	a[27] = 100;
 	a[28] = 200;
 	a[29] = 300;
 	a[30] = 400;
 	a[31] = 500;
 
 	b[0] = b[1] = b[2] = b[3] = b[4] = b[5] = b[6] = b[7] = b[8] = 0;
 	b[9] = b[10] = 1;
 	b[11] = b[12] = 2;
 	b[13] = b[14] = b[15] = b[16] = 3;
 	b[17] = b[18] = b[19] = b[20] = b[21] = 4;
 	b[22] = b[23] = b[24] = b[25] = b[26] = b[27] = 5;
 	b[28] = b[29] = 6;
 	b[30] = b[31] = 7;
 }
 
 /* The minimum additive increment value for the congestion control table */
 #define CC_MIN_INCR 2U
 
 /**
  *	t4_load_mtus - write the MTU and congestion control HW tables
  *	@adap: the adapter
  *	@mtus: the values for the MTU table
  *	@alpha: the values for the congestion control alpha parameter
  *	@beta: the values for the congestion control beta parameter
  *
  *	Write the HW MTU table with the supplied MTUs and the high-speed
  *	congestion control table with the supplied alpha, beta, and MTUs.
  *	We write the two tables together because the additive increments
  *	depend on the MTUs.
  */
 void t4_load_mtus(struct adapter *adap, const unsigned short *mtus,
 		  const unsigned short *alpha, const unsigned short *beta)
 {
 	static const unsigned int avg_pkts[NCCTRL_WIN] = {
 		2, 6, 10, 14, 20, 28, 40, 56, 80, 112, 160, 224, 320, 448, 640,
 		896, 1281, 1792, 2560, 3584, 5120, 7168, 10240, 14336, 20480,
 		28672, 40960, 57344, 81920, 114688, 163840, 229376
 	};
 
 	unsigned int i, w;
 
 	for (i = 0; i < NMTUS; ++i) {
 		unsigned int mtu = mtus[i];
 		unsigned int log2 = fls(mtu);
 
 		if (!(mtu & ((1 << log2) >> 2)))     /* round */
 			log2--;
 		t4_write_reg(adap, A_TP_MTU_TABLE, V_MTUINDEX(i) |
 			     V_MTUWIDTH(log2) | V_MTUVALUE(mtu));
 
 		for (w = 0; w < NCCTRL_WIN; ++w) {
 			unsigned int inc;
 
 			inc = max(((mtu - 40) * alpha[w]) / avg_pkts[w],
 				  CC_MIN_INCR);
 
 			t4_write_reg(adap, A_TP_CCTRL_TABLE, (i << 21) |
 				     (w << 16) | (beta[w] << 13) | inc);
 		}
 	}
 }
 
 /**
  *	t4_set_pace_tbl - set the pace table
  *	@adap: the adapter
  *	@pace_vals: the pace values in microseconds
  *	@start: index of the first entry in the HW pace table to set
  *	@n: how many entries to set
  *
  *	Sets (a subset of the) HW pace table.
  */
 int t4_set_pace_tbl(struct adapter *adap, const unsigned int *pace_vals,
 		     unsigned int start, unsigned int n)
 {
 	unsigned int vals[NTX_SCHED], i;
 	unsigned int tick_ns = dack_ticks_to_usec(adap, 1000);
 
 	if (n > NTX_SCHED)
 	    return -ERANGE;
     
 	/* convert values from us to dack ticks, rounding to closest value */
 	for (i = 0; i < n; i++, pace_vals++) {
 		vals[i] = (1000 * *pace_vals + tick_ns / 2) / tick_ns;
 		if (vals[i] > 0x7ff)
 			return -ERANGE;
 		if (*pace_vals && vals[i] == 0)
 			return -ERANGE;
 	}
 	for (i = 0; i < n; i++, start++)
 		t4_write_reg(adap, A_TP_PACE_TABLE, (start << 16) | vals[i]);
 	return 0;
 }
 
 /**
  *	t4_set_sched_bps - set the bit rate for a HW traffic scheduler
  *	@adap: the adapter
  *	@kbps: target rate in Kbps
  *	@sched: the scheduler index
  *
  *	Configure a Tx HW scheduler for the target rate.
  */
 int t4_set_sched_bps(struct adapter *adap, int sched, unsigned int kbps)
 {
 	unsigned int v, tps, cpt, bpt, delta, mindelta = ~0;
 	unsigned int clk = adap->params.vpd.cclk * 1000;
 	unsigned int selected_cpt = 0, selected_bpt = 0;
 
 	if (kbps > 0) {
 		kbps *= 125;     /* -> bytes */
 		for (cpt = 1; cpt <= 255; cpt++) {
 			tps = clk / cpt;
 			bpt = (kbps + tps / 2) / tps;
 			if (bpt > 0 && bpt <= 255) {
 				v = bpt * tps;
 				delta = v >= kbps ? v - kbps : kbps - v;
 				if (delta < mindelta) {
 					mindelta = delta;
 					selected_cpt = cpt;
 					selected_bpt = bpt;
 				}
 			} else if (selected_cpt)
 				break;
 		}
 		if (!selected_cpt)
 			return -EINVAL;
 	}
 	t4_write_reg(adap, A_TP_TM_PIO_ADDR,
 		     A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2);
 	v = t4_read_reg(adap, A_TP_TM_PIO_DATA);
 	if (sched & 1)
 		v = (v & 0xffff) | (selected_cpt << 16) | (selected_bpt << 24);
 	else
 		v = (v & 0xffff0000) | selected_cpt | (selected_bpt << 8);
 	t4_write_reg(adap, A_TP_TM_PIO_DATA, v);
 	return 0;
 }
 
 /**
  *	t4_set_sched_ipg - set the IPG for a Tx HW packet rate scheduler
  *	@adap: the adapter
  *	@sched: the scheduler index
  *	@ipg: the interpacket delay in tenths of nanoseconds
  *
  *	Set the interpacket delay for a HW packet rate scheduler.
  */
 int t4_set_sched_ipg(struct adapter *adap, int sched, unsigned int ipg)
 {
 	unsigned int v, addr = A_TP_TX_MOD_Q1_Q0_TIMER_SEPARATOR - sched / 2;
 
 	/* convert ipg to nearest number of core clocks */
 	ipg *= core_ticks_per_usec(adap);
 	ipg = (ipg + 5000) / 10000;
 	if (ipg > M_TXTIMERSEPQ0)
 		return -EINVAL;
 
 	t4_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 	v = t4_read_reg(adap, A_TP_TM_PIO_DATA);
 	if (sched & 1)
 		v = (v & V_TXTIMERSEPQ0(M_TXTIMERSEPQ0)) | V_TXTIMERSEPQ1(ipg);
 	else
 		v = (v & V_TXTIMERSEPQ1(M_TXTIMERSEPQ1)) | V_TXTIMERSEPQ0(ipg);
 	t4_write_reg(adap, A_TP_TM_PIO_DATA, v);
 	t4_read_reg(adap, A_TP_TM_PIO_DATA);
 	return 0;
 }
 
 /**
  *	t4_get_tx_sched - get the configuration of a Tx HW traffic scheduler
  *	@adap: the adapter
  *	@sched: the scheduler index
  *	@kbps: the byte rate in Kbps
  *	@ipg: the interpacket delay in tenths of nanoseconds
  *
  *	Return the current configuration of a HW Tx scheduler.
  */
 void t4_get_tx_sched(struct adapter *adap, unsigned int sched, unsigned int *kbps,
 		     unsigned int *ipg)
 {
 	unsigned int v, addr, bpt, cpt;
 
 	if (kbps) {
 		addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
 		t4_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 		v = t4_read_reg(adap, A_TP_TM_PIO_DATA);
 		if (sched & 1)
 			v >>= 16;
 		bpt = (v >> 8) & 0xff;
 		cpt = v & 0xff;
 		if (!cpt)
 			*kbps = 0;        /* scheduler disabled */
 		else {
 			v = (adap->params.vpd.cclk * 1000) / cpt; /* ticks/s */
 			*kbps = (v * bpt) / 125;
 		}
 	}
 	if (ipg) {
 		addr = A_TP_TX_MOD_Q1_Q0_TIMER_SEPARATOR - sched / 2;
 		t4_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 		v = t4_read_reg(adap, A_TP_TM_PIO_DATA);
 		if (sched & 1)
 			v >>= 16;
 		v &= 0xffff;
 		*ipg = (10000 * v) / core_ticks_per_usec(adap);
 	}
 }
 
 /*
  * Calculates a rate in bytes/s given the number of 256-byte units per 4K core
  * clocks.  The formula is
  *
  * bytes/s = bytes256 * 256 * ClkFreq / 4096
  *
  * which is equivalent to
  *
  * bytes/s = 62.5 * bytes256 * ClkFreq_ms
  */
 static u64 chan_rate(struct adapter *adap, unsigned int bytes256)
 {
 	u64 v = bytes256 * adap->params.vpd.cclk;
 
 	return v * 62 + v / 2;
 }
 
 /**
  *	t4_get_chan_txrate - get the current per channel Tx rates
  *	@adap: the adapter
  *	@nic_rate: rates for NIC traffic
  *	@ofld_rate: rates for offloaded traffic
  *
  *	Return the current Tx rates in bytes/s for NIC and offloaded traffic
  *	for each channel.
  */
 void t4_get_chan_txrate(struct adapter *adap, u64 *nic_rate, u64 *ofld_rate)
 {
 	u32 v;
 
 	v = t4_read_reg(adap, A_TP_TX_TRATE);
 	nic_rate[0] = chan_rate(adap, G_TNLRATE0(v));
 	nic_rate[1] = chan_rate(adap, G_TNLRATE1(v));
 	nic_rate[2] = chan_rate(adap, G_TNLRATE2(v));
 	nic_rate[3] = chan_rate(adap, G_TNLRATE3(v));
 
 	v = t4_read_reg(adap, A_TP_TX_ORATE);
 	ofld_rate[0] = chan_rate(adap, G_OFDRATE0(v));
 	ofld_rate[1] = chan_rate(adap, G_OFDRATE1(v));
 	ofld_rate[2] = chan_rate(adap, G_OFDRATE2(v));
 	ofld_rate[3] = chan_rate(adap, G_OFDRATE3(v));
 }
 
 /**
  *	t4_set_trace_filter - configure one of the tracing filters
  *	@adap: the adapter
  *	@tp: the desired trace filter parameters
  *	@idx: which filter to configure
  *	@enable: whether to enable or disable the filter
  *
  *	Configures one of the tracing filters available in HW.  If @tp is %NULL
  *	it indicates that the filter is already written in the register and it
  *	just needs to be enabled or disabled.
  */
 int t4_set_trace_filter(struct adapter *adap, const struct trace_params *tp,
     int idx, int enable)
 {
 	int i, ofst = idx * 4;
 	u32 data_reg, mask_reg, cfg;
 	u32 multitrc = F_TRCMULTIFILTER;
 	u32 en = is_t4(adap) ? F_TFEN : F_T5_TFEN;
 
 	if (idx < 0 || idx >= NTRACE)
 		return -EINVAL;
 
 	if (tp == NULL || !enable) {
 		t4_set_reg_field(adap, A_MPS_TRC_FILTER_MATCH_CTL_A + ofst, en,
 		    enable ? en : 0);
 		return 0;
 	}
 
 	/*
 	 * TODO - After T4 data book is updated, specify the exact
 	 * section below.
 	 *
 	 * See T4 data book - MPS section for a complete description 
 	 * of the below if..else handling of A_MPS_TRC_CFG register 
 	 * value.
 	 */ 
 	cfg = t4_read_reg(adap, A_MPS_TRC_CFG);
 	if (cfg & F_TRCMULTIFILTER) {
 		/*
 		 * If multiple tracers are enabled, then maximum
 		 * capture size is 2.5KB (FIFO size of a single channel)
 		 * minus 2 flits for CPL_TRACE_PKT header.
 		 */
 		if (tp->snap_len > ((10 * 1024 / 4) - (2 * 8)))
 			return -EINVAL;		
 	} else {
 		/*
 		 * If multiple tracers are disabled, to avoid deadlocks 
 		 * maximum packet capture size of 9600 bytes is recommended.
 		 * Also in this mode, only trace0 can be enabled and running.
 		 */
 		multitrc = 0;
 		if (tp->snap_len > 9600 || idx)
 			return -EINVAL;
 	}
 
 	if (tp->port > (is_t4(adap) ? 11 : 19) || tp->invert > 1 ||
 	    tp->skip_len > M_TFLENGTH || tp->skip_ofst > M_TFOFFSET ||
 	    tp->min_len > M_TFMINPKTSIZE)
 		return -EINVAL;
 
 	/* stop the tracer we'll be changing */
 	t4_set_reg_field(adap, A_MPS_TRC_FILTER_MATCH_CTL_A + ofst, en, 0);
 
 	idx *= (A_MPS_TRC_FILTER1_MATCH - A_MPS_TRC_FILTER0_MATCH);
 	data_reg = A_MPS_TRC_FILTER0_MATCH + idx;
 	mask_reg = A_MPS_TRC_FILTER0_DONT_CARE + idx;
 
 	for (i = 0; i < TRACE_LEN / 4; i++, data_reg += 4, mask_reg += 4) {
 		t4_write_reg(adap, data_reg, tp->data[i]);
 		t4_write_reg(adap, mask_reg, ~tp->mask[i]);
 	}
 	t4_write_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_B + ofst,
 		     V_TFCAPTUREMAX(tp->snap_len) |
 		     V_TFMINPKTSIZE(tp->min_len));
 	t4_write_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_A + ofst,
 		     V_TFOFFSET(tp->skip_ofst) | V_TFLENGTH(tp->skip_len) | en |
 		     (is_t4(adap) ?
 		     V_TFPORT(tp->port) | V_TFINVERTMATCH(tp->invert) :
 		     V_T5_TFPORT(tp->port) | V_T5_TFINVERTMATCH(tp->invert)));
 
 	return 0;
 }
 
 /**
  *	t4_get_trace_filter - query one of the tracing filters
  *	@adap: the adapter
  *	@tp: the current trace filter parameters
  *	@idx: which trace filter to query
  *	@enabled: non-zero if the filter is enabled
  *
  *	Returns the current settings of one of the HW tracing filters.
  */
 void t4_get_trace_filter(struct adapter *adap, struct trace_params *tp, int idx,
 			 int *enabled)
 {
 	u32 ctla, ctlb;
 	int i, ofst = idx * 4;
 	u32 data_reg, mask_reg;
 
 	ctla = t4_read_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_A + ofst);
 	ctlb = t4_read_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_B + ofst);
 
 	if (is_t4(adap)) {
 		*enabled = !!(ctla & F_TFEN);
 		tp->port =  G_TFPORT(ctla);
 		tp->invert = !!(ctla & F_TFINVERTMATCH);
 	} else {
 		*enabled = !!(ctla & F_T5_TFEN);
 		tp->port = G_T5_TFPORT(ctla);
 		tp->invert = !!(ctla & F_T5_TFINVERTMATCH);
 	}
 	tp->snap_len = G_TFCAPTUREMAX(ctlb);
 	tp->min_len = G_TFMINPKTSIZE(ctlb);
 	tp->skip_ofst = G_TFOFFSET(ctla);
 	tp->skip_len = G_TFLENGTH(ctla);
 
 	ofst = (A_MPS_TRC_FILTER1_MATCH - A_MPS_TRC_FILTER0_MATCH) * idx;
 	data_reg = A_MPS_TRC_FILTER0_MATCH + ofst;
 	mask_reg = A_MPS_TRC_FILTER0_DONT_CARE + ofst;
 
 	for (i = 0; i < TRACE_LEN / 4; i++, data_reg += 4, mask_reg += 4) {
 		tp->mask[i] = ~t4_read_reg(adap, mask_reg);
 		tp->data[i] = t4_read_reg(adap, data_reg) & tp->mask[i];
 	}
 }
 
 /**
  *	t4_pmtx_get_stats - returns the HW stats from PMTX
  *	@adap: the adapter
  *	@cnt: where to store the count statistics
  *	@cycles: where to store the cycle statistics
  *
  *	Returns performance statistics from PMTX.
  */
 void t4_pmtx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[])
 {
 	int i;
 	u32 data[2];
 
 	for (i = 0; i < PM_NSTATS; i++) {
 		t4_write_reg(adap, A_PM_TX_STAT_CONFIG, i + 1);
 		cnt[i] = t4_read_reg(adap, A_PM_TX_STAT_COUNT);
 		if (is_t4(adap))
 			cycles[i] = t4_read_reg64(adap, A_PM_TX_STAT_LSB);
 		else {
 			t4_read_indirect(adap, A_PM_TX_DBG_CTRL,
 					 A_PM_TX_DBG_DATA, data, 2,
 					 A_PM_TX_DBG_STAT_MSB);
 			cycles[i] = (((u64)data[0] << 32) | data[1]);
 		}
 	}
 }
 
 /**
  *	t4_pmrx_get_stats - returns the HW stats from PMRX
  *	@adap: the adapter
  *	@cnt: where to store the count statistics
  *	@cycles: where to store the cycle statistics
  *
  *	Returns performance statistics from PMRX.
  */
 void t4_pmrx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[])
 {
 	int i;
 	u32 data[2];
 
 	for (i = 0; i < PM_NSTATS; i++) {
 		t4_write_reg(adap, A_PM_RX_STAT_CONFIG, i + 1);
 		cnt[i] = t4_read_reg(adap, A_PM_RX_STAT_COUNT);
 		if (is_t4(adap))
 			cycles[i] = t4_read_reg64(adap, A_PM_RX_STAT_LSB);
 		else {
 			t4_read_indirect(adap, A_PM_RX_DBG_CTRL,
 					 A_PM_RX_DBG_DATA, data, 2,
 					 A_PM_RX_DBG_STAT_MSB);
 			cycles[i] = (((u64)data[0] << 32) | data[1]);
 		}
 	}
 }
 
 /**
  *	get_mps_bg_map - return the buffer groups associated with a port
  *	@adap: the adapter
  *	@idx: the port index
  *
  *	Returns a bitmap indicating which MPS buffer groups are associated
  *	with the given port.  Bit i is set if buffer group i is used by the
  *	port.
  */
 static unsigned int get_mps_bg_map(struct adapter *adap, int idx)
 {
 	u32 n = G_NUMPORTS(t4_read_reg(adap, A_MPS_CMN_CTL));
 
 	if (n == 0)
 		return idx == 0 ? 0xf : 0;
 	if (n == 1)
 		return idx < 2 ? (3 << (2 * idx)) : 0;
 	return 1 << idx;
 }
 
 /**
  *      t4_get_port_stats_offset - collect port stats relative to a previous
  *                                 snapshot
  *      @adap: The adapter
  *      @idx: The port
  *      @stats: Current stats to fill
  *      @offset: Previous stats snapshot
  */
 void t4_get_port_stats_offset(struct adapter *adap, int idx,
 		struct port_stats *stats,
 		struct port_stats *offset)
 {
 	u64 *s, *o;
 	int i;
 
 	t4_get_port_stats(adap, idx, stats);
 	for (i = 0, s = (u64 *)stats, o = (u64 *)offset ;
 			i < (sizeof(struct port_stats)/sizeof(u64)) ;
 			i++, s++, o++)
 		*s -= *o;
 }
 
 /**
  *	t4_get_port_stats - collect port statistics
  *	@adap: the adapter
  *	@idx: the port index
  *	@p: the stats structure to fill
  *
  *	Collect statistics related to the given port from HW.
  */
 void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p)
 {
 	u32 bgmap = get_mps_bg_map(adap, idx);
 
 #define GET_STAT(name) \
 	t4_read_reg64(adap, \
 	(is_t4(adap) ? PORT_REG(idx, A_MPS_PORT_STAT_##name##_L) : \
 	T5_PORT_REG(idx, A_MPS_PORT_STAT_##name##_L)))
 #define GET_STAT_COM(name) t4_read_reg64(adap, A_MPS_STAT_##name##_L)
 
 	p->tx_pause            = GET_STAT(TX_PORT_PAUSE);
 	p->tx_octets           = GET_STAT(TX_PORT_BYTES);
 	p->tx_frames           = GET_STAT(TX_PORT_FRAMES);
 	p->tx_bcast_frames     = GET_STAT(TX_PORT_BCAST);
 	p->tx_mcast_frames     = GET_STAT(TX_PORT_MCAST);
 	p->tx_ucast_frames     = GET_STAT(TX_PORT_UCAST);
 	p->tx_error_frames     = GET_STAT(TX_PORT_ERROR);
 	p->tx_frames_64        = GET_STAT(TX_PORT_64B);
 	p->tx_frames_65_127    = GET_STAT(TX_PORT_65B_127B);
 	p->tx_frames_128_255   = GET_STAT(TX_PORT_128B_255B);
 	p->tx_frames_256_511   = GET_STAT(TX_PORT_256B_511B);
 	p->tx_frames_512_1023  = GET_STAT(TX_PORT_512B_1023B);
 	p->tx_frames_1024_1518 = GET_STAT(TX_PORT_1024B_1518B);
 	p->tx_frames_1519_max  = GET_STAT(TX_PORT_1519B_MAX);
 	p->tx_drop             = GET_STAT(TX_PORT_DROP);
 	p->tx_ppp0             = GET_STAT(TX_PORT_PPP0);
 	p->tx_ppp1             = GET_STAT(TX_PORT_PPP1);
 	p->tx_ppp2             = GET_STAT(TX_PORT_PPP2);
 	p->tx_ppp3             = GET_STAT(TX_PORT_PPP3);
 	p->tx_ppp4             = GET_STAT(TX_PORT_PPP4);
 	p->tx_ppp5             = GET_STAT(TX_PORT_PPP5);
 	p->tx_ppp6             = GET_STAT(TX_PORT_PPP6);
 	p->tx_ppp7             = GET_STAT(TX_PORT_PPP7);
 
 	p->rx_pause            = GET_STAT(RX_PORT_PAUSE);
 	p->rx_octets           = GET_STAT(RX_PORT_BYTES);
 	p->rx_frames           = GET_STAT(RX_PORT_FRAMES);
 	p->rx_bcast_frames     = GET_STAT(RX_PORT_BCAST);
 	p->rx_mcast_frames     = GET_STAT(RX_PORT_MCAST);
 	p->rx_ucast_frames     = GET_STAT(RX_PORT_UCAST);
 	p->rx_too_long         = GET_STAT(RX_PORT_MTU_ERROR);
 	p->rx_jabber           = GET_STAT(RX_PORT_MTU_CRC_ERROR);
 	p->rx_fcs_err          = GET_STAT(RX_PORT_CRC_ERROR);
 	p->rx_len_err          = GET_STAT(RX_PORT_LEN_ERROR);
 	p->rx_symbol_err       = GET_STAT(RX_PORT_SYM_ERROR);
 	p->rx_runt             = GET_STAT(RX_PORT_LESS_64B);
 	p->rx_frames_64        = GET_STAT(RX_PORT_64B);
 	p->rx_frames_65_127    = GET_STAT(RX_PORT_65B_127B);
 	p->rx_frames_128_255   = GET_STAT(RX_PORT_128B_255B);
 	p->rx_frames_256_511   = GET_STAT(RX_PORT_256B_511B);
 	p->rx_frames_512_1023  = GET_STAT(RX_PORT_512B_1023B);
 	p->rx_frames_1024_1518 = GET_STAT(RX_PORT_1024B_1518B);
 	p->rx_frames_1519_max  = GET_STAT(RX_PORT_1519B_MAX);
 	p->rx_ppp0             = GET_STAT(RX_PORT_PPP0);
 	p->rx_ppp1             = GET_STAT(RX_PORT_PPP1);
 	p->rx_ppp2             = GET_STAT(RX_PORT_PPP2);
 	p->rx_ppp3             = GET_STAT(RX_PORT_PPP3);
 	p->rx_ppp4             = GET_STAT(RX_PORT_PPP4);
 	p->rx_ppp5             = GET_STAT(RX_PORT_PPP5);
 	p->rx_ppp6             = GET_STAT(RX_PORT_PPP6);
 	p->rx_ppp7             = GET_STAT(RX_PORT_PPP7);
 
 	p->rx_ovflow0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0;
 	p->rx_ovflow1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0;
 	p->rx_ovflow2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0;
 	p->rx_ovflow3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0;
 	p->rx_trunc0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0;
 	p->rx_trunc1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0;
 	p->rx_trunc2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0;
 	p->rx_trunc3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0;
 
 #undef GET_STAT
 #undef GET_STAT_COM
 }
 
 /**
  *	t4_clr_port_stats - clear port statistics
  *	@adap: the adapter
  *	@idx: the port index
  *
  *	Clear HW statistics for the given port.
  */
 void t4_clr_port_stats(struct adapter *adap, int idx)
 {
 	unsigned int i;
 	u32 bgmap = get_mps_bg_map(adap, idx);
 	u32 port_base_addr;
 
 	if (is_t4(adap))
 		port_base_addr = PORT_BASE(idx);
 	else
 		port_base_addr = T5_PORT_BASE(idx);
 
 	for (i = A_MPS_PORT_STAT_TX_PORT_BYTES_L;
 			i <= A_MPS_PORT_STAT_TX_PORT_PPP7_H; i += 8)
 		t4_write_reg(adap, port_base_addr + i, 0);
 	for (i = A_MPS_PORT_STAT_RX_PORT_BYTES_L;
 			i <= A_MPS_PORT_STAT_RX_PORT_LESS_64B_H; i += 8)
 		t4_write_reg(adap, port_base_addr + i, 0);
 	for (i = 0; i < 4; i++)
 		if (bgmap & (1 << i)) {
 			t4_write_reg(adap,
 				A_MPS_STAT_RX_BG_0_MAC_DROP_FRAME_L + i * 8, 0);
 			t4_write_reg(adap,
 				A_MPS_STAT_RX_BG_0_MAC_TRUNC_FRAME_L + i * 8, 0);
 		}
 }
 
 /**
  *	t4_get_lb_stats - collect loopback port statistics
  *	@adap: the adapter
  *	@idx: the loopback port index
  *	@p: the stats structure to fill
  *
  *	Return HW statistics for the given loopback port.
  */
 void t4_get_lb_stats(struct adapter *adap, int idx, struct lb_port_stats *p)
 {
 	u32 bgmap = get_mps_bg_map(adap, idx);
 
 #define GET_STAT(name) \
 	t4_read_reg64(adap, \
 	(is_t4(adap) ? \
 	PORT_REG(idx, A_MPS_PORT_STAT_LB_PORT_##name##_L) : \
 	T5_PORT_REG(idx, A_MPS_PORT_STAT_LB_PORT_##name##_L)))
 #define GET_STAT_COM(name) t4_read_reg64(adap, A_MPS_STAT_##name##_L)
 
 	p->octets           = GET_STAT(BYTES);
 	p->frames           = GET_STAT(FRAMES);
 	p->bcast_frames     = GET_STAT(BCAST);
 	p->mcast_frames     = GET_STAT(MCAST);
 	p->ucast_frames     = GET_STAT(UCAST);
 	p->error_frames     = GET_STAT(ERROR);
 
 	p->frames_64        = GET_STAT(64B);
 	p->frames_65_127    = GET_STAT(65B_127B);
 	p->frames_128_255   = GET_STAT(128B_255B);
 	p->frames_256_511   = GET_STAT(256B_511B);
 	p->frames_512_1023  = GET_STAT(512B_1023B);
 	p->frames_1024_1518 = GET_STAT(1024B_1518B);
 	p->frames_1519_max  = GET_STAT(1519B_MAX);
 	p->drop             = GET_STAT(DROP_FRAMES);
 
 	p->ovflow0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_LB_DROP_FRAME) : 0;
 	p->ovflow1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_LB_DROP_FRAME) : 0;
 	p->ovflow2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_LB_DROP_FRAME) : 0;
 	p->ovflow3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_LB_DROP_FRAME) : 0;
 	p->trunc0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_LB_TRUNC_FRAME) : 0;
 	p->trunc1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_LB_TRUNC_FRAME) : 0;
 	p->trunc2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_LB_TRUNC_FRAME) : 0;
 	p->trunc3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_LB_TRUNC_FRAME) : 0;
 
 #undef GET_STAT
 #undef GET_STAT_COM
 }
 
 /**
  *	t4_wol_magic_enable - enable/disable magic packet WoL
  *	@adap: the adapter
  *	@port: the physical port index
  *	@addr: MAC address expected in magic packets, %NULL to disable
  *
  *	Enables/disables magic packet wake-on-LAN for the selected port.
  */
 void t4_wol_magic_enable(struct adapter *adap, unsigned int port,
 			 const u8 *addr)
 {
 	u32 mag_id_reg_l, mag_id_reg_h, port_cfg_reg;
 
 	if (is_t4(adap)) {
 		mag_id_reg_l = PORT_REG(port, A_XGMAC_PORT_MAGIC_MACID_LO);
 		mag_id_reg_h = PORT_REG(port, A_XGMAC_PORT_MAGIC_MACID_HI);
 		port_cfg_reg = PORT_REG(port, A_XGMAC_PORT_CFG2);
 	} else {
 		mag_id_reg_l = T5_PORT_REG(port, A_MAC_PORT_MAGIC_MACID_LO);
 		mag_id_reg_h = T5_PORT_REG(port, A_MAC_PORT_MAGIC_MACID_HI);
 		port_cfg_reg = T5_PORT_REG(port, A_MAC_PORT_CFG2);
 	}
 
 	if (addr) {
 		t4_write_reg(adap, mag_id_reg_l,
 			     (addr[2] << 24) | (addr[3] << 16) |
 			     (addr[4] << 8) | addr[5]);
 		t4_write_reg(adap, mag_id_reg_h,
 			     (addr[0] << 8) | addr[1]);
 	}
 	t4_set_reg_field(adap, port_cfg_reg, F_MAGICEN,
 			 V_MAGICEN(addr != NULL));
 }
 
 /**
  *	t4_wol_pat_enable - enable/disable pattern-based WoL
  *	@adap: the adapter
  *	@port: the physical port index
  *	@map: bitmap of which HW pattern filters to set
  *	@mask0: byte mask for bytes 0-63 of a packet
  *	@mask1: byte mask for bytes 64-127 of a packet
  *	@crc: Ethernet CRC for selected bytes
  *	@enable: enable/disable switch
  *
  *	Sets the pattern filters indicated in @map to mask out the bytes
  *	specified in @mask0/@mask1 in received packets and compare the CRC of
  *	the resulting packet against @crc.  If @enable is %true pattern-based
  *	WoL is enabled, otherwise disabled.
  */
 int t4_wol_pat_enable(struct adapter *adap, unsigned int port, unsigned int map,
 		      u64 mask0, u64 mask1, unsigned int crc, bool enable)
 {
 	int i;
 	u32 port_cfg_reg;
 
 	if (is_t4(adap))
 		port_cfg_reg = PORT_REG(port, A_XGMAC_PORT_CFG2);
 	else
 		port_cfg_reg = T5_PORT_REG(port, A_MAC_PORT_CFG2);
 
 	if (!enable) {
 		t4_set_reg_field(adap, port_cfg_reg, F_PATEN, 0);
 		return 0;
 	}
 	if (map > 0xff)
 		return -EINVAL;
 
 #define EPIO_REG(name) \
 	(is_t4(adap) ? PORT_REG(port, A_XGMAC_PORT_EPIO_##name) : \
 	T5_PORT_REG(port, A_MAC_PORT_EPIO_##name))
 
 	t4_write_reg(adap, EPIO_REG(DATA1), mask0 >> 32);
 	t4_write_reg(adap, EPIO_REG(DATA2), mask1);
 	t4_write_reg(adap, EPIO_REG(DATA3), mask1 >> 32);
 
 	for (i = 0; i < NWOL_PAT; i++, map >>= 1) {
 		if (!(map & 1))
 			continue;
 
 		/* write byte masks */
 		t4_write_reg(adap, EPIO_REG(DATA0), mask0);
 		t4_write_reg(adap, EPIO_REG(OP), V_ADDRESS(i) | F_EPIOWR);
 		t4_read_reg(adap, EPIO_REG(OP));                /* flush */
 		if (t4_read_reg(adap, EPIO_REG(OP)) & F_BUSY)
 			return -ETIMEDOUT;
 
 		/* write CRC */
 		t4_write_reg(adap, EPIO_REG(DATA0), crc);
 		t4_write_reg(adap, EPIO_REG(OP), V_ADDRESS(i + 32) | F_EPIOWR);
 		t4_read_reg(adap, EPIO_REG(OP));                /* flush */
 		if (t4_read_reg(adap, EPIO_REG(OP)) & F_BUSY)
 			return -ETIMEDOUT;
 	}
 #undef EPIO_REG
 
 	t4_set_reg_field(adap, port_cfg_reg, 0, F_PATEN);
 	return 0;
 }
 
 /**
  *	t4_mk_filtdelwr - create a delete filter WR
  *	@ftid: the filter ID
  *	@wr: the filter work request to populate
  *	@qid: ingress queue to receive the delete notification
  *
  *	Creates a filter work request to delete the supplied filter.  If @qid is
  *	negative the delete notification is suppressed.
  */
 void t4_mk_filtdelwr(unsigned int ftid, struct fw_filter_wr *wr, int qid)
 {
 	memset(wr, 0, sizeof(*wr));
 	wr->op_pkd = htonl(V_FW_WR_OP(FW_FILTER_WR));
 	wr->len16_pkd = htonl(V_FW_WR_LEN16(sizeof(*wr) / 16));
 	wr->tid_to_iq = htonl(V_FW_FILTER_WR_TID(ftid) |
 			      V_FW_FILTER_WR_NOREPLY(qid < 0));
 	wr->del_filter_to_l2tix = htonl(F_FW_FILTER_WR_DEL_FILTER);
 	if (qid >= 0)
 		wr->rx_chan_rx_rpl_iq = htons(V_FW_FILTER_WR_RX_RPL_IQ(qid));
 }
 
 #define INIT_CMD(var, cmd, rd_wr) do { \
 	(var).op_to_write = htonl(V_FW_CMD_OP(FW_##cmd##_CMD) | \
 				  F_FW_CMD_REQUEST | F_FW_CMD_##rd_wr); \
 	(var).retval_len16 = htonl(FW_LEN16(var)); \
 } while (0)
 
 int t4_fwaddrspace_write(struct adapter *adap, unsigned int mbox, u32 addr, u32 val)
 {
 	struct fw_ldst_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_addrspace = htonl(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST |
 		F_FW_CMD_WRITE | V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_FIRMWARE));
 	c.cycles_to_len16 = htonl(FW_LEN16(c));
 	c.u.addrval.addr = htonl(addr);
 	c.u.addrval.val = htonl(val);
 
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
 /**
  *	t4_mdio_rd - read a PHY register through MDIO
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@phy_addr: the PHY address
  *	@mmd: the PHY MMD to access (0 for clause 22 PHYs)
  *	@reg: the register to read
  *	@valp: where to store the value
  *
  *	Issues a FW command through the given mailbox to read a PHY register.
  */
 int t4_mdio_rd(struct adapter *adap, unsigned int mbox, unsigned int phy_addr,
 	       unsigned int mmd, unsigned int reg, unsigned int *valp)
 {
 	int ret;
 	struct fw_ldst_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_addrspace = htonl(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST |
 		F_FW_CMD_READ | V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MDIO));
 	c.cycles_to_len16 = htonl(FW_LEN16(c));
 	c.u.mdio.paddr_mmd = htons(V_FW_LDST_CMD_PADDR(phy_addr) |
 				   V_FW_LDST_CMD_MMD(mmd));
 	c.u.mdio.raddr = htons(reg);
 
 	ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
 	if (ret == 0)
 		*valp = ntohs(c.u.mdio.rval);
 	return ret;
 }
 
 /**
  *	t4_mdio_wr - write a PHY register through MDIO
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@phy_addr: the PHY address
  *	@mmd: the PHY MMD to access (0 for clause 22 PHYs)
  *	@reg: the register to write
  *	@valp: value to write
  *
  *	Issues a FW command through the given mailbox to write a PHY register.
  */
 int t4_mdio_wr(struct adapter *adap, unsigned int mbox, unsigned int phy_addr,
 	       unsigned int mmd, unsigned int reg, unsigned int val)
 {
 	struct fw_ldst_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_addrspace = htonl(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST |
 		F_FW_CMD_WRITE | V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MDIO));
 	c.cycles_to_len16 = htonl(FW_LEN16(c));
 	c.u.mdio.paddr_mmd = htons(V_FW_LDST_CMD_PADDR(phy_addr) |
 				   V_FW_LDST_CMD_MMD(mmd));
 	c.u.mdio.raddr = htons(reg);
 	c.u.mdio.rval = htons(val);
 
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
 /**
  *	t4_i2c_rd - read I2C data from adapter
  *	@adap: the adapter
  *	@port: Port number if per-port device; <0 if not
  *	@devid: per-port device ID or absolute device ID
  *	@offset: byte offset into device I2C space
  *	@len: byte length of I2C space data
  *	@buf: buffer in which to return I2C data
  *
  *	Reads the I2C data from the indicated device and location.
  */
 int t4_i2c_rd(struct adapter *adap, unsigned int mbox,
 	      int port, unsigned int devid,
 	      unsigned int offset, unsigned int len,
 	      u8 *buf)
 {
 	struct fw_ldst_cmd ldst;
 	int ret;
 
 	if (port >= 4 ||
 	    devid >= 256 ||
 	    offset >= 256 ||
 	    len > sizeof ldst.u.i2c.data)
 		return -EINVAL;
 
 	memset(&ldst, 0, sizeof ldst);
 	ldst.op_to_addrspace =
 		cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) |
 			    F_FW_CMD_REQUEST |
 			    F_FW_CMD_READ |
 			    V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_I2C));
 	ldst.cycles_to_len16 = cpu_to_be32(FW_LEN16(ldst));
 	ldst.u.i2c.pid = (port < 0 ? 0xff : port);
 	ldst.u.i2c.did = devid;
 	ldst.u.i2c.boffset = offset;
 	ldst.u.i2c.blen = len;
 	ret = t4_wr_mbox(adap, mbox, &ldst, sizeof ldst, &ldst);
 	if (!ret)
 		memcpy(buf, ldst.u.i2c.data, len);
 	return ret;
 }
 
 /**
  *	t4_i2c_wr - write I2C data to adapter
  *	@adap: the adapter
  *	@port: Port number if per-port device; <0 if not
  *	@devid: per-port device ID or absolute device ID
  *	@offset: byte offset into device I2C space
  *	@len: byte length of I2C space data
  *	@buf: buffer containing new I2C data
  *
  *	Write the I2C data to the indicated device and location.
  */
 int t4_i2c_wr(struct adapter *adap, unsigned int mbox,
 	      int port, unsigned int devid,
 	      unsigned int offset, unsigned int len,
 	      u8 *buf)
 {
 	struct fw_ldst_cmd ldst;
 
 	if (port >= 4 ||
 	    devid >= 256 ||
 	    offset >= 256 ||
 	    len > sizeof ldst.u.i2c.data)
 		return -EINVAL;
 
 	memset(&ldst, 0, sizeof ldst);
 	ldst.op_to_addrspace =
 		cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) |
 			    F_FW_CMD_REQUEST |
 			    F_FW_CMD_WRITE |
 			    V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_I2C));
 	ldst.cycles_to_len16 = cpu_to_be32(FW_LEN16(ldst));
 	ldst.u.i2c.pid = (port < 0 ? 0xff : port);
 	ldst.u.i2c.did = devid;
 	ldst.u.i2c.boffset = offset;
 	ldst.u.i2c.blen = len;
 	memcpy(ldst.u.i2c.data, buf, len);
 	return t4_wr_mbox(adap, mbox, &ldst, sizeof ldst, &ldst);
 }
 
 /**
  *	t4_sge_ctxt_flush - flush the SGE context cache
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *
  *	Issues a FW command through the given mailbox to flush the
  *	SGE context cache.
  */
 int t4_sge_ctxt_flush(struct adapter *adap, unsigned int mbox)
 {
 	int ret;
 	struct fw_ldst_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_addrspace = htonl(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST |
 			F_FW_CMD_READ |
 			V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_SGE_EGRC));
 	c.cycles_to_len16 = htonl(FW_LEN16(c));
 	c.u.idctxt.msg_ctxtflush = htonl(F_FW_LDST_CMD_CTXTFLUSH);
 
 	ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
 	return ret;
 }
 
 /**
  *	t4_sge_ctxt_rd - read an SGE context through FW
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@cid: the context id
  *	@ctype: the context type
  *	@data: where to store the context data
  *
  *	Issues a FW command through the given mailbox to read an SGE context.
  */
 int t4_sge_ctxt_rd(struct adapter *adap, unsigned int mbox, unsigned int cid,
 		   enum ctxt_type ctype, u32 *data)
 {
 	int ret;
 	struct fw_ldst_cmd c;
 
 	if (ctype == CTXT_EGRESS)
 		ret = FW_LDST_ADDRSPC_SGE_EGRC;
 	else if (ctype == CTXT_INGRESS)
 		ret = FW_LDST_ADDRSPC_SGE_INGC;
 	else if (ctype == CTXT_FLM)
 		ret = FW_LDST_ADDRSPC_SGE_FLMC;
 	else
 		ret = FW_LDST_ADDRSPC_SGE_CONMC;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_addrspace = htonl(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST |
 				  F_FW_CMD_READ | V_FW_LDST_CMD_ADDRSPACE(ret));
 	c.cycles_to_len16 = htonl(FW_LEN16(c));
 	c.u.idctxt.physid = htonl(cid);
 
 	ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
 	if (ret == 0) {
 		data[0] = ntohl(c.u.idctxt.ctxt_data0);
 		data[1] = ntohl(c.u.idctxt.ctxt_data1);
 		data[2] = ntohl(c.u.idctxt.ctxt_data2);
 		data[3] = ntohl(c.u.idctxt.ctxt_data3);
 		data[4] = ntohl(c.u.idctxt.ctxt_data4);
 		data[5] = ntohl(c.u.idctxt.ctxt_data5);
 	}
 	return ret;
 }
 
 /**
  *	t4_sge_ctxt_rd_bd - read an SGE context bypassing FW
  *	@adap: the adapter
  *	@cid: the context id
  *	@ctype: the context type
  *	@data: where to store the context data
  *
  *	Reads an SGE context directly, bypassing FW.  This is only for
  *	debugging when FW is unavailable.
  */
 int t4_sge_ctxt_rd_bd(struct adapter *adap, unsigned int cid, enum ctxt_type ctype,
 		      u32 *data)
 {
 	int i, ret;
 
 	t4_write_reg(adap, A_SGE_CTXT_CMD, V_CTXTQID(cid) | V_CTXTTYPE(ctype));
 	ret = t4_wait_op_done(adap, A_SGE_CTXT_CMD, F_BUSY, 0, 3, 1);
 	if (!ret)
 		for (i = A_SGE_CTXT_DATA0; i <= A_SGE_CTXT_DATA5; i += 4)
 			*data++ = t4_read_reg(adap, i);
 	return ret;
 }
 
 /**
  *	t4_fw_hello - establish communication with FW
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@evt_mbox: mailbox to receive async FW events
  *	@master: specifies the caller's willingness to be the device master
  *	@state: returns the current device state (if non-NULL)
  *
  *	Issues a command to establish communication with FW.  Returns either
  *	an error (negative integer) or the mailbox of the Master PF.
  */
 int t4_fw_hello(struct adapter *adap, unsigned int mbox, unsigned int evt_mbox,
 		enum dev_master master, enum dev_state *state)
 {
 	int ret;
 	struct fw_hello_cmd c;
 	u32 v;
 	unsigned int master_mbox;
 	int retries = FW_CMD_HELLO_RETRIES;
 
 retry:
 	memset(&c, 0, sizeof(c));
 	INIT_CMD(c, HELLO, WRITE);
 	c.err_to_clearinit = htonl(
 		V_FW_HELLO_CMD_MASTERDIS(master == MASTER_CANT) |
 		V_FW_HELLO_CMD_MASTERFORCE(master == MASTER_MUST) |
 		V_FW_HELLO_CMD_MBMASTER(master == MASTER_MUST ? mbox :
 			M_FW_HELLO_CMD_MBMASTER) |
 		V_FW_HELLO_CMD_MBASYNCNOT(evt_mbox) |
 		V_FW_HELLO_CMD_STAGE(FW_HELLO_CMD_STAGE_OS) |
 		F_FW_HELLO_CMD_CLEARINIT);
 
 	/*
 	 * Issue the HELLO command to the firmware.  If it's not successful
 	 * but indicates that we got a "busy" or "timeout" condition, retry
 	 * the HELLO until we exhaust our retry limit.  If we do exceed our
 	 * retry limit, check to see if the firmware left us any error
 	 * information and report that if so ...
 	 */
 	ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
 	if (ret != FW_SUCCESS) {
 		if ((ret == -EBUSY || ret == -ETIMEDOUT) && retries-- > 0)
 			goto retry;
 		if (t4_read_reg(adap, A_PCIE_FW) & F_PCIE_FW_ERR)
 			t4_report_fw_error(adap);
 		return ret;
 	}
 
 	v = ntohl(c.err_to_clearinit);
 	master_mbox = G_FW_HELLO_CMD_MBMASTER(v);
 	if (state) {
 		if (v & F_FW_HELLO_CMD_ERR)
 			*state = DEV_STATE_ERR;
 		else if (v & F_FW_HELLO_CMD_INIT)
 			*state = DEV_STATE_INIT;
 		else
 			*state = DEV_STATE_UNINIT;
 	}
 
 	/*
 	 * If we're not the Master PF then we need to wait around for the
 	 * Master PF Driver to finish setting up the adapter.
 	 *
 	 * Note that we also do this wait if we're a non-Master-capable PF and
 	 * there is no current Master PF; a Master PF may show up momentarily
 	 * and we wouldn't want to fail pointlessly.  (This can happen when an
 	 * OS loads lots of different drivers rapidly at the same time).  In
 	 * this case, the Master PF returned by the firmware will be
 	 * M_PCIE_FW_MASTER so the test below will work ...
 	 */
 	if ((v & (F_FW_HELLO_CMD_ERR|F_FW_HELLO_CMD_INIT)) == 0 &&
 	    master_mbox != mbox) {
 		int waiting = FW_CMD_HELLO_TIMEOUT;
 
 		/*
 		 * Wait for the firmware to either indicate an error or
 		 * initialized state.  If we see either of these we bail out
 		 * and report the issue to the caller.  If we exhaust the
 		 * "hello timeout" and we haven't exhausted our retries, try
 		 * again.  Otherwise bail with a timeout error.
 		 */
 		for (;;) {
 			u32 pcie_fw;
 
 			msleep(50);
 			waiting -= 50;
 
 			/*
 			 * If neither Error nor Initialialized are indicated
 			 * by the firmware keep waiting till we exhaust our
 			 * timeout ... and then retry if we haven't exhausted
 			 * our retries ...
 			 */
 			pcie_fw = t4_read_reg(adap, A_PCIE_FW);
 			if (!(pcie_fw & (F_PCIE_FW_ERR|F_PCIE_FW_INIT))) {
 				if (waiting <= 0) {
 					if (retries-- > 0)
 						goto retry;
 
 					return -ETIMEDOUT;
 				}
 				continue;
 			}
 
 			/*
 			 * We either have an Error or Initialized condition
 			 * report errors preferentially.
 			 */
 			if (state) {
 				if (pcie_fw & F_PCIE_FW_ERR)
 					*state = DEV_STATE_ERR;
 				else if (pcie_fw & F_PCIE_FW_INIT)
 					*state = DEV_STATE_INIT;
 			}
 
 			/*
 			 * If we arrived before a Master PF was selected and
 			 * there's not a valid Master PF, grab its identity
 			 * for our caller.
 			 */
 			if (master_mbox == M_PCIE_FW_MASTER &&
 			    (pcie_fw & F_PCIE_FW_MASTER_VLD))
 				master_mbox = G_PCIE_FW_MASTER(pcie_fw);
 			break;
 		}
 	}
 
 	return master_mbox;
 }
 
 /**
  *	t4_fw_bye - end communication with FW
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *
  *	Issues a command to terminate communication with FW.
  */
 int t4_fw_bye(struct adapter *adap, unsigned int mbox)
 {
 	struct fw_bye_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	INIT_CMD(c, BYE, WRITE);
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
 /**
  *	t4_fw_reset - issue a reset to FW
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@reset: specifies the type of reset to perform
  *
  *	Issues a reset command of the specified type to FW.
  */
 int t4_fw_reset(struct adapter *adap, unsigned int mbox, int reset)
 {
 	struct fw_reset_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	INIT_CMD(c, RESET, WRITE);
 	c.val = htonl(reset);
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
 /**
  *	t4_fw_halt - issue a reset/halt to FW and put uP into RESET
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW RESET command (if desired)
  *	@force: force uP into RESET even if FW RESET command fails
  *
  *	Issues a RESET command to firmware (if desired) with a HALT indication
  *	and then puts the microprocessor into RESET state.  The RESET command
  *	will only be issued if a legitimate mailbox is provided (mbox <=
  *	M_PCIE_FW_MASTER).
  *
  *	This is generally used in order for the host to safely manipulate the
  *	adapter without fear of conflicting with whatever the firmware might
  *	be doing.  The only way out of this state is to RESTART the firmware
  *	...
  */
 int t4_fw_halt(struct adapter *adap, unsigned int mbox, int force)
 {
 	int ret = 0;
 
 	/*
 	 * If a legitimate mailbox is provided, issue a RESET command
 	 * with a HALT indication.
 	 */
 	if (mbox <= M_PCIE_FW_MASTER) {
 		struct fw_reset_cmd c;
 
 		memset(&c, 0, sizeof(c));
 		INIT_CMD(c, RESET, WRITE);
 		c.val = htonl(F_PIORST | F_PIORSTMODE);
 		c.halt_pkd = htonl(F_FW_RESET_CMD_HALT);
 		ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 	}
 
 	/*
 	 * Normally we won't complete the operation if the firmware RESET
 	 * command fails but if our caller insists we'll go ahead and put the
 	 * uP into RESET.  This can be useful if the firmware is hung or even
 	 * missing ...  We'll have to take the risk of putting the uP into
 	 * RESET without the cooperation of firmware in that case.
 	 *
 	 * We also force the firmware's HALT flag to be on in case we bypassed
 	 * the firmware RESET command above or we're dealing with old firmware
 	 * which doesn't have the HALT capability.  This will serve as a flag
 	 * for the incoming firmware to know that it's coming out of a HALT
 	 * rather than a RESET ... if it's new enough to understand that ...
 	 */
 	if (ret == 0 || force) {
 		t4_set_reg_field(adap, A_CIM_BOOT_CFG, F_UPCRST, F_UPCRST);
 		t4_set_reg_field(adap, A_PCIE_FW, F_PCIE_FW_HALT, F_PCIE_FW_HALT);
 	}
 
 	/*
 	 * And we always return the result of the firmware RESET command
 	 * even when we force the uP into RESET ...
 	 */
 	return ret;
 }
 
 /**
  *	t4_fw_restart - restart the firmware by taking the uP out of RESET
  *	@adap: the adapter
  *	@reset: if we want to do a RESET to restart things
  *
  *	Restart firmware previously halted by t4_fw_halt().  On successful
  *	return the previous PF Master remains as the new PF Master and there
  *	is no need to issue a new HELLO command, etc.
  *
  *	We do this in two ways:
  *
  *	 1. If we're dealing with newer firmware we'll simply want to take
  *	    the chip's microprocessor out of RESET.  This will cause the
  *	    firmware to start up from its start vector.  And then we'll loop
  *	    until the firmware indicates it's started again (PCIE_FW.HALT
  *	    reset to 0) or we timeout.
  *
  *	 2. If we're dealing with older firmware then we'll need to RESET
  *	    the chip since older firmware won't recognize the PCIE_FW.HALT
  *	    flag and automatically RESET itself on startup.
  */
 int t4_fw_restart(struct adapter *adap, unsigned int mbox, int reset)
 {
 	if (reset) {
 		/*
 		 * Since we're directing the RESET instead of the firmware
 		 * doing it automatically, we need to clear the PCIE_FW.HALT
 		 * bit.
 		 */
 		t4_set_reg_field(adap, A_PCIE_FW, F_PCIE_FW_HALT, 0);
 
 		/*
 		 * If we've been given a valid mailbox, first try to get the
 		 * firmware to do the RESET.  If that works, great and we can
 		 * return success.  Otherwise, if we haven't been given a
 		 * valid mailbox or the RESET command failed, fall back to
 		 * hitting the chip with a hammer.
 		 */
 		if (mbox <= M_PCIE_FW_MASTER) {
 			t4_set_reg_field(adap, A_CIM_BOOT_CFG, F_UPCRST, 0);
 			msleep(100);
 			if (t4_fw_reset(adap, mbox,
 					F_PIORST | F_PIORSTMODE) == 0)
 				return 0;
 		}
 
 		t4_write_reg(adap, A_PL_RST, F_PIORST | F_PIORSTMODE);
 		msleep(2000);
 	} else {
 		int ms;
 
 		t4_set_reg_field(adap, A_CIM_BOOT_CFG, F_UPCRST, 0);
 		for (ms = 0; ms < FW_CMD_MAX_TIMEOUT; ) {
 			if (!(t4_read_reg(adap, A_PCIE_FW) & F_PCIE_FW_HALT))
 				return FW_SUCCESS;
 			msleep(100);
 			ms += 100;
 		}
 		return -ETIMEDOUT;
 	}
 	return 0;
 }
 
 /**
  *	t4_fw_upgrade - perform all of the steps necessary to upgrade FW
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW RESET command (if desired)
  *	@fw_data: the firmware image to write
  *	@size: image size
  *	@force: force upgrade even if firmware doesn't cooperate
  *
  *	Perform all of the steps necessary for upgrading an adapter's
  *	firmware image.  Normally this requires the cooperation of the
  *	existing firmware in order to halt all existing activities
  *	but if an invalid mailbox token is passed in we skip that step
  *	(though we'll still put the adapter microprocessor into RESET in
  *	that case).
  *
  *	On successful return the new firmware will have been loaded and
  *	the adapter will have been fully RESET losing all previous setup
  *	state.  On unsuccessful return the adapter may be completely hosed ...
  *	positive errno indicates that the adapter is ~probably~ intact, a
  *	negative errno indicates that things are looking bad ...
  */
 int t4_fw_upgrade(struct adapter *adap, unsigned int mbox,
 		  const u8 *fw_data, unsigned int size, int force)
 {
 	const struct fw_hdr *fw_hdr = (const struct fw_hdr *)fw_data;
 	unsigned int bootstrap = ntohl(fw_hdr->magic) == FW_HDR_MAGIC_BOOTSTRAP;
 	int reset, ret;
 
 	if (!bootstrap) {
 		ret = t4_fw_halt(adap, mbox, force);
 		if (ret < 0 && !force)
 			return ret;
 	}
 
 	ret = t4_load_fw(adap, fw_data, size);
 	if (ret < 0 || bootstrap)
 		return ret;
 
 	/*
 	 * Older versions of the firmware don't understand the new
 	 * PCIE_FW.HALT flag and so won't know to perform a RESET when they
 	 * restart.  So for newly loaded older firmware we'll have to do the
 	 * RESET for it so it starts up on a clean slate.  We can tell if
 	 * the newly loaded firmware will handle this right by checking
 	 * its header flags to see if it advertises the capability.
 	 */
 	reset = ((ntohl(fw_hdr->flags) & FW_HDR_FLAGS_RESET_HALT) == 0);
 	return t4_fw_restart(adap, mbox, reset);
 }
 
 /**
  *	t4_fw_initialize - ask FW to initialize the device
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *
  *	Issues a command to FW to partially initialize the device.  This
  *	performs initialization that generally doesn't depend on user input.
  */
 int t4_fw_initialize(struct adapter *adap, unsigned int mbox)
 {
 	struct fw_initialize_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	INIT_CMD(c, INITIALIZE, WRITE);
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
 /**
  *	t4_query_params - query FW or device parameters
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@pf: the PF
  *	@vf: the VF
  *	@nparams: the number of parameters
  *	@params: the parameter names
  *	@val: the parameter values
  *
  *	Reads the value of FW or device parameters.  Up to 7 parameters can be
  *	queried at once.
  */
 int t4_query_params(struct adapter *adap, unsigned int mbox, unsigned int pf,
 		    unsigned int vf, unsigned int nparams, const u32 *params,
 		    u32 *val)
 {
 	int i, ret;
 	struct fw_params_cmd c;
 	__be32 *p = &c.param[0].mnem;
 
 	if (nparams > 7)
 		return -EINVAL;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_PARAMS_CMD) | F_FW_CMD_REQUEST |
 			    F_FW_CMD_READ | V_FW_PARAMS_CMD_PFN(pf) |
 			    V_FW_PARAMS_CMD_VFN(vf));
 	c.retval_len16 = htonl(FW_LEN16(c));
 
 	for (i = 0; i < nparams; i++, p += 2, params++)
 		*p = htonl(*params);
 
 	ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
 	if (ret == 0)
 		for (i = 0, p = &c.param[0].val; i < nparams; i++, p += 2)
 			*val++ = ntohl(*p);
 	return ret;
 }
 
 /**
  *	t4_set_params - sets FW or device parameters
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@pf: the PF
  *	@vf: the VF
  *	@nparams: the number of parameters
  *	@params: the parameter names
  *	@val: the parameter values
  *
  *	Sets the value of FW or device parameters.  Up to 7 parameters can be
  *	specified at once.
  */
 int t4_set_params(struct adapter *adap, unsigned int mbox, unsigned int pf,
 		  unsigned int vf, unsigned int nparams, const u32 *params,
 		  const u32 *val)
 {
 	struct fw_params_cmd c;
 	__be32 *p = &c.param[0].mnem;
 
 	if (nparams > 7)
 		return -EINVAL;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_PARAMS_CMD) | F_FW_CMD_REQUEST |
 			    F_FW_CMD_WRITE | V_FW_PARAMS_CMD_PFN(pf) |
 			    V_FW_PARAMS_CMD_VFN(vf));
 	c.retval_len16 = htonl(FW_LEN16(c));
 
 	while (nparams--) {
 		*p++ = htonl(*params);
 		params++;
 		*p++ = htonl(*val);
 		val++;
 	}
 
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
 /**
  *	t4_cfg_pfvf - configure PF/VF resource limits
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@pf: the PF being configured
  *	@vf: the VF being configured
  *	@txq: the max number of egress queues
  *	@txq_eth_ctrl: the max number of egress Ethernet or control queues
  *	@rxqi: the max number of interrupt-capable ingress queues
  *	@rxq: the max number of interruptless ingress queues
  *	@tc: the PCI traffic class
  *	@vi: the max number of virtual interfaces
  *	@cmask: the channel access rights mask for the PF/VF
  *	@pmask: the port access rights mask for the PF/VF
  *	@nexact: the maximum number of exact MPS filters
  *	@rcaps: read capabilities
  *	@wxcaps: write/execute capabilities
  *
  *	Configures resource limits and capabilities for a physical or virtual
  *	function.
  */
 int t4_cfg_pfvf(struct adapter *adap, unsigned int mbox, unsigned int pf,
 		unsigned int vf, unsigned int txq, unsigned int txq_eth_ctrl,
 		unsigned int rxqi, unsigned int rxq, unsigned int tc,
 		unsigned int vi, unsigned int cmask, unsigned int pmask,
 		unsigned int nexact, unsigned int rcaps, unsigned int wxcaps)
 {
 	struct fw_pfvf_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_PFVF_CMD) | F_FW_CMD_REQUEST |
 			    F_FW_CMD_WRITE | V_FW_PFVF_CMD_PFN(pf) |
 			    V_FW_PFVF_CMD_VFN(vf));
 	c.retval_len16 = htonl(FW_LEN16(c));
 	c.niqflint_niq = htonl(V_FW_PFVF_CMD_NIQFLINT(rxqi) |
 			       V_FW_PFVF_CMD_NIQ(rxq));
 	c.type_to_neq = htonl(V_FW_PFVF_CMD_CMASK(cmask) |
 			      V_FW_PFVF_CMD_PMASK(pmask) |
 			      V_FW_PFVF_CMD_NEQ(txq));
 	c.tc_to_nexactf = htonl(V_FW_PFVF_CMD_TC(tc) | V_FW_PFVF_CMD_NVI(vi) |
 				V_FW_PFVF_CMD_NEXACTF(nexact));
 	c.r_caps_to_nethctrl = htonl(V_FW_PFVF_CMD_R_CAPS(rcaps) |
 				     V_FW_PFVF_CMD_WX_CAPS(wxcaps) |
 				     V_FW_PFVF_CMD_NETHCTRL(txq_eth_ctrl));
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
 /**
  *	t4_alloc_vi_func - allocate a virtual interface
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@port: physical port associated with the VI
  *	@pf: the PF owning the VI
  *	@vf: the VF owning the VI
  *	@nmac: number of MAC addresses needed (1 to 5)
  *	@mac: the MAC addresses of the VI
  *	@rss_size: size of RSS table slice associated with this VI
  *	@portfunc: which Port Application Function MAC Address is desired
  *	@idstype: Intrusion Detection Type
  *
  *	Allocates a virtual interface for the given physical port.  If @mac is
  *	not %NULL it contains the MAC addresses of the VI as assigned by FW.
  *	@mac should be large enough to hold @nmac Ethernet addresses, they are
  *	stored consecutively so the space needed is @nmac * 6 bytes.
  *	Returns a negative error number or the non-negative VI id.
  */
 int t4_alloc_vi_func(struct adapter *adap, unsigned int mbox,
 		     unsigned int port, unsigned int pf, unsigned int vf,
-		     unsigned int nmac, u8 *mac, unsigned int *rss_size,
+		     unsigned int nmac, u8 *mac, u16 *rss_size,
 		     unsigned int portfunc, unsigned int idstype)
 {
 	int ret;
 	struct fw_vi_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_VI_CMD) | F_FW_CMD_REQUEST |
 			    F_FW_CMD_WRITE | F_FW_CMD_EXEC |
 			    V_FW_VI_CMD_PFN(pf) | V_FW_VI_CMD_VFN(vf));
 	c.alloc_to_len16 = htonl(F_FW_VI_CMD_ALLOC | FW_LEN16(c));
 	c.type_to_viid = htons(V_FW_VI_CMD_TYPE(idstype) |
 			       V_FW_VI_CMD_FUNC(portfunc));
 	c.portid_pkd = V_FW_VI_CMD_PORTID(port);
 	c.nmac = nmac - 1;
 
 	ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
 	if (ret)
 		return ret;
 
 	if (mac) {
 		memcpy(mac, c.mac, sizeof(c.mac));
 		switch (nmac) {
 		case 5:
 			memcpy(mac + 24, c.nmac3, sizeof(c.nmac3));
 		case 4:
 			memcpy(mac + 18, c.nmac2, sizeof(c.nmac2));
 		case 3:
 			memcpy(mac + 12, c.nmac1, sizeof(c.nmac1));
 		case 2:
 			memcpy(mac + 6,  c.nmac0, sizeof(c.nmac0));
 		}
 	}
 	if (rss_size)
 		*rss_size = G_FW_VI_CMD_RSSSIZE(ntohs(c.norss_rsssize));
 	return G_FW_VI_CMD_VIID(htons(c.type_to_viid));
 }
 
 /**
  *	t4_alloc_vi - allocate an [Ethernet Function] virtual interface
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@port: physical port associated with the VI
  *	@pf: the PF owning the VI
  *	@vf: the VF owning the VI
  *	@nmac: number of MAC addresses needed (1 to 5)
  *	@mac: the MAC addresses of the VI
  *	@rss_size: size of RSS table slice associated with this VI
  *
  *	backwards compatible and convieniance routine to allocate a Virtual
  *	Interface with a Ethernet Port Application Function and Intrustion
  *	Detection System disabled.
  */
 int t4_alloc_vi(struct adapter *adap, unsigned int mbox, unsigned int port,
 		unsigned int pf, unsigned int vf, unsigned int nmac, u8 *mac,
-		unsigned int *rss_size)
+		u16 *rss_size)
 {
 	return t4_alloc_vi_func(adap, mbox, port, pf, vf, nmac, mac, rss_size,
 				FW_VI_FUNC_ETH, 0);
 }
 
 /**
  *	t4_free_vi - free a virtual interface
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@pf: the PF owning the VI
  *	@vf: the VF owning the VI
  *	@viid: virtual interface identifiler
  *
  *	Free a previously allocated virtual interface.
  */
 int t4_free_vi(struct adapter *adap, unsigned int mbox, unsigned int pf,
 	       unsigned int vf, unsigned int viid)
 {
 	struct fw_vi_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_VI_CMD) |
 			    F_FW_CMD_REQUEST |
 			    F_FW_CMD_EXEC |
 			    V_FW_VI_CMD_PFN(pf) |
 			    V_FW_VI_CMD_VFN(vf));
 	c.alloc_to_len16 = htonl(F_FW_VI_CMD_FREE | FW_LEN16(c));
 	c.type_to_viid = htons(V_FW_VI_CMD_VIID(viid));
 
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
 }
 
 /**
  *	t4_set_rxmode - set Rx properties of a virtual interface
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@viid: the VI id
  *	@mtu: the new MTU or -1
  *	@promisc: 1 to enable promiscuous mode, 0 to disable it, -1 no change
  *	@all_multi: 1 to enable all-multi mode, 0 to disable it, -1 no change
  *	@bcast: 1 to enable broadcast Rx, 0 to disable it, -1 no change
  *	@vlanex: 1 to enable HVLAN extraction, 0 to disable it, -1 no change
  *	@sleep_ok: if true we may sleep while awaiting command completion
  *
  *	Sets Rx properties of a virtual interface.
  */
 int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		  int mtu, int promisc, int all_multi, int bcast, int vlanex,
 		  bool sleep_ok)
 {
 	struct fw_vi_rxmode_cmd c;
 
 	/* convert to FW values */
 	if (mtu < 0)
 		mtu = M_FW_VI_RXMODE_CMD_MTU;
 	if (promisc < 0)
 		promisc = M_FW_VI_RXMODE_CMD_PROMISCEN;
 	if (all_multi < 0)
 		all_multi = M_FW_VI_RXMODE_CMD_ALLMULTIEN;
 	if (bcast < 0)
 		bcast = M_FW_VI_RXMODE_CMD_BROADCASTEN;
 	if (vlanex < 0)
 		vlanex = M_FW_VI_RXMODE_CMD_VLANEXEN;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_viid = htonl(V_FW_CMD_OP(FW_VI_RXMODE_CMD) | F_FW_CMD_REQUEST |
 			     F_FW_CMD_WRITE | V_FW_VI_RXMODE_CMD_VIID(viid));
 	c.retval_len16 = htonl(FW_LEN16(c));
 	c.mtu_to_vlanexen = htonl(V_FW_VI_RXMODE_CMD_MTU(mtu) |
 				  V_FW_VI_RXMODE_CMD_PROMISCEN(promisc) |
 				  V_FW_VI_RXMODE_CMD_ALLMULTIEN(all_multi) |
 				  V_FW_VI_RXMODE_CMD_BROADCASTEN(bcast) |
 				  V_FW_VI_RXMODE_CMD_VLANEXEN(vlanex));
 	return t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), NULL, sleep_ok);
 }
 
 /**
  *	t4_alloc_mac_filt - allocates exact-match filters for MAC addresses
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@viid: the VI id
  *	@free: if true any existing filters for this VI id are first removed
  *	@naddr: the number of MAC addresses to allocate filters for (up to 7)
  *	@addr: the MAC address(es)
  *	@idx: where to store the index of each allocated filter
  *	@hash: pointer to hash address filter bitmap
  *	@sleep_ok: call is allowed to sleep
  *
  *	Allocates an exact-match filter for each of the supplied addresses and
  *	sets it to the corresponding address.  If @idx is not %NULL it should
  *	have at least @naddr entries, each of which will be set to the index of
  *	the filter allocated for the corresponding MAC address.  If a filter
  *	could not be allocated for an address its index is set to 0xffff.
  *	If @hash is not %NULL addresses that fail to allocate an exact filter
  *	are hashed and update the hash filter bitmap pointed at by @hash.
  *
  *	Returns a negative error number or the number of filters allocated.
  */
 int t4_alloc_mac_filt(struct adapter *adap, unsigned int mbox,
 		      unsigned int viid, bool free, unsigned int naddr,
 		      const u8 **addr, u16 *idx, u64 *hash, bool sleep_ok)
 {
 	int offset, ret = 0;
 	struct fw_vi_mac_cmd c;
 	unsigned int nfilters = 0;
 	unsigned int max_naddr = is_t4(adap) ?
 				       NUM_MPS_CLS_SRAM_L_INSTANCES :
 				       NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
 	unsigned int rem = naddr;
 
 	if (naddr > max_naddr)
 		return -EINVAL;
 
 	for (offset = 0; offset < naddr ; /**/) {
 		unsigned int fw_naddr = (rem < ARRAY_SIZE(c.u.exact)
 					 ? rem
 					 : ARRAY_SIZE(c.u.exact));
 		size_t len16 = DIV_ROUND_UP(offsetof(struct fw_vi_mac_cmd,
 						     u.exact[fw_naddr]), 16);
 		struct fw_vi_mac_exact *p;
 		int i;
 
 		memset(&c, 0, sizeof(c));
 		c.op_to_viid = htonl(V_FW_CMD_OP(FW_VI_MAC_CMD) |
 				     F_FW_CMD_REQUEST |
 				     F_FW_CMD_WRITE |
 				     V_FW_CMD_EXEC(free) |
 				     V_FW_VI_MAC_CMD_VIID(viid));
 		c.freemacs_to_len16 = htonl(V_FW_VI_MAC_CMD_FREEMACS(free) |
 					    V_FW_CMD_LEN16(len16));
 
 		for (i = 0, p = c.u.exact; i < fw_naddr; i++, p++) {
 			p->valid_to_idx = htons(
 				F_FW_VI_MAC_CMD_VALID |
 				V_FW_VI_MAC_CMD_IDX(FW_VI_MAC_ADD_MAC));
 			memcpy(p->macaddr, addr[offset+i], sizeof(p->macaddr));
 		}
 
 		/*
 		 * It's okay if we run out of space in our MAC address arena.
 		 * Some of the addresses we submit may get stored so we need
 		 * to run through the reply to see what the results were ...
 		 */
 		ret = t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), &c, sleep_ok);
 		if (ret && ret != -FW_ENOMEM)
 			break;
 
 		for (i = 0, p = c.u.exact; i < fw_naddr; i++, p++) {
 			u16 index = G_FW_VI_MAC_CMD_IDX(ntohs(p->valid_to_idx));
 
 			if (idx)
 				idx[offset+i] = (index >=  max_naddr
 						 ? 0xffff
 						 : index);
 			if (index < max_naddr)
 				nfilters++;
 			else if (hash)
 				*hash |= (1ULL << hash_mac_addr(addr[offset+i]));
 		}
 
 		free = false;
 		offset += fw_naddr;
 		rem -= fw_naddr;
 	}
 
 	if (ret == 0 || ret == -FW_ENOMEM)
 		ret = nfilters; 
 	return ret;
 }
 
 /**
  *	t4_change_mac - modifies the exact-match filter for a MAC address
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@viid: the VI id
  *	@idx: index of existing filter for old value of MAC address, or -1
  *	@addr: the new MAC address value
  *	@persist: whether a new MAC allocation should be persistent
  *	@add_smt: if true also add the address to the HW SMT
  *
  *	Modifies an exact-match filter and sets it to the new MAC address if
  *	@idx >= 0, or adds the MAC address to a new filter if @idx < 0.  In the
  *	latter case the address is added persistently if @persist is %true.
  *
  *	Note that in general it is not possible to modify the value of a given
  *	filter so the generic way to modify an address filter is to free the one
  *	being used by the old address value and allocate a new filter for the
  *	new address value.
  *
  *	Returns a negative error number or the index of the filter with the new
  *	MAC value.  Note that this index may differ from @idx.
  */
 int t4_change_mac(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		  int idx, const u8 *addr, bool persist, bool add_smt)
 {
 	int ret, mode;
 	struct fw_vi_mac_cmd c;
 	struct fw_vi_mac_exact *p = c.u.exact;
 	unsigned int max_mac_addr = is_t4(adap) ?
 				    NUM_MPS_CLS_SRAM_L_INSTANCES :
 				    NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
 
 	if (idx < 0)                             /* new allocation */
 		idx = persist ? FW_VI_MAC_ADD_PERSIST_MAC : FW_VI_MAC_ADD_MAC;
 	mode = add_smt ? FW_VI_MAC_SMT_AND_MPSTCAM : FW_VI_MAC_MPS_TCAM_ENTRY;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_viid = htonl(V_FW_CMD_OP(FW_VI_MAC_CMD) | F_FW_CMD_REQUEST |
 			     F_FW_CMD_WRITE | V_FW_VI_MAC_CMD_VIID(viid));
 	c.freemacs_to_len16 = htonl(V_FW_CMD_LEN16(1));
 	p->valid_to_idx = htons(F_FW_VI_MAC_CMD_VALID |
 				V_FW_VI_MAC_CMD_SMAC_RESULT(mode) |
 				V_FW_VI_MAC_CMD_IDX(idx));
 	memcpy(p->macaddr, addr, sizeof(p->macaddr));
 
 	ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
 	if (ret == 0) {
 		ret = G_FW_VI_MAC_CMD_IDX(ntohs(p->valid_to_idx));
 		if (ret >= max_mac_addr)
 			ret = -ENOMEM;
 	}
 	return ret;
 }
 
 /**
  *	t4_set_addr_hash - program the MAC inexact-match hash filter
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@viid: the VI id
  *	@ucast: whether the hash filter should also match unicast addresses
  *	@vec: the value to be written to the hash filter
  *	@sleep_ok: call is allowed to sleep
  *
  *	Sets the 64-bit inexact-match hash filter for a virtual interface.
  */
 int t4_set_addr_hash(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		     bool ucast, u64 vec, bool sleep_ok)
 {
 	struct fw_vi_mac_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_viid = htonl(V_FW_CMD_OP(FW_VI_MAC_CMD) | F_FW_CMD_REQUEST |
 			     F_FW_CMD_WRITE | V_FW_VI_ENABLE_CMD_VIID(viid));
 	c.freemacs_to_len16 = htonl(F_FW_VI_MAC_CMD_HASHVECEN |
 				    V_FW_VI_MAC_CMD_HASHUNIEN(ucast) |
 				    V_FW_CMD_LEN16(1));
 	c.u.hash.hashvec = cpu_to_be64(vec);
 	return t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), NULL, sleep_ok);
 }
 
 /**
  *	t4_enable_vi - enable/disable a virtual interface
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@viid: the VI id
  *	@rx_en: 1=enable Rx, 0=disable Rx
  *	@tx_en: 1=enable Tx, 0=disable Tx
  *
  *	Enables/disables a virtual interface.
  */
 int t4_enable_vi(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		 bool rx_en, bool tx_en)
 {
 	struct fw_vi_enable_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_viid = htonl(V_FW_CMD_OP(FW_VI_ENABLE_CMD) | F_FW_CMD_REQUEST |
 			     F_FW_CMD_EXEC | V_FW_VI_ENABLE_CMD_VIID(viid));
 	c.ien_to_len16 = htonl(V_FW_VI_ENABLE_CMD_IEN(rx_en) |
 			       V_FW_VI_ENABLE_CMD_EEN(tx_en) | FW_LEN16(c));
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
 /**
  *	t4_identify_port - identify a VI's port by blinking its LED
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@viid: the VI id
  *	@nblinks: how many times to blink LED at 2.5 Hz
  *
  *	Identifies a VI's port by blinking its LED.
  */
 int t4_identify_port(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		     unsigned int nblinks)
 {
 	struct fw_vi_enable_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_viid = htonl(V_FW_CMD_OP(FW_VI_ENABLE_CMD) | F_FW_CMD_REQUEST |
 			     F_FW_CMD_EXEC | V_FW_VI_ENABLE_CMD_VIID(viid));
 	c.ien_to_len16 = htonl(F_FW_VI_ENABLE_CMD_LED | FW_LEN16(c));
 	c.blinkdur = htons(nblinks);
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
 /**
  *	t4_iq_start_stop - enable/disable an ingress queue and its FLs
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@start: %true to enable the queues, %false to disable them
  *	@pf: the PF owning the queues
  *	@vf: the VF owning the queues
  *	@iqid: ingress queue id
  *	@fl0id: FL0 queue id or 0xffff if no attached FL0
  *	@fl1id: FL1 queue id or 0xffff if no attached FL1
  *
  *	Starts or stops an ingress queue and its associated FLs, if any.
  */
 int t4_iq_start_stop(struct adapter *adap, unsigned int mbox, bool start,
 		     unsigned int pf, unsigned int vf, unsigned int iqid,
 		     unsigned int fl0id, unsigned int fl1id)
 {
 	struct fw_iq_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
 			    F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(pf) |
 			    V_FW_IQ_CMD_VFN(vf));
 	c.alloc_to_len16 = htonl(V_FW_IQ_CMD_IQSTART(start) |
 				 V_FW_IQ_CMD_IQSTOP(!start) | FW_LEN16(c));
 	c.iqid = htons(iqid);
 	c.fl0id = htons(fl0id);
 	c.fl1id = htons(fl1id);
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
 /**
  *	t4_iq_free - free an ingress queue and its FLs
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@pf: the PF owning the queues
  *	@vf: the VF owning the queues
  *	@iqtype: the ingress queue type (FW_IQ_TYPE_FL_INT_CAP, etc.)
  *	@iqid: ingress queue id
  *	@fl0id: FL0 queue id or 0xffff if no attached FL0
  *	@fl1id: FL1 queue id or 0xffff if no attached FL1
  *
  *	Frees an ingress queue and its associated FLs, if any.
  */
 int t4_iq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
 	       unsigned int vf, unsigned int iqtype, unsigned int iqid,
 	       unsigned int fl0id, unsigned int fl1id)
 {
 	struct fw_iq_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
 			    F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(pf) |
 			    V_FW_IQ_CMD_VFN(vf));
 	c.alloc_to_len16 = htonl(F_FW_IQ_CMD_FREE | FW_LEN16(c));
 	c.type_to_iqandstindex = htonl(V_FW_IQ_CMD_TYPE(iqtype));
 	c.iqid = htons(iqid);
 	c.fl0id = htons(fl0id);
 	c.fl1id = htons(fl1id);
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
 /**
  *	t4_eth_eq_free - free an Ethernet egress queue
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@pf: the PF owning the queue
  *	@vf: the VF owning the queue
  *	@eqid: egress queue id
  *
  *	Frees an Ethernet egress queue.
  */
 int t4_eth_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
 		   unsigned int vf, unsigned int eqid)
 {
 	struct fw_eq_eth_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
 			    F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(pf) |
 			    V_FW_EQ_ETH_CMD_VFN(vf));
 	c.alloc_to_len16 = htonl(F_FW_EQ_ETH_CMD_FREE | FW_LEN16(c));
 	c.eqid_pkd = htonl(V_FW_EQ_ETH_CMD_EQID(eqid));
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
 /**
  *	t4_ctrl_eq_free - free a control egress queue
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@pf: the PF owning the queue
  *	@vf: the VF owning the queue
  *	@eqid: egress queue id
  *
  *	Frees a control egress queue.
  */
 int t4_ctrl_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
 		    unsigned int vf, unsigned int eqid)
 {
 	struct fw_eq_ctrl_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
 			    F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(pf) |
 			    V_FW_EQ_CTRL_CMD_VFN(vf));
 	c.alloc_to_len16 = htonl(F_FW_EQ_CTRL_CMD_FREE | FW_LEN16(c));
 	c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_EQID(eqid));
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
 /**
  *	t4_ofld_eq_free - free an offload egress queue
  *	@adap: the adapter
  *	@mbox: mailbox to use for the FW command
  *	@pf: the PF owning the queue
  *	@vf: the VF owning the queue
  *	@eqid: egress queue id
  *
  *	Frees a control egress queue.
  */
 int t4_ofld_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
 		    unsigned int vf, unsigned int eqid)
 {
 	struct fw_eq_ofld_cmd c;
 
 	memset(&c, 0, sizeof(c));
 	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST |
 			    F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(pf) |
 			    V_FW_EQ_OFLD_CMD_VFN(vf));
 	c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_FREE | FW_LEN16(c));
 	c.eqid_pkd = htonl(V_FW_EQ_OFLD_CMD_EQID(eqid));
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
 /**
  *	t4_handle_fw_rpl - process a FW reply message
  *	@adap: the adapter
  *	@rpl: start of the FW message
  *
  *	Processes a FW message, such as link state change messages.
  */
 int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl)
 {
 	u8 opcode = *(const u8 *)rpl;
 	const struct fw_port_cmd *p = (const void *)rpl;
 	unsigned int action = G_FW_PORT_CMD_ACTION(ntohl(p->action_to_len16));
 
 	if (opcode == FW_PORT_CMD && action == FW_PORT_ACTION_GET_PORT_INFO) {
 		/* link/module state change message */
 		int speed = 0, fc = 0, i;
 		int chan = G_FW_PORT_CMD_PORTID(ntohl(p->op_to_portid));
 		struct port_info *pi = NULL;
 		struct link_config *lc;
 		u32 stat = ntohl(p->u.info.lstatus_to_modtype);
 		int link_ok = (stat & F_FW_PORT_CMD_LSTATUS) != 0;
 		u32 mod = G_FW_PORT_CMD_MODTYPE(stat);
 
 		if (stat & F_FW_PORT_CMD_RXPAUSE)
 			fc |= PAUSE_RX;
 		if (stat & F_FW_PORT_CMD_TXPAUSE)
 			fc |= PAUSE_TX;
 		if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_100M))
 			speed = SPEED_100;
 		else if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_1G))
 			speed = SPEED_1000;
 		else if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_10G))
 			speed = SPEED_10000;
 		else if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_40G))
 			speed = SPEED_40000;
 
 		for_each_port(adap, i) {
 			pi = adap2pinfo(adap, i);
 			if (pi->tx_chan == chan)
 				break;
 		}
 		lc = &pi->link_cfg;
 
 		if (link_ok != lc->link_ok || speed != lc->speed ||
 		    fc != lc->fc) {                    /* something changed */
 			int reason;
 
 			if (!link_ok && lc->link_ok)
 				reason = G_FW_PORT_CMD_LINKDNRC(stat);
 			else
 				reason = -1;
 
 			lc->link_ok = link_ok;
 			lc->speed = speed;
 			lc->fc = fc;
 			lc->supported = ntohs(p->u.info.pcap);
 			t4_os_link_changed(adap, i, link_ok, reason);
 		}
 		if (mod != pi->mod_type) {
 			pi->mod_type = mod;
 			t4_os_portmod_changed(adap, i);
 		}
 	} else {
 		CH_WARN_RATELIMIT(adap,
 		    "Unknown firmware reply 0x%x (0x%x)\n", opcode, action);
 		return -EINVAL;
 	}
 	return 0;
 }
 
 /**
  *	get_pci_mode - determine a card's PCI mode
  *	@adapter: the adapter
  *	@p: where to store the PCI settings
  *
  *	Determines a card's PCI mode and associated parameters, such as speed
  *	and width.
  */
 static void __devinit get_pci_mode(struct adapter *adapter,
 				   struct pci_params *p)
 {
 	u16 val;
 	u32 pcie_cap;
 
 	pcie_cap = t4_os_find_pci_capability(adapter, PCI_CAP_ID_EXP);
 	if (pcie_cap) {
 		t4_os_pci_read_cfg2(adapter, pcie_cap + PCI_EXP_LNKSTA, &val);
 		p->speed = val & PCI_EXP_LNKSTA_CLS;
 		p->width = (val & PCI_EXP_LNKSTA_NLW) >> 4;
 	}
 }
 
 /**
  *	init_link_config - initialize a link's SW state
  *	@lc: structure holding the link state
  *	@caps: link capabilities
  *
  *	Initializes the SW state maintained for each link, including the link's
  *	capabilities and default speed/flow-control/autonegotiation settings.
  */
 static void __devinit init_link_config(struct link_config *lc,
 				       unsigned int caps)
 {
 	lc->supported = caps;
 	lc->requested_speed = 0;
 	lc->speed = 0;
 	lc->requested_fc = lc->fc = PAUSE_RX | PAUSE_TX;
 	if (lc->supported & FW_PORT_CAP_ANEG) {
 		lc->advertising = lc->supported & ADVERT_MASK;
 		lc->autoneg = AUTONEG_ENABLE;
 		lc->requested_fc |= PAUSE_AUTONEG;
 	} else {
 		lc->advertising = 0;
 		lc->autoneg = AUTONEG_DISABLE;
 	}
 }
 
 static int __devinit get_flash_params(struct adapter *adapter)
 {
 	int ret;
 	u32 info = 0;
 
 	ret = sf1_write(adapter, 1, 1, 0, SF_RD_ID);
 	if (!ret)
 		ret = sf1_read(adapter, 3, 0, 1, &info);
 	t4_write_reg(adapter, A_SF_OP, 0);               /* unlock SF */
 	if (ret < 0)
 		return ret;
 
 	if ((info & 0xff) != 0x20)             /* not a Numonix flash */
 		return -EINVAL;
 	info >>= 16;                           /* log2 of size */
 	if (info >= 0x14 && info < 0x18)
 		adapter->params.sf_nsec = 1 << (info - 16);
 	else if (info == 0x18)
 		adapter->params.sf_nsec = 64;
 	else
 		return -EINVAL;
 	adapter->params.sf_size = 1 << info;
 	return 0;
 }
 
 static void __devinit set_pcie_completion_timeout(struct adapter *adapter,
 						  u8 range)
 {
 	u16 val;
 	u32 pcie_cap;
 
 	pcie_cap = t4_os_find_pci_capability(adapter, PCI_CAP_ID_EXP);
 	if (pcie_cap) {
 		t4_os_pci_read_cfg2(adapter, pcie_cap + PCI_EXP_DEVCTL2, &val);
 		val &= 0xfff0;
 		val |= range ;
 		t4_os_pci_write_cfg2(adapter, pcie_cap + PCI_EXP_DEVCTL2, val);
 	}
 }
 
 /**
  *	t4_prep_adapter - prepare SW and HW for operation
  *	@adapter: the adapter
  *	@reset: if true perform a HW reset
  *
  *	Initialize adapter SW state for the various HW modules, set initial
  *	values for some adapter tunables, take PHYs out of reset, and
  *	initialize the MDIO interface.
  */
 int __devinit t4_prep_adapter(struct adapter *adapter)
 {
 	int ret;
 	uint16_t device_id;
 	uint32_t pl_rev;
 
 	get_pci_mode(adapter, &adapter->params.pci);
 
 	pl_rev = t4_read_reg(adapter, A_PL_REV);
 	adapter->params.chipid = G_CHIPID(pl_rev);
 	adapter->params.rev = G_REV(pl_rev);
 	if (adapter->params.chipid == 0) {
 		/* T4 did not have chipid in PL_REV (T5 onwards do) */
 		adapter->params.chipid = CHELSIO_T4;
 
 		/* T4A1 chip is not supported */
 		if (adapter->params.rev == 1) {
 			CH_ALERT(adapter, "T4 rev 1 chip is not supported.\n");
 			return -EINVAL;
 		}
 	}
 	adapter->params.pci.vpd_cap_addr =
 	    t4_os_find_pci_capability(adapter, PCI_CAP_ID_VPD);
 
 	ret = get_flash_params(adapter);
 	if (ret < 0)
 		return ret;
 
 	ret = get_vpd_params(adapter, &adapter->params.vpd);
 	if (ret < 0)
 		return ret;
 
 	/* Cards with real ASICs have the chipid in the PCIe device id */
 	t4_os_pci_read_cfg2(adapter, PCI_DEVICE_ID, &device_id);
 	if (device_id >> 12 == adapter->params.chipid)
 		adapter->params.cim_la_size = CIMLA_SIZE;
 	else {
 		/* FPGA */
 		adapter->params.fpga = 1;
 		adapter->params.cim_la_size = 2 * CIMLA_SIZE;
 	}
 
 	init_cong_ctrl(adapter->params.a_wnd, adapter->params.b_wnd);
 
 	/*
 	 * Default port and clock for debugging in case we can't reach FW.
 	 */
 	adapter->params.nports = 1;
 	adapter->params.portvec = 1;
 	adapter->params.vpd.cclk = 50000;
 
 	/* Set pci completion timeout value to 4 seconds. */
 	set_pcie_completion_timeout(adapter, 0xd);
 	return 0;
 }
 
 /**
  *	t4_init_tp_params - initialize adap->params.tp
  *	@adap: the adapter
  *
  *	Initialize various fields of the adapter's TP Parameters structure.
  */
 int __devinit t4_init_tp_params(struct adapter *adap)
 {
 	int chan;
 	u32 v;
 
 	v = t4_read_reg(adap, A_TP_TIMER_RESOLUTION);
 	adap->params.tp.tre = G_TIMERRESOLUTION(v);
 	adap->params.tp.dack_re = G_DELAYEDACKRESOLUTION(v);
 
 	/* MODQ_REQ_MAP defaults to setting queues 0-3 to chan 0-3 */
 	for (chan = 0; chan < NCHAN; chan++)
 		adap->params.tp.tx_modq[chan] = chan;
 
 	/*
 	 * Cache the adapter's Compressed Filter Mode and global Incress
 	 * Configuration.
 	 */
         t4_read_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA,
                          &adap->params.tp.vlan_pri_map, 1,
                          A_TP_VLAN_PRI_MAP);
 	t4_read_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA,
 			 &adap->params.tp.ingress_config, 1,
 			 A_TP_INGRESS_CONFIG);
 
 	/*
 	 * Now that we have TP_VLAN_PRI_MAP cached, we can calculate the field
 	 * shift positions of several elements of the Compressed Filter Tuple
 	 * for this adapter which we need frequently ...
 	 */
 	adap->params.tp.vlan_shift = t4_filter_field_shift(adap, F_VLAN);
 	adap->params.tp.vnic_shift = t4_filter_field_shift(adap, F_VNIC_ID);
 	adap->params.tp.port_shift = t4_filter_field_shift(adap, F_PORT);
 	adap->params.tp.protocol_shift = t4_filter_field_shift(adap, F_PROTOCOL);
 
 	/*
 	 * If TP_INGRESS_CONFIG.VNID == 0, then TP_VLAN_PRI_MAP.VNIC_ID
 	 * represents the presense of an Outer VLAN instead of a VNIC ID.
 	 */
 	if ((adap->params.tp.ingress_config & F_VNIC) == 0)
 		adap->params.tp.vnic_shift = -1;
 
 	return 0;
 }
 
 /**
  *	t4_filter_field_shift - calculate filter field shift
  *	@adap: the adapter
  *	@filter_sel: the desired field (from TP_VLAN_PRI_MAP bits)
  *
  *	Return the shift position of a filter field within the Compressed
  *	Filter Tuple.  The filter field is specified via its selection bit
  *	within TP_VLAN_PRI_MAL (filter mode).  E.g. F_VLAN.
  */
 int t4_filter_field_shift(const struct adapter *adap, int filter_sel)
 {
 	unsigned int filter_mode = adap->params.tp.vlan_pri_map;
 	unsigned int sel;
 	int field_shift;
 
 	if ((filter_mode & filter_sel) == 0)
 		return -1;
 
 	for (sel = 1, field_shift = 0; sel < filter_sel; sel <<= 1) {
 	    switch (filter_mode & sel) {
 		case F_FCOE:          field_shift += W_FT_FCOE;          break;
 		case F_PORT:          field_shift += W_FT_PORT;          break;
 		case F_VNIC_ID:       field_shift += W_FT_VNIC_ID;       break;
 		case F_VLAN:          field_shift += W_FT_VLAN;          break;
 		case F_TOS:           field_shift += W_FT_TOS;           break;
 		case F_PROTOCOL:      field_shift += W_FT_PROTOCOL;      break;
 		case F_ETHERTYPE:     field_shift += W_FT_ETHERTYPE;     break;
 		case F_MACMATCH:      field_shift += W_FT_MACMATCH;      break;
 		case F_MPSHITTYPE:    field_shift += W_FT_MPSHITTYPE;    break;
 		case F_FRAGMENTATION: field_shift += W_FT_FRAGMENTATION; break;
 	    }
 	}
 	return field_shift;
 }
 
 int __devinit t4_port_init(struct port_info *p, int mbox, int pf, int vf)
 {
 	u8 addr[6];
 	int ret, i, j;
 	struct fw_port_cmd c;
-	unsigned int rss_size;
+	u16 rss_size;
 	adapter_t *adap = p->adapter;
 
 	memset(&c, 0, sizeof(c));
 
 	for (i = 0, j = -1; i <= p->port_id; i++) {
 		do {
 			j++;
 		} while ((adap->params.portvec & (1 << j)) == 0);
 	}
 
 	c.op_to_portid = htonl(V_FW_CMD_OP(FW_PORT_CMD) |
 			       F_FW_CMD_REQUEST | F_FW_CMD_READ |
 			       V_FW_PORT_CMD_PORTID(j));
 	c.action_to_len16 = htonl(
 		V_FW_PORT_CMD_ACTION(FW_PORT_ACTION_GET_PORT_INFO) |
 		FW_LEN16(c));
 	ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
 	if (ret)
 		return ret;
 
 	ret = t4_alloc_vi(adap, mbox, j, pf, vf, 1, addr, &rss_size);
 	if (ret < 0)
 		return ret;
 
 	p->viid = ret;
 	p->tx_chan = j;
 	p->rx_chan_map = get_mps_bg_map(adap, j);
 	p->lport = j;
 	p->rss_size = rss_size;
 	t4_os_set_hw_addr(adap, p->port_id, addr);
 
 	ret = ntohl(c.u.info.lstatus_to_modtype);
 	p->mdio_addr = (ret & F_FW_PORT_CMD_MDIOCAP) ?
 		G_FW_PORT_CMD_MDIOADDR(ret) : -1;
 	p->port_type = G_FW_PORT_CMD_PTYPE(ret);
 	p->mod_type = G_FW_PORT_CMD_MODTYPE(ret);
 
 	init_link_config(&p->link_cfg, ntohs(c.u.info.pcap));
 
 	return 0;
 }
 
-int t4_sched_config(struct adapter *adapter, int type, int minmaxen)
+int t4_sched_config(struct adapter *adapter, int type, int minmaxen,
+    		    int sleep_ok)
 {
 	struct fw_sched_cmd cmd;
 
 	memset(&cmd, 0, sizeof(cmd));
 	cmd.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_SCHED_CMD) |
 				      F_FW_CMD_REQUEST |
 				      F_FW_CMD_WRITE);
 	cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd));
 
 	cmd.u.config.sc = FW_SCHED_SC_CONFIG;
 	cmd.u.config.type = type;
 	cmd.u.config.minmaxen = minmaxen;
 
 	return t4_wr_mbox_meat(adapter,adapter->mbox, &cmd, sizeof(cmd),
-			       NULL, 1);
+			       NULL, sleep_ok);
 }
 
 int t4_sched_params(struct adapter *adapter, int type, int level, int mode,
 		    int rateunit, int ratemode, int channel, int cl,
-		    int minrate, int maxrate, int weight, int pktsize)
+		    int minrate, int maxrate, int weight, int pktsize,
+		    int sleep_ok)
 {
 	struct fw_sched_cmd cmd;
 
 	memset(&cmd, 0, sizeof(cmd));
 	cmd.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_SCHED_CMD) |
 				      F_FW_CMD_REQUEST |
 				      F_FW_CMD_WRITE);
 	cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd));
 
 	cmd.u.params.sc = FW_SCHED_SC_PARAMS;
 	cmd.u.params.type = type;
 	cmd.u.params.level = level;
 	cmd.u.params.mode = mode;
 	cmd.u.params.ch = channel;
 	cmd.u.params.cl = cl;
 	cmd.u.params.unit = rateunit;
 	cmd.u.params.rate = ratemode;
 	cmd.u.params.min = cpu_to_be32(minrate);
 	cmd.u.params.max = cpu_to_be32(maxrate);
 	cmd.u.params.weight = cpu_to_be16(weight);
 	cmd.u.params.pktsize = cpu_to_be16(pktsize);
 
 	return t4_wr_mbox_meat(adapter,adapter->mbox, &cmd, sizeof(cmd),
-			       NULL, 1);
+			       NULL, sleep_ok);
 }
Index: stable/10/sys/dev/cxgbe/offload.h
===================================================================
--- stable/10/sys/dev/cxgbe/offload.h	(revision 270296)
+++ stable/10/sys/dev/cxgbe/offload.h	(revision 270297)
@@ -1,158 +1,158 @@
 /*-
  * Copyright (c) 2010 Chelsio Communications, Inc.
  * All rights reserved.
  * Written by: Navdeep Parhar <np@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $FreeBSD$
  *
  */
 
 #ifndef __T4_OFFLOAD_H__
 #define __T4_OFFLOAD_H__
 
 #define INIT_ULPTX_WRH(w, wrlen, atomic, tid) do { \
 	(w)->wr_hi = htonl(V_FW_WR_OP(FW_ULPTX_WR) | V_FW_WR_ATOMIC(atomic)); \
 	(w)->wr_mid = htonl(V_FW_WR_LEN16(DIV_ROUND_UP(wrlen, 16)) | \
 			       V_FW_WR_FLOWID(tid)); \
 	(w)->wr_lo = cpu_to_be64(0); \
 } while (0)
 
 #define INIT_ULPTX_WR(w, wrlen, atomic, tid) \
     INIT_ULPTX_WRH(&((w)->wr), wrlen, atomic, tid)
 
 #define INIT_TP_WR(w, tid) do { \
 	(w)->wr.wr_hi = htonl(V_FW_WR_OP(FW_TP_WR) | \
                               V_FW_WR_IMMDLEN(sizeof(*w) - sizeof(w->wr))); \
 	(w)->wr.wr_mid = htonl(V_FW_WR_LEN16(DIV_ROUND_UP(sizeof(*w), 16)) | \
                                V_FW_WR_FLOWID(tid)); \
 	(w)->wr.wr_lo = cpu_to_be64(0); \
 } while (0)
 
 #define INIT_TP_WR_MIT_CPL(w, cpl, tid) do { \
 	INIT_TP_WR(w, tid); \
 	OPCODE_TID(w) = htonl(MK_OPCODE_TID(cpl, tid)); \
 } while (0)
 
 TAILQ_HEAD(stid_head, stid_region);
 struct listen_ctx;
 
 struct stid_region {
 	TAILQ_ENTRY(stid_region) link;
 	u_int used;	/* # of stids used by this region */
 	u_int free;	/* # of contiguous stids free right after this region */
 };
 
 /*
  * Max # of ATIDs.  The absolute HW max is 16K but we keep it lower.
  */
 #define MAX_ATIDS 8192U
 
 union aopen_entry {
 	void *data;
 	union aopen_entry *next;
 };
 
 /*
  * Holds the size, base address, free list start, etc of the TID, server TID,
  * and active-open TID tables.  The tables themselves are allocated dynamically.
  */
 struct tid_info {
 	void **tid_tab;
 	u_int ntids;
 	u_int tids_in_use;
 
 	struct mtx stid_lock __aligned(CACHE_LINE_SIZE);
 	struct listen_ctx **stid_tab;
 	u_int nstids;
 	u_int stid_base;
 	u_int stids_in_use;
 	u_int nstids_free_head;	/* # of available stids at the begining */
 	struct stid_head stids;
 
 	struct mtx atid_lock __aligned(CACHE_LINE_SIZE);
 	union aopen_entry *atid_tab;
 	u_int natids;
 	union aopen_entry *afree;
 	u_int atids_in_use;
 
 	struct mtx ftid_lock __aligned(CACHE_LINE_SIZE);
 	struct filter_entry *ftid_tab;
 	u_int nftids;
 	u_int ftid_base;
 	u_int ftids_in_use;
 
 	struct mtx etid_lock __aligned(CACHE_LINE_SIZE);
 	struct etid_entry *etid_tab;
 	u_int netids;
 	u_int etid_base;
 };
 
 struct t4_range {
 	u_int start;
 	u_int size;
 };
 
 struct t4_virt_res {                      /* virtualized HW resources */
 	struct t4_range ddp;
 	struct t4_range iscsi;
 	struct t4_range stag;
 	struct t4_range rq;
 	struct t4_range pbl;
 	struct t4_range qp;
 	struct t4_range cq;
 	struct t4_range ocq;
 	struct t4_range l2t;
 };
 
 #ifdef TCP_OFFLOAD
 enum {
 	ULD_TOM = 1,
 	ULD_IWARP = 2,
 };
 
 struct adapter;
 struct port_info;
 struct uld_info {
 	SLIST_ENTRY(uld_info) link;
 	int refcount;
 	int uld_id;
 	int (*activate)(struct adapter *);
 	int (*deactivate)(struct adapter *);
 };
 
 struct tom_tunables {
 	int sndbuf;
 	int ddp;
 	int indsz;
 	int ddp_thres;
 	int rx_coalesce;
 };
 
 int t4_register_uld(struct uld_info *);
 int t4_unregister_uld(struct uld_info *);
 int t4_activate_uld(struct adapter *, int);
 int t4_deactivate_uld(struct adapter *, int);
+void t4_iscsi_init(struct ifnet *, unsigned int, const unsigned int *);
 #endif
-
 #endif
Index: stable/10/sys/dev/cxgbe/t4_main.c
===================================================================
--- stable/10/sys/dev/cxgbe/t4_main.c	(revision 270296)
+++ stable/10/sys/dev/cxgbe/t4_main.c	(revision 270297)
@@ -1,8186 +1,8372 @@
 /*-
  * Copyright (c) 2011 Chelsio Communications, Inc.
  * All rights reserved.
  * Written by: Navdeep Parhar <np@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
 #include "opt_inet6.h"
 
 #include <sys/param.h>
 #include <sys/conf.h>
 #include <sys/priv.h>
 #include <sys/kernel.h>
 #include <sys/bus.h>
 #include <sys/module.h>
 #include <sys/malloc.h>
 #include <sys/queue.h>
 #include <sys/taskqueue.h>
 #include <sys/pciio.h>
 #include <dev/pci/pcireg.h>
 #include <dev/pci/pcivar.h>
 #include <dev/pci/pci_private.h>
 #include <sys/firmware.h>
 #include <sys/sbuf.h>
 #include <sys/smp.h>
 #include <sys/socket.h>
 #include <sys/sockio.h>
 #include <sys/sysctl.h>
 #include <net/ethernet.h>
 #include <net/if.h>
 #include <net/if_types.h>
 #include <net/if_dl.h>
 #include <net/if_vlan_var.h>
 #if defined(__i386__) || defined(__amd64__)
 #include <vm/vm.h>
 #include <vm/pmap.h>
 #endif
 
 #include "common/common.h"
 #include "common/t4_msg.h"
 #include "common/t4_regs.h"
 #include "common/t4_regs_values.h"
 #include "t4_ioctl.h"
 #include "t4_l2t.h"
 
 /* T4 bus driver interface */
 static int t4_probe(device_t);
 static int t4_attach(device_t);
 static int t4_detach(device_t);
 static device_method_t t4_methods[] = {
 	DEVMETHOD(device_probe,		t4_probe),
 	DEVMETHOD(device_attach,	t4_attach),
 	DEVMETHOD(device_detach,	t4_detach),
 
 	DEVMETHOD_END
 };
 static driver_t t4_driver = {
 	"t4nex",
 	t4_methods,
 	sizeof(struct adapter)
 };
 
 
 /* T4 port (cxgbe) interface */
 static int cxgbe_probe(device_t);
 static int cxgbe_attach(device_t);
 static int cxgbe_detach(device_t);
 static device_method_t cxgbe_methods[] = {
 	DEVMETHOD(device_probe,		cxgbe_probe),
 	DEVMETHOD(device_attach,	cxgbe_attach),
 	DEVMETHOD(device_detach,	cxgbe_detach),
 	{ 0, 0 }
 };
 static driver_t cxgbe_driver = {
 	"cxgbe",
 	cxgbe_methods,
 	sizeof(struct port_info)
 };
 
 static d_ioctl_t t4_ioctl;
 static d_open_t t4_open;
 static d_close_t t4_close;
 
 static struct cdevsw t4_cdevsw = {
        .d_version = D_VERSION,
        .d_flags = 0,
        .d_open = t4_open,
        .d_close = t4_close,
        .d_ioctl = t4_ioctl,
        .d_name = "t4nex",
 };
 
 /* T5 bus driver interface */
 static int t5_probe(device_t);
 static device_method_t t5_methods[] = {
 	DEVMETHOD(device_probe,		t5_probe),
 	DEVMETHOD(device_attach,	t4_attach),
 	DEVMETHOD(device_detach,	t4_detach),
 
 	DEVMETHOD_END
 };
 static driver_t t5_driver = {
 	"t5nex",
 	t5_methods,
 	sizeof(struct adapter)
 };
 
 
 /* T5 port (cxl) interface */
 static driver_t cxl_driver = {
 	"cxl",
 	cxgbe_methods,
 	sizeof(struct port_info)
 };
 
 static struct cdevsw t5_cdevsw = {
        .d_version = D_VERSION,
        .d_flags = 0,
        .d_open = t4_open,
        .d_close = t4_close,
        .d_ioctl = t4_ioctl,
        .d_name = "t5nex",
 };
 
 /* ifnet + media interface */
 static void cxgbe_init(void *);
 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 static void cxgbe_qflush(struct ifnet *);
 static int cxgbe_media_change(struct ifnet *);
 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 
 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 
 /*
  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
  * then ADAPTER_LOCK, then t4_uld_list_lock.
  */
 static struct sx t4_list_lock;
 SLIST_HEAD(, adapter) t4_list;
 #ifdef TCP_OFFLOAD
 static struct sx t4_uld_list_lock;
 SLIST_HEAD(, uld_info) t4_uld_list;
 #endif
 
 /*
  * Tunables.  See tweak_tunables() too.
  *
  * Each tunable is set to a default value here if it's known at compile-time.
  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
  * provide a reasonable default when the driver is loaded.
  *
  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
  * T5 are under hw.cxl.
  */
 
 /*
  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
  */
 #define NTXQ_10G 16
 static int t4_ntxq10g = -1;
 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 
 #define NRXQ_10G 8
 static int t4_nrxq10g = -1;
 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 
 #define NTXQ_1G 4
 static int t4_ntxq1g = -1;
 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 
 #define NRXQ_1G 2
 static int t4_nrxq1g = -1;
 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 
 static int t4_rsrv_noflowq = 0;
 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 
 #ifdef TCP_OFFLOAD
 #define NOFLDTXQ_10G 8
 static int t4_nofldtxq10g = -1;
 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 
 #define NOFLDRXQ_10G 2
 static int t4_nofldrxq10g = -1;
 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 
 #define NOFLDTXQ_1G 2
 static int t4_nofldtxq1g = -1;
 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 
 #define NOFLDRXQ_1G 1
 static int t4_nofldrxq1g = -1;
 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 #endif
 
+#ifdef DEV_NETMAP
+#define NNMTXQ_10G 2
+static int t4_nnmtxq10g = -1;
+TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
+
+#define NNMRXQ_10G 2
+static int t4_nnmrxq10g = -1;
+TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
+
+#define NNMTXQ_1G 1
+static int t4_nnmtxq1g = -1;
+TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
+
+#define NNMRXQ_1G 1
+static int t4_nnmrxq1g = -1;
+TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
+#endif
+
 /*
  * Holdoff parameters for 10G and 1G ports.
  */
 #define TMR_IDX_10G 1
 static int t4_tmr_idx_10g = TMR_IDX_10G;
 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 
 #define PKTC_IDX_10G (-1)
 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 
 #define TMR_IDX_1G 1
 static int t4_tmr_idx_1g = TMR_IDX_1G;
 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 
 #define PKTC_IDX_1G (-1)
 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 
 /*
  * Size (# of entries) of each tx and rx queue.
  */
 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 
 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 
 /*
  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
  */
 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 
 /*
  * Configuration file.
  */
 #define DEFAULT_CF	"default"
 #define FLASH_CF	"flash"
 #define UWIRE_CF	"uwire"
 #define FPGA_CF		"fpga"
 static char t4_cfg_file[32] = DEFAULT_CF;
 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 
 /*
  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
  * encouraged respectively).
  */
 static unsigned int t4_fw_install = 1;
 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 
 /*
  * ASIC features that will be used.  Disable the ones you don't want so that the
  * chip resources aren't wasted on features that will not be used.
  */
 static int t4_linkcaps_allowed = 0;	/* No DCBX, PPP, etc. by default */
 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 
 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 
 static int t4_toecaps_allowed = -1;
 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 
 static int t4_rdmacaps_allowed = 0;
 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 
 static int t4_iscsicaps_allowed = 0;
 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 
 static int t4_fcoecaps_allowed = 0;
 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 
 static int t5_write_combine = 0;
 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 
 struct intrs_and_queues {
-	int intr_type;		/* INTx, MSI, or MSI-X */
-	int nirq;		/* Number of vectors */
-	int intr_flags;
-	int ntxq10g;		/* # of NIC txq's for each 10G port */
-	int nrxq10g;		/* # of NIC rxq's for each 10G port */
-	int ntxq1g;		/* # of NIC txq's for each 1G port */
-	int nrxq1g;		/* # of NIC rxq's for each 1G port */
-	int rsrv_noflowq;	/* Flag whether to reserve queue 0 */
+	uint16_t intr_type;	/* INTx, MSI, or MSI-X */
+	uint16_t nirq;		/* Total # of vectors */
+	uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
+	uint16_t intr_flags_1g;	/* Interrupt flags for each 1G port */
+	uint16_t ntxq10g;	/* # of NIC txq's for each 10G port */
+	uint16_t nrxq10g;	/* # of NIC rxq's for each 10G port */
+	uint16_t ntxq1g;	/* # of NIC txq's for each 1G port */
+	uint16_t nrxq1g;	/* # of NIC rxq's for each 1G port */
+	uint16_t rsrv_noflowq;	/* Flag whether to reserve queue 0 */
 #ifdef TCP_OFFLOAD
-	int nofldtxq10g;	/* # of TOE txq's for each 10G port */
-	int nofldrxq10g;	/* # of TOE rxq's for each 10G port */
-	int nofldtxq1g;		/* # of TOE txq's for each 1G port */
-	int nofldrxq1g;		/* # of TOE rxq's for each 1G port */
+	uint16_t nofldtxq10g;	/* # of TOE txq's for each 10G port */
+	uint16_t nofldrxq10g;	/* # of TOE rxq's for each 10G port */
+	uint16_t nofldtxq1g;	/* # of TOE txq's for each 1G port */
+	uint16_t nofldrxq1g;	/* # of TOE rxq's for each 1G port */
 #endif
+#ifdef DEV_NETMAP
+	uint16_t nnmtxq10g;	/* # of netmap txq's for each 10G port */
+	uint16_t nnmrxq10g;	/* # of netmap rxq's for each 10G port */
+	uint16_t nnmtxq1g;	/* # of netmap txq's for each 1G port */
+	uint16_t nnmrxq1g;	/* # of netmap rxq's for each 1G port */
+#endif
 };
 
 struct filter_entry {
         uint32_t valid:1;	/* filter allocated and valid */
         uint32_t locked:1;	/* filter is administratively locked */
         uint32_t pending:1;	/* filter action is pending firmware reply */
 	uint32_t smtidx:8;	/* Source MAC Table index for smac */
 	struct l2t_entry *l2t;	/* Layer Two Table entry for dmac */
 
         struct t4_filter_specification fs;
 };
 
-enum {
-	XGMAC_MTU	= (1 << 0),
-	XGMAC_PROMISC	= (1 << 1),
-	XGMAC_ALLMULTI	= (1 << 2),
-	XGMAC_VLANEX	= (1 << 3),
-	XGMAC_UCADDR	= (1 << 4),
-	XGMAC_MCADDRS	= (1 << 5),
-
-	XGMAC_ALL	= 0xffff
-};
-
 static int map_bars_0_and_4(struct adapter *);
 static int map_bar_2(struct adapter *);
 static void setup_memwin(struct adapter *);
 static int validate_mem_range(struct adapter *, uint32_t, int);
 static int fwmtype_to_hwmtype(int);
 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
     uint32_t *);
 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 static int cfg_itype_and_nqueues(struct adapter *, int, int,
     struct intrs_and_queues *);
 static int prep_firmware(struct adapter *);
 static int partition_resources(struct adapter *, const struct firmware *,
     const char *);
 static int get_params__pre_init(struct adapter *);
 static int get_params__post_init(struct adapter *);
 static int set_params__post_init(struct adapter *);
 static void t4_set_desc(struct adapter *);
-static void build_medialist(struct port_info *);
-static int update_mac_settings(struct port_info *, int);
+static void build_medialist(struct port_info *, struct ifmedia *);
 static int cxgbe_init_synchronized(struct port_info *);
 static int cxgbe_uninit_synchronized(struct port_info *);
 static int setup_intr_handlers(struct adapter *);
-static int adapter_full_init(struct adapter *);
-static int adapter_full_uninit(struct adapter *);
-static int port_full_init(struct port_info *);
-static int port_full_uninit(struct port_info *);
 static void quiesce_eq(struct adapter *, struct sge_eq *);
 static void quiesce_iq(struct adapter *, struct sge_iq *);
 static void quiesce_fl(struct adapter *, struct sge_fl *);
 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
     driver_intr_t *, void *, char *);
 static int t4_free_irq(struct adapter *, struct irq *);
 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
     unsigned int);
 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 static void cxgbe_tick(void *);
 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
     struct mbuf *);
 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 static int t4_sysctls(struct adapter *);
 static int cxgbe_sysctls(struct port_info *);
 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 #ifdef SBUF_DRAIN
 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 #endif
 static inline void txq_start(struct ifnet *, struct sge_txq *);
 static uint32_t fconf_to_mode(uint32_t);
 static uint32_t mode_to_fconf(uint32_t);
 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 static int get_filter_mode(struct adapter *, uint32_t *);
 static int set_filter_mode(struct adapter *, uint32_t);
 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 static int get_filter(struct adapter *, struct t4_filter *);
 static int set_filter(struct adapter *, struct t4_filter *);
 static int del_filter(struct adapter *, struct t4_filter *);
 static void clear_filter(struct filter_entry *);
 static int set_filter_wr(struct adapter *, int);
 static int del_filter_wr(struct adapter *, int);
 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 static int load_fw(struct adapter *, struct t4_data *);
 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 #ifdef TCP_OFFLOAD
 static int toe_capability(struct port_info *, int);
 #endif
 static int mod_event(module_t, int, void *);
 
 struct {
 	uint16_t device;
 	char *desc;
 } t4_pciids[] = {
 	{0xa000, "Chelsio Terminator 4 FPGA"},
 	{0x4400, "Chelsio T440-dbg"},
 	{0x4401, "Chelsio T420-CR"},
 	{0x4402, "Chelsio T422-CR"},
 	{0x4403, "Chelsio T440-CR"},
 	{0x4404, "Chelsio T420-BCH"},
 	{0x4405, "Chelsio T440-BCH"},
 	{0x4406, "Chelsio T440-CH"},
 	{0x4407, "Chelsio T420-SO"},
 	{0x4408, "Chelsio T420-CX"},
 	{0x4409, "Chelsio T420-BT"},
 	{0x440a, "Chelsio T404-BT"},
 	{0x440e, "Chelsio T440-LP-CR"},
 }, t5_pciids[] = {
 	{0xb000, "Chelsio Terminator 5 FPGA"},
 	{0x5400, "Chelsio T580-dbg"},
 	{0x5401,  "Chelsio T520-CR"},		/* 2 x 10G */
 	{0x5402,  "Chelsio T522-CR"},		/* 2 x 10G, 2 X 1G */
 	{0x5403,  "Chelsio T540-CR"},		/* 4 x 10G */
 	{0x5407,  "Chelsio T520-SO"},		/* 2 x 10G, nomem */
 	{0x5409,  "Chelsio T520-BT"},		/* 2 x 10GBaseT */
 	{0x540a,  "Chelsio T504-BT"},		/* 4 x 1G */
 	{0x540d,  "Chelsio T580-CR"},		/* 2 x 40G */
 	{0x540e,  "Chelsio T540-LP-CR"},	/* 4 x 10G */
 	{0x5410,  "Chelsio T580-LP-CR"},	/* 2 x 40G */
 	{0x5411,  "Chelsio T520-LL-CR"},	/* 2 x 10G */
 	{0x5412,  "Chelsio T560-CR"},		/* 1 x 40G, 2 x 10G */
 	{0x5414,  "Chelsio T580-LP-SO-CR"},	/* 2 x 40G, nomem */
 #ifdef notyet
 	{0x5404,  "Chelsio T520-BCH"},
 	{0x5405,  "Chelsio T540-BCH"},
 	{0x5406,  "Chelsio T540-CH"},
 	{0x5408,  "Chelsio T520-CX"},
 	{0x540b,  "Chelsio B520-SR"},
 	{0x540c,  "Chelsio B504-BT"},
 	{0x540f,  "Chelsio Amsterdam"},
 	{0x5413,  "Chelsio T580-CHR"},
 #endif
 };
 
 #ifdef TCP_OFFLOAD
 /*
  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
  * exactly the same for both rxq and ofld_rxq.
  */
 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 #endif
 
 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 
 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 
 static int
 t4_probe(device_t dev)
 {
 	int i;
 	uint16_t v = pci_get_vendor(dev);
 	uint16_t d = pci_get_device(dev);
 	uint8_t f = pci_get_function(dev);
 
 	if (v != PCI_VENDOR_ID_CHELSIO)
 		return (ENXIO);
 
 	/* Attach only to PF0 of the FPGA */
 	if (d == 0xa000 && f != 0)
 		return (ENXIO);
 
 	for (i = 0; i < nitems(t4_pciids); i++) {
 		if (d == t4_pciids[i].device) {
 			device_set_desc(dev, t4_pciids[i].desc);
 			return (BUS_PROBE_DEFAULT);
 		}
 	}
 
 	return (ENXIO);
 }
 
 static int
 t5_probe(device_t dev)
 {
 	int i;
 	uint16_t v = pci_get_vendor(dev);
 	uint16_t d = pci_get_device(dev);
 	uint8_t f = pci_get_function(dev);
 
 	if (v != PCI_VENDOR_ID_CHELSIO)
 		return (ENXIO);
 
 	/* Attach only to PF0 of the FPGA */
 	if (d == 0xb000 && f != 0)
 		return (ENXIO);
 
 	for (i = 0; i < nitems(t5_pciids); i++) {
 		if (d == t5_pciids[i].device) {
 			device_set_desc(dev, t5_pciids[i].desc);
 			return (BUS_PROBE_DEFAULT);
 		}
 	}
 
 	return (ENXIO);
 }
 
 static int
 t4_attach(device_t dev)
 {
 	struct adapter *sc;
 	int rc = 0, i, n10g, n1g, rqidx, tqidx;
 	struct intrs_and_queues iaq;
 	struct sge *s;
 #ifdef TCP_OFFLOAD
 	int ofld_rqidx, ofld_tqidx;
 #endif
+#ifdef DEV_NETMAP
+	int nm_rqidx, nm_tqidx;
+#endif
 
 	sc = device_get_softc(dev);
 	sc->dev = dev;
 
 	pci_enable_busmaster(dev);
 	if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 		uint32_t v;
 
 		pci_set_max_read_req(dev, 4096);
 		v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 		v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 		pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 	}
 
 	sc->traceq = -1;
 	mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 	snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 	    device_get_nameunit(dev));
 
 	snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 	    device_get_nameunit(dev));
 	mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 	sx_xlock(&t4_list_lock);
 	SLIST_INSERT_HEAD(&t4_list, sc, link);
 	sx_xunlock(&t4_list_lock);
 
 	mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 	TAILQ_INIT(&sc->sfl);
 	callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 
 	rc = map_bars_0_and_4(sc);
 	if (rc != 0)
 		goto done; /* error message displayed already */
 
 	/*
 	 * This is the real PF# to which we're attaching.  Works from within PCI
 	 * passthrough environments too, where pci_get_function() could return a
 	 * different PF# depending on the passthrough configuration.  We need to
 	 * use the real PF# in all our communication with the firmware.
 	 */
 	sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 	sc->mbox = sc->pf;
 
 	memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 	sc->an_handler = an_not_handled;
 	for (i = 0; i < nitems(sc->cpl_handler); i++)
 		sc->cpl_handler[i] = cpl_not_handled;
 	for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 		sc->fw_msg_handler[i] = fw_msg_not_handled;
 	t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 	t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 	t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 	t4_init_sge_cpl_handlers(sc);
 
 	/* Prepare the adapter for operation */
 	rc = -t4_prep_adapter(sc);
 	if (rc != 0) {
 		device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 		goto done;
 	}
 
 	/*
 	 * Do this really early, with the memory windows set up even before the
 	 * character device.  The userland tool's register i/o and mem read
 	 * will work even in "recovery mode".
 	 */
 	setup_memwin(sc);
 	sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 	    device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 	    device_get_nameunit(dev));
 	if (sc->cdev == NULL)
 		device_printf(dev, "failed to create nexus char device.\n");
 	else
 		sc->cdev->si_drv1 = sc;
 
 	/* Go no further if recovery mode has been requested. */
 	if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 		device_printf(dev, "recovery mode.\n");
 		goto done;
 	}
 
 	/* Prepare the firmware for operation */
 	rc = prep_firmware(sc);
 	if (rc != 0)
 		goto done; /* error message displayed already */
 
 	rc = get_params__post_init(sc);
 	if (rc != 0)
 		goto done; /* error message displayed already */
 
 	rc = set_params__post_init(sc);
 	if (rc != 0)
 		goto done; /* error message displayed already */
 
 	rc = map_bar_2(sc);
 	if (rc != 0)
 		goto done; /* error message displayed already */
 
 	rc = t4_create_dma_tag(sc);
 	if (rc != 0)
 		goto done; /* error message displayed already */
 
 	/*
 	 * First pass over all the ports - allocate VIs and initialize some
 	 * basic parameters like mac address, port type, etc.  We also figure
 	 * out whether a port is 10G or 1G and use that information when
 	 * calculating how many interrupts to attempt to allocate.
 	 */
 	n10g = n1g = 0;
 	for_each_port(sc, i) {
 		struct port_info *pi;
 
 		pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 		sc->port[i] = pi;
 
 		/* These must be set before t4_port_init */
 		pi->adapter = sc;
 		pi->port_id = i;
 
 		/* Allocate the vi and initialize parameters like mac addr */
 		rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 		if (rc != 0) {
 			device_printf(dev, "unable to initialize port %d: %d\n",
 			    i, rc);
 			free(pi, M_CXGBE);
 			sc->port[i] = NULL;
 			goto done;
 		}
+		rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
+		if (rc != 0) {
+			device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
+			free(pi, M_CXGBE);
+			sc->port[i] = NULL;
+			goto done;
+		}
 
 		snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 		    device_get_nameunit(dev), i);
 		mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 		sc->chan_map[pi->tx_chan] = i;
 
 		if (is_10G_port(pi) || is_40G_port(pi)) {
 			n10g++;
 			pi->tmr_idx = t4_tmr_idx_10g;
 			pi->pktc_idx = t4_pktc_idx_10g;
 		} else {
 			n1g++;
 			pi->tmr_idx = t4_tmr_idx_1g;
 			pi->pktc_idx = t4_pktc_idx_1g;
 		}
 
 		pi->xact_addr_filt = -1;
 		pi->linkdnrc = -1;
 
 		pi->qsize_rxq = t4_qsize_rxq;
 		pi->qsize_txq = t4_qsize_txq;
 
 		pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 		if (pi->dev == NULL) {
 			device_printf(dev,
 			    "failed to add device for port %d.\n", i);
 			rc = ENXIO;
 			goto done;
 		}
 		device_set_softc(pi->dev, pi);
 	}
 
 	/*
 	 * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 	 */
 	rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 	if (rc != 0)
 		goto done; /* error message displayed already */
 
 	sc->intr_type = iaq.intr_type;
 	sc->intr_count = iaq.nirq;
-	sc->flags |= iaq.intr_flags;
 
 	s = &sc->sge;
 	s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 	s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 	s->neq = s->ntxq + s->nrxq;	/* the free list in an rxq is an eq */
 	s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 	s->niq = s->nrxq + 1;		/* 1 extra for firmware event queue */
-
 #ifdef TCP_OFFLOAD
 	if (is_offload(sc)) {
-
 		s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 		s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 		s->neq += s->nofldtxq + s->nofldrxq;
 		s->niq += s->nofldrxq;
 
 		s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 		    M_CXGBE, M_ZERO | M_WAITOK);
 		s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 		    M_CXGBE, M_ZERO | M_WAITOK);
 	}
 #endif
+#ifdef DEV_NETMAP
+	s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
+	s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
+	s->neq += s->nnmtxq + s->nnmrxq;
+	s->niq += s->nnmrxq;
 
+	s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
+	    M_CXGBE, M_ZERO | M_WAITOK);
+	s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
+	    M_CXGBE, M_ZERO | M_WAITOK);
+#endif
+
 	s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 	    M_ZERO | M_WAITOK);
 	s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 	    M_ZERO | M_WAITOK);
 	s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 	    M_ZERO | M_WAITOK);
 	s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 	    M_ZERO | M_WAITOK);
 	s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 	    M_ZERO | M_WAITOK);
 
 	sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 	    M_ZERO | M_WAITOK);
 
 	t4_init_l2t(sc, M_WAITOK);
 
 	/*
 	 * Second pass over the ports.  This time we know the number of rx and
 	 * tx queues that each port should get.
 	 */
 	rqidx = tqidx = 0;
 #ifdef TCP_OFFLOAD
 	ofld_rqidx = ofld_tqidx = 0;
 #endif
+#ifdef DEV_NETMAP
+	nm_rqidx = nm_tqidx = 0;
+#endif
 	for_each_port(sc, i) {
 		struct port_info *pi = sc->port[i];
 
 		if (pi == NULL)
 			continue;
 
 		pi->first_rxq = rqidx;
 		pi->first_txq = tqidx;
 		if (is_10G_port(pi) || is_40G_port(pi)) {
+			pi->flags |= iaq.intr_flags_10g;
 			pi->nrxq = iaq.nrxq10g;
 			pi->ntxq = iaq.ntxq10g;
 		} else {
+			pi->flags |= iaq.intr_flags_1g;
 			pi->nrxq = iaq.nrxq1g;
 			pi->ntxq = iaq.ntxq1g;
 		}
 
 		if (pi->ntxq > 1)
 			pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 		else
 			pi->rsrv_noflowq = 0;
 
 		rqidx += pi->nrxq;
 		tqidx += pi->ntxq;
-
 #ifdef TCP_OFFLOAD
 		if (is_offload(sc)) {
 			pi->first_ofld_rxq = ofld_rqidx;
 			pi->first_ofld_txq = ofld_tqidx;
 			if (is_10G_port(pi) || is_40G_port(pi)) {
 				pi->nofldrxq = iaq.nofldrxq10g;
 				pi->nofldtxq = iaq.nofldtxq10g;
 			} else {
 				pi->nofldrxq = iaq.nofldrxq1g;
 				pi->nofldtxq = iaq.nofldtxq1g;
 			}
 			ofld_rqidx += pi->nofldrxq;
 			ofld_tqidx += pi->nofldtxq;
 		}
 #endif
+#ifdef DEV_NETMAP
+		pi->first_nm_rxq = nm_rqidx;
+		pi->first_nm_txq = nm_tqidx;
+		if (is_10G_port(pi) || is_40G_port(pi)) {
+			pi->nnmrxq = iaq.nnmrxq10g;
+			pi->nnmtxq = iaq.nnmtxq10g;
+		} else {
+			pi->nnmrxq = iaq.nnmrxq1g;
+			pi->nnmtxq = iaq.nnmtxq1g;
+		}
+		nm_rqidx += pi->nnmrxq;
+		nm_tqidx += pi->nnmtxq;
+#endif
 	}
 
 	rc = setup_intr_handlers(sc);
 	if (rc != 0) {
 		device_printf(dev,
 		    "failed to setup interrupt handlers: %d\n", rc);
 		goto done;
 	}
 
 	rc = bus_generic_attach(dev);
 	if (rc != 0) {
 		device_printf(dev,
 		    "failed to attach all child ports: %d\n", rc);
 		goto done;
 	}
 
 	device_printf(dev,
 	    "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 	    sc->params.pci.width, sc->params.nports, sc->intr_count,
 	    sc->intr_type == INTR_MSIX ? "MSI-X" :
 	    (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 	    sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 
 	t4_set_desc(sc);
 
 done:
 	if (rc != 0 && sc->cdev) {
 		/* cdev was created and so cxgbetool works; recover that way. */
 		device_printf(dev,
 		    "error during attach, adapter is now in recovery mode.\n");
 		rc = 0;
 	}
 
 	if (rc != 0)
 		t4_detach(dev);
 	else
 		t4_sysctls(sc);
 
 	return (rc);
 }
 
 /*
  * Idempotent
  */
 static int
 t4_detach(device_t dev)
 {
 	struct adapter *sc;
 	struct port_info *pi;
 	int i, rc;
 
 	sc = device_get_softc(dev);
 
 	if (sc->flags & FULL_INIT_DONE)
 		t4_intr_disable(sc);
 
 	if (sc->cdev) {
 		destroy_dev(sc->cdev);
 		sc->cdev = NULL;
 	}
 
 	rc = bus_generic_detach(dev);
 	if (rc) {
 		device_printf(dev,
 		    "failed to detach child devices: %d\n", rc);
 		return (rc);
 	}
 
 	for (i = 0; i < sc->intr_count; i++)
 		t4_free_irq(sc, &sc->irq[i]);
 
 	for (i = 0; i < MAX_NPORTS; i++) {
 		pi = sc->port[i];
 		if (pi) {
-			t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid);
+			t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 			if (pi->dev)
 				device_delete_child(dev, pi->dev);
 
 			mtx_destroy(&pi->pi_lock);
 			free(pi, M_CXGBE);
 		}
 	}
 
 	if (sc->flags & FULL_INIT_DONE)
 		adapter_full_uninit(sc);
 
 	if (sc->flags & FW_OK)
 		t4_fw_bye(sc, sc->mbox);
 
 	if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 		pci_release_msi(dev);
 
 	if (sc->regs_res)
 		bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 		    sc->regs_res);
 
 	if (sc->udbs_res)
 		bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 		    sc->udbs_res);
 
 	if (sc->msix_res)
 		bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 		    sc->msix_res);
 
 	if (sc->l2t)
 		t4_free_l2t(sc->l2t);
 
 #ifdef TCP_OFFLOAD
 	free(sc->sge.ofld_rxq, M_CXGBE);
 	free(sc->sge.ofld_txq, M_CXGBE);
 #endif
+#ifdef DEV_NETMAP
+	free(sc->sge.nm_rxq, M_CXGBE);
+	free(sc->sge.nm_txq, M_CXGBE);
+#endif
 	free(sc->irq, M_CXGBE);
 	free(sc->sge.rxq, M_CXGBE);
 	free(sc->sge.txq, M_CXGBE);
 	free(sc->sge.ctrlq, M_CXGBE);
 	free(sc->sge.iqmap, M_CXGBE);
 	free(sc->sge.eqmap, M_CXGBE);
 	free(sc->tids.ftid_tab, M_CXGBE);
 	t4_destroy_dma_tag(sc);
 	if (mtx_initialized(&sc->sc_lock)) {
 		sx_xlock(&t4_list_lock);
 		SLIST_REMOVE(&t4_list, sc, adapter, link);
 		sx_xunlock(&t4_list_lock);
 		mtx_destroy(&sc->sc_lock);
 	}
 
 	if (mtx_initialized(&sc->tids.ftid_lock))
 		mtx_destroy(&sc->tids.ftid_lock);
 	if (mtx_initialized(&sc->sfl_lock))
 		mtx_destroy(&sc->sfl_lock);
 	if (mtx_initialized(&sc->ifp_lock))
 		mtx_destroy(&sc->ifp_lock);
 
 	bzero(sc, sizeof(*sc));
 
 	return (0);
 }
 
-
 static int
 cxgbe_probe(device_t dev)
 {
 	char buf[128];
 	struct port_info *pi = device_get_softc(dev);
 
 	snprintf(buf, sizeof(buf), "port %d", pi->port_id);
 	device_set_desc_copy(dev, buf);
 
 	return (BUS_PROBE_DEFAULT);
 }
 
 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
 #define T4_CAP_ENABLE (T4_CAP)
 
 static int
 cxgbe_attach(device_t dev)
 {
 	struct port_info *pi = device_get_softc(dev);
 	struct ifnet *ifp;
+	char *s;
+	int n, o;
 
 	/* Allocate an ifnet and set it up */
 	ifp = if_alloc(IFT_ETHER);
 	if (ifp == NULL) {
 		device_printf(dev, "Cannot allocate ifnet\n");
 		return (ENOMEM);
 	}
 	pi->ifp = ifp;
 	ifp->if_softc = pi;
 
 	callout_init(&pi->tick, CALLOUT_MPSAFE);
 
 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 
 	ifp->if_init = cxgbe_init;
 	ifp->if_ioctl = cxgbe_ioctl;
 	ifp->if_transmit = cxgbe_transmit;
 	ifp->if_qflush = cxgbe_qflush;
 
 	ifp->if_capabilities = T4_CAP;
 #ifdef TCP_OFFLOAD
 	if (is_offload(pi->adapter))
 		ifp->if_capabilities |= IFCAP_TOE;
 #endif
 	ifp->if_capenable = T4_CAP_ENABLE;
 	ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
 	    CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
 
 	/* Initialize ifmedia for this port */
 	ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
 	    cxgbe_media_status);
-	build_medialist(pi);
+	build_medialist(pi, &pi->media);
 
 	pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
 	    EVENTHANDLER_PRI_ANY);
 
 	ether_ifattach(ifp, pi->hw_addr);
 
+	n = 128;
+	s = malloc(n, M_CXGBE, M_WAITOK);
+	o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
+	MPASS(n > o);
 #ifdef TCP_OFFLOAD
 	if (is_offload(pi->adapter)) {
-		device_printf(dev,
-		    "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n",
-		    pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq);
-	} else
+		o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
+		    pi->nofldtxq, pi->nofldrxq);
+		MPASS(n > o);
+	}
 #endif
-		device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq);
+#ifdef DEV_NETMAP
+	o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
+	    pi->nnmrxq);
+	MPASS(n > o);
+#endif
+	device_printf(dev, "%s\n", s);
+	free(s, M_CXGBE);
 
+#ifdef DEV_NETMAP
+	/* nm_media handled here to keep implementation private to this file */
+	ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
+	    cxgbe_media_status);
+	build_medialist(pi, &pi->nm_media);
+	create_netmap_ifnet(pi);	/* logs errors it something fails */
+#endif
 	cxgbe_sysctls(pi);
 
 	return (0);
 }
 
 static int
 cxgbe_detach(device_t dev)
 {
 	struct port_info *pi = device_get_softc(dev);
 	struct adapter *sc = pi->adapter;
 	struct ifnet *ifp = pi->ifp;
 
 	/* Tell if_ioctl and if_init that the port is going away */
 	ADAPTER_LOCK(sc);
 	SET_DOOMED(pi);
 	wakeup(&sc->flags);
 	while (IS_BUSY(sc))
 		mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
 	SET_BUSY(sc);
 #ifdef INVARIANTS
 	sc->last_op = "t4detach";
 	sc->last_op_thr = curthread;
 #endif
 	ADAPTER_UNLOCK(sc);
 
 	if (pi->flags & HAS_TRACEQ) {
 		sc->traceq = -1;	/* cloner should not create ifnet */
 		t4_tracer_port_detach(sc);
 	}
 
 	if (pi->vlan_c)
 		EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
 
 	PORT_LOCK(pi);
 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 	callout_stop(&pi->tick);
 	PORT_UNLOCK(pi);
 	callout_drain(&pi->tick);
 
 	/* Let detach proceed even if these fail. */
 	cxgbe_uninit_synchronized(pi);
 	port_full_uninit(pi);
 
 	ifmedia_removeall(&pi->media);
 	ether_ifdetach(pi->ifp);
 	if_free(pi->ifp);
 
+#ifdef DEV_NETMAP
+	/* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
+	destroy_netmap_ifnet(pi);
+#endif
+
 	ADAPTER_LOCK(sc);
 	CLR_BUSY(sc);
 	wakeup(&sc->flags);
 	ADAPTER_UNLOCK(sc);
 
 	return (0);
 }
 
 static void
 cxgbe_init(void *arg)
 {
 	struct port_info *pi = arg;
 	struct adapter *sc = pi->adapter;
 
 	if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
 		return;
 	cxgbe_init_synchronized(pi);
 	end_synchronized_op(sc, 0);
 }
 
 static int
 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
 {
-	int rc = 0, mtu, flags;
+	int rc = 0, mtu, flags, can_sleep;
 	struct port_info *pi = ifp->if_softc;
 	struct adapter *sc = pi->adapter;
 	struct ifreq *ifr = (struct ifreq *)data;
 	uint32_t mask;
 
 	switch (cmd) {
 	case SIOCSIFMTU:
 		mtu = ifr->ifr_mtu;
 		if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
 			return (EINVAL);
 
 		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
 		if (rc)
 			return (rc);
 		ifp->if_mtu = mtu;
 		if (pi->flags & PORT_INIT_DONE) {
 			t4_update_fl_bufsize(ifp);
 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
-				rc = update_mac_settings(pi, XGMAC_MTU);
+				rc = update_mac_settings(ifp, XGMAC_MTU);
 		}
 		end_synchronized_op(sc, 0);
 		break;
 
 	case SIOCSIFFLAGS:
-		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg");
+		can_sleep = 0;
+redo_sifflags:
+		rc = begin_synchronized_op(sc, pi,
+		    can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
 		if (rc)
 			return (rc);
 
 		if (ifp->if_flags & IFF_UP) {
 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 				flags = pi->if_flags;
 				if ((ifp->if_flags ^ flags) &
 				    (IFF_PROMISC | IFF_ALLMULTI)) {
-					rc = update_mac_settings(pi,
+					if (can_sleep == 1) {
+						end_synchronized_op(sc, 0);
+						can_sleep = 0;
+						goto redo_sifflags;
+					}
+					rc = update_mac_settings(ifp,
 					    XGMAC_PROMISC | XGMAC_ALLMULTI);
 				}
-			} else
+			} else {
+				if (can_sleep == 0) {
+					end_synchronized_op(sc, LOCK_HELD);
+					can_sleep = 1;
+					goto redo_sifflags;
+				}
 				rc = cxgbe_init_synchronized(pi);
+			}
 			pi->if_flags = ifp->if_flags;
-		} else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+		} else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+			if (can_sleep == 0) {
+				end_synchronized_op(sc, LOCK_HELD);
+				can_sleep = 1;
+				goto redo_sifflags;
+			}
 			rc = cxgbe_uninit_synchronized(pi);
-		end_synchronized_op(sc, 0);
+		}
+		end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
 		break;
 
-	case SIOCADDMULTI:	
+	case SIOCADDMULTI:
 	case SIOCDELMULTI: /* these two are called with a mutex held :-( */
 		rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
 		if (rc)
 			return (rc);
 		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
-			rc = update_mac_settings(pi, XGMAC_MCADDRS);
+			rc = update_mac_settings(ifp, XGMAC_MCADDRS);
 		end_synchronized_op(sc, LOCK_HELD);
 		break;
 
 	case SIOCSIFCAP:
 		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
 		if (rc)
 			return (rc);
 
 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 		if (mask & IFCAP_TXCSUM) {
 			ifp->if_capenable ^= IFCAP_TXCSUM;
 			ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
 
 			if (IFCAP_TSO4 & ifp->if_capenable &&
 			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
 				ifp->if_capenable &= ~IFCAP_TSO4;
 				if_printf(ifp,
 				    "tso4 disabled due to -txcsum.\n");
 			}
 		}
 		if (mask & IFCAP_TXCSUM_IPV6) {
 			ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
 			ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
 
 			if (IFCAP_TSO6 & ifp->if_capenable &&
 			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
 				ifp->if_capenable &= ~IFCAP_TSO6;
 				if_printf(ifp,
 				    "tso6 disabled due to -txcsum6.\n");
 			}
 		}
 		if (mask & IFCAP_RXCSUM)
 			ifp->if_capenable ^= IFCAP_RXCSUM;
 		if (mask & IFCAP_RXCSUM_IPV6)
 			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
 
 		/*
 		 * Note that we leave CSUM_TSO alone (it is always set).  The
 		 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
 		 * sending a TSO request our way, so it's sufficient to toggle
 		 * IFCAP_TSOx only.
 		 */
 		if (mask & IFCAP_TSO4) {
 			if (!(IFCAP_TSO4 & ifp->if_capenable) &&
 			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
 				if_printf(ifp, "enable txcsum first.\n");
 				rc = EAGAIN;
 				goto fail;
 			}
 			ifp->if_capenable ^= IFCAP_TSO4;
 		}
 		if (mask & IFCAP_TSO6) {
 			if (!(IFCAP_TSO6 & ifp->if_capenable) &&
 			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
 				if_printf(ifp, "enable txcsum6 first.\n");
 				rc = EAGAIN;
 				goto fail;
 			}
 			ifp->if_capenable ^= IFCAP_TSO6;
 		}
 		if (mask & IFCAP_LRO) {
 #if defined(INET) || defined(INET6)
 			int i;
 			struct sge_rxq *rxq;
 
 			ifp->if_capenable ^= IFCAP_LRO;
 			for_each_rxq(pi, i, rxq) {
 				if (ifp->if_capenable & IFCAP_LRO)
 					rxq->iq.flags |= IQ_LRO_ENABLED;
 				else
 					rxq->iq.flags &= ~IQ_LRO_ENABLED;
 			}
 #endif
 		}
 #ifdef TCP_OFFLOAD
 		if (mask & IFCAP_TOE) {
 			int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
 
 			rc = toe_capability(pi, enable);
 			if (rc != 0)
 				goto fail;
 
 			ifp->if_capenable ^= mask;
 		}
 #endif
 		if (mask & IFCAP_VLAN_HWTAGGING) {
 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
-				rc = update_mac_settings(pi, XGMAC_VLANEX);
+				rc = update_mac_settings(ifp, XGMAC_VLANEX);
 		}
 		if (mask & IFCAP_VLAN_MTU) {
 			ifp->if_capenable ^= IFCAP_VLAN_MTU;
 
 			/* Need to find out how to disable auto-mtu-inflation */
 		}
 		if (mask & IFCAP_VLAN_HWTSO)
 			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
 		if (mask & IFCAP_VLAN_HWCSUM)
 			ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
 
 #ifdef VLAN_CAPABILITIES
 		VLAN_CAPABILITIES(ifp);
 #endif
 fail:
 		end_synchronized_op(sc, 0);
 		break;
 
 	case SIOCSIFMEDIA:
 	case SIOCGIFMEDIA:
 		ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
 		break;
 
 	default:
 		rc = ether_ioctl(ifp, cmd, data);
 	}
 
 	return (rc);
 }
 
 static int
 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
 {
 	struct port_info *pi = ifp->if_softc;
 	struct adapter *sc = pi->adapter;
 	struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
 	struct buf_ring *br;
 	int rc;
 
 	M_ASSERTPKTHDR(m);
 
 	if (__predict_false(pi->link_cfg.link_ok == 0)) {
 		m_freem(m);
 		return (ENETDOWN);
 	}
 
 	if (m->m_flags & M_FLOWID)
 		txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
 		    + pi->rsrv_noflowq);
 	br = txq->br;
 
 	if (TXQ_TRYLOCK(txq) == 0) {
 		struct sge_eq *eq = &txq->eq;
 
 		/*
 		 * It is possible that t4_eth_tx finishes up and releases the
 		 * lock between the TRYLOCK above and the drbr_enqueue here.  We
 		 * need to make sure that this mbuf doesn't just sit there in
 		 * the drbr.
 		 */
 
 		rc = drbr_enqueue(ifp, br, m);
 		if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
 		    !(eq->flags & EQ_DOOMED))
 			callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
 		return (rc);
 	}
 
 	/*
 	 * txq->m is the mbuf that is held up due to a temporary shortage of
 	 * resources and it should be put on the wire first.  Then what's in
 	 * drbr and finally the mbuf that was just passed in to us.
 	 *
 	 * Return code should indicate the fate of the mbuf that was passed in
 	 * this time.
 	 */
 
 	TXQ_LOCK_ASSERT_OWNED(txq);
 	if (drbr_needs_enqueue(ifp, br) || txq->m) {
 
 		/* Queued for transmission. */
 
 		rc = drbr_enqueue(ifp, br, m);
 		m = txq->m ? txq->m : drbr_dequeue(ifp, br);
 		(void) t4_eth_tx(ifp, txq, m);
 		TXQ_UNLOCK(txq);
 		return (rc);
 	}
 
 	/* Direct transmission. */
 	rc = t4_eth_tx(ifp, txq, m);
 	if (rc != 0 && txq->m)
 		rc = 0;	/* held, will be transmitted soon (hopefully) */
 
 	TXQ_UNLOCK(txq);
 	return (rc);
 }
 
 static void
 cxgbe_qflush(struct ifnet *ifp)
 {
 	struct port_info *pi = ifp->if_softc;
 	struct sge_txq *txq;
 	int i;
 	struct mbuf *m;
 
 	/* queues do not exist if !PORT_INIT_DONE. */
 	if (pi->flags & PORT_INIT_DONE) {
 		for_each_txq(pi, i, txq) {
 			TXQ_LOCK(txq);
 			m_freem(txq->m);
 			txq->m = NULL;
 			while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
 				m_freem(m);
 			TXQ_UNLOCK(txq);
 		}
 	}
 	if_qflush(ifp);
 }
 
 static int
 cxgbe_media_change(struct ifnet *ifp)
 {
 	struct port_info *pi = ifp->if_softc;
 
 	device_printf(pi->dev, "%s unimplemented.\n", __func__);
 
 	return (EOPNOTSUPP);
 }
 
 static void
 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 {
 	struct port_info *pi = ifp->if_softc;
-	struct ifmedia_entry *cur = pi->media.ifm_cur;
+	struct ifmedia *media = NULL;
+	struct ifmedia_entry *cur;
 	int speed = pi->link_cfg.speed;
 	int data = (pi->port_type << 8) | pi->mod_type;
 
+	if (ifp == pi->ifp)
+		media = &pi->media;
+#ifdef DEV_NETMAP
+	else if (ifp == pi->nm_ifp)
+		media = &pi->nm_media;
+#endif
+	MPASS(media != NULL);
+
+	cur = media->ifm_cur;
 	if (cur->ifm_data != data) {
-		build_medialist(pi);
-		cur = pi->media.ifm_cur;
+		build_medialist(pi, media);
+		cur = media->ifm_cur;
 	}
 
 	ifmr->ifm_status = IFM_AVALID;
 	if (!pi->link_cfg.link_ok)
 		return;
 
 	ifmr->ifm_status |= IFM_ACTIVE;
 
 	/* active and current will differ iff current media is autoselect. */
 	if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
 		return;
 
 	ifmr->ifm_active = IFM_ETHER | IFM_FDX;
 	if (speed == SPEED_10000)
 		ifmr->ifm_active |= IFM_10G_T;
 	else if (speed == SPEED_1000)
 		ifmr->ifm_active |= IFM_1000_T;
 	else if (speed == SPEED_100)
 		ifmr->ifm_active |= IFM_100_TX;
 	else if (speed == SPEED_10)
 		ifmr->ifm_active |= IFM_10_T;
 	else
 		KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
 			    speed));
 }
 
 void
 t4_fatal_err(struct adapter *sc)
 {
 	t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
 	t4_intr_disable(sc);
 	log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
 	    device_get_nameunit(sc->dev));
 }
 
 static int
 map_bars_0_and_4(struct adapter *sc)
 {
 	sc->regs_rid = PCIR_BAR(0);
 	sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
 	    &sc->regs_rid, RF_ACTIVE);
 	if (sc->regs_res == NULL) {
 		device_printf(sc->dev, "cannot map registers.\n");
 		return (ENXIO);
 	}
 	sc->bt = rman_get_bustag(sc->regs_res);
 	sc->bh = rman_get_bushandle(sc->regs_res);
 	sc->mmio_len = rman_get_size(sc->regs_res);
 	setbit(&sc->doorbells, DOORBELL_KDB);
 
 	sc->msix_rid = PCIR_BAR(4);
 	sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
 	    &sc->msix_rid, RF_ACTIVE);
 	if (sc->msix_res == NULL) {
 		device_printf(sc->dev, "cannot map MSI-X BAR.\n");
 		return (ENXIO);
 	}
 
 	return (0);
 }
 
 static int
 map_bar_2(struct adapter *sc)
 {
 
 	/*
 	 * T4: only iWARP driver uses the userspace doorbells.  There is no need
 	 * to map it if RDMA is disabled.
 	 */
 	if (is_t4(sc) && sc->rdmacaps == 0)
 		return (0);
 
 	sc->udbs_rid = PCIR_BAR(2);
 	sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
 	    &sc->udbs_rid, RF_ACTIVE);
 	if (sc->udbs_res == NULL) {
 		device_printf(sc->dev, "cannot map doorbell BAR.\n");
 		return (ENXIO);
 	}
 	sc->udbs_base = rman_get_virtual(sc->udbs_res);
 
 	if (is_t5(sc)) {
 		setbit(&sc->doorbells, DOORBELL_UDB);
 #if defined(__i386__) || defined(__amd64__)
 		if (t5_write_combine) {
 			int rc;
 
 			/*
 			 * Enable write combining on BAR2.  This is the
 			 * userspace doorbell BAR and is split into 128B
 			 * (UDBS_SEG_SIZE) doorbell regions, each associated
 			 * with an egress queue.  The first 64B has the doorbell
 			 * and the second 64B can be used to submit a tx work
 			 * request with an implicit doorbell.
 			 */
 
 			rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
 			    rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
 			if (rc == 0) {
 				clrbit(&sc->doorbells, DOORBELL_UDB);
 				setbit(&sc->doorbells, DOORBELL_WCWR);
 				setbit(&sc->doorbells, DOORBELL_UDBWC);
 			} else {
 				device_printf(sc->dev,
 				    "couldn't enable write combining: %d\n",
 				    rc);
 			}
 
 			t4_write_reg(sc, A_SGE_STAT_CFG,
 			    V_STATSOURCE_T5(7) | V_STATMODE(0));
 		}
 #endif
 	}
 
 	return (0);
 }
 
 static const struct memwin t4_memwin[] = {
 	{ MEMWIN0_BASE, MEMWIN0_APERTURE },
 	{ MEMWIN1_BASE, MEMWIN1_APERTURE },
 	{ MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
 };
 
 static const struct memwin t5_memwin[] = {
 	{ MEMWIN0_BASE, MEMWIN0_APERTURE },
 	{ MEMWIN1_BASE, MEMWIN1_APERTURE },
 	{ MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
 };
 
 static void
 setup_memwin(struct adapter *sc)
 {
 	const struct memwin *mw;
 	int i, n;
 	uint32_t bar0;
 
 	if (is_t4(sc)) {
 		/*
 		 * Read low 32b of bar0 indirectly via the hardware backdoor
 		 * mechanism.  Works from within PCI passthrough environments
 		 * too, where rman_get_start() can return a different value.  We
 		 * need to program the T4 memory window decoders with the actual
 		 * addresses that will be coming across the PCIe link.
 		 */
 		bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
 		bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
 
 		mw = &t4_memwin[0];
 		n = nitems(t4_memwin);
 	} else {
 		/* T5 uses the relative offset inside the PCIe BAR */
 		bar0 = 0;
 
 		mw = &t5_memwin[0];
 		n = nitems(t5_memwin);
 	}
 
 	for (i = 0; i < n; i++, mw++) {
 		t4_write_reg(sc,
 		    PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
 		    (mw->base + bar0) | V_BIR(0) |
 		    V_WINDOW(ilog2(mw->aperture) - 10));
 	}
 
 	/* flush */
 	t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
 }
 
 /*
  * Verify that the memory range specified by the addr/len pair is valid and lies
  * entirely within a single region (EDCx or MCx).
  */
 static int
 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
 {
 	uint32_t em, addr_len, maddr, mlen;
 
 	/* Memory can only be accessed in naturally aligned 4 byte units */
 	if (addr & 3 || len & 3 || len == 0)
 		return (EINVAL);
 
 	/* Enabled memories */
 	em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
 	if (em & F_EDRAM0_ENABLE) {
 		addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
 		maddr = G_EDRAM0_BASE(addr_len) << 20;
 		mlen = G_EDRAM0_SIZE(addr_len) << 20;
 		if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
 		    addr + len <= maddr + mlen)
 			return (0);
 	}
 	if (em & F_EDRAM1_ENABLE) {
 		addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
 		maddr = G_EDRAM1_BASE(addr_len) << 20;
 		mlen = G_EDRAM1_SIZE(addr_len) << 20;
 		if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
 		    addr + len <= maddr + mlen)
 			return (0);
 	}
 	if (em & F_EXT_MEM_ENABLE) {
 		addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
 		maddr = G_EXT_MEM_BASE(addr_len) << 20;
 		mlen = G_EXT_MEM_SIZE(addr_len) << 20;
 		if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
 		    addr + len <= maddr + mlen)
 			return (0);
 	}
 	if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
 		addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
 		maddr = G_EXT_MEM1_BASE(addr_len) << 20;
 		mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
 		if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
 		    addr + len <= maddr + mlen)
 			return (0);
 	}
 
 	return (EFAULT);
 }
 
 static int
 fwmtype_to_hwmtype(int mtype)
 {
 
 	switch (mtype) {
 	case FW_MEMTYPE_EDC0:
 		return (MEM_EDC0);
 	case FW_MEMTYPE_EDC1:
 		return (MEM_EDC1);
 	case FW_MEMTYPE_EXTMEM:
 		return (MEM_MC0);
 	case FW_MEMTYPE_EXTMEM1:
 		return (MEM_MC1);
 	default:
 		panic("%s: cannot translate fw mtype %d.", __func__, mtype);
 	}
 }
 
 /*
  * Verify that the memory range specified by the memtype/offset/len pair is
  * valid and lies entirely within the memtype specified.  The global address of
  * the start of the range is returned in addr.
  */
 static int
 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
     uint32_t *addr)
 {
 	uint32_t em, addr_len, maddr, mlen;
 
 	/* Memory can only be accessed in naturally aligned 4 byte units */
 	if (off & 3 || len & 3 || len == 0)
 		return (EINVAL);
 
 	em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
 	switch (fwmtype_to_hwmtype(mtype)) {
 	case MEM_EDC0:
 		if (!(em & F_EDRAM0_ENABLE))
 			return (EINVAL);
 		addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
 		maddr = G_EDRAM0_BASE(addr_len) << 20;
 		mlen = G_EDRAM0_SIZE(addr_len) << 20;
 		break;
 	case MEM_EDC1:
 		if (!(em & F_EDRAM1_ENABLE))
 			return (EINVAL);
 		addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
 		maddr = G_EDRAM1_BASE(addr_len) << 20;
 		mlen = G_EDRAM1_SIZE(addr_len) << 20;
 		break;
 	case MEM_MC:
 		if (!(em & F_EXT_MEM_ENABLE))
 			return (EINVAL);
 		addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
 		maddr = G_EXT_MEM_BASE(addr_len) << 20;
 		mlen = G_EXT_MEM_SIZE(addr_len) << 20;
 		break;
 	case MEM_MC1:
 		if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
 			return (EINVAL);
 		addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
 		maddr = G_EXT_MEM1_BASE(addr_len) << 20;
 		mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
 		break;
 	default:
 		return (EINVAL);
 	}
 
 	if (mlen > 0 && off < mlen && off + len <= mlen) {
 		*addr = maddr + off;	/* global address */
 		return (0);
 	}
 
 	return (EFAULT);
 }
 
 static void
 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
 {
 	const struct memwin *mw;
 
 	if (is_t4(sc)) {
 		KASSERT(win >= 0 && win < nitems(t4_memwin),
 		    ("%s: incorrect memwin# (%d)", __func__, win));
 		mw = &t4_memwin[win];
 	} else {
 		KASSERT(win >= 0 && win < nitems(t5_memwin),
 		    ("%s: incorrect memwin# (%d)", __func__, win));
 		mw = &t5_memwin[win];
 	}
 
 	if (base != NULL)
 		*base = mw->base;
 	if (aperture != NULL)
 		*aperture = mw->aperture;
 }
 
 /*
  * Positions the memory window such that it can be used to access the specified
  * address in the chip's address space.  The return value is the offset of addr
  * from the start of the window.
  */
 static uint32_t
 position_memwin(struct adapter *sc, int n, uint32_t addr)
 {
 	uint32_t start, pf;
 	uint32_t reg;
 
 	KASSERT(n >= 0 && n <= 3,
 	    ("%s: invalid window %d.", __func__, n));
 	KASSERT((addr & 3) == 0,
 	    ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
 
 	if (is_t4(sc)) {
 		pf = 0;
 		start = addr & ~0xf;	/* start must be 16B aligned */
 	} else {
 		pf = V_PFNUM(sc->pf);
 		start = addr & ~0x7f;	/* start must be 128B aligned */
 	}
 	reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
 
 	t4_write_reg(sc, reg, start | pf);
 	t4_read_reg(sc, reg);
 
 	return (addr - start);
 }
 
 static int
 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
     struct intrs_and_queues *iaq)
 {
 	int rc, itype, navail, nrxq10g, nrxq1g, n;
 	int nofldrxq10g = 0, nofldrxq1g = 0;
+	int nnmrxq10g = 0, nnmrxq1g = 0;
 
 	bzero(iaq, sizeof(*iaq));
 
 	iaq->ntxq10g = t4_ntxq10g;
 	iaq->ntxq1g = t4_ntxq1g;
 	iaq->nrxq10g = nrxq10g = t4_nrxq10g;
 	iaq->nrxq1g = nrxq1g = t4_nrxq1g;
 	iaq->rsrv_noflowq = t4_rsrv_noflowq;
 #ifdef TCP_OFFLOAD
 	if (is_offload(sc)) {
 		iaq->nofldtxq10g = t4_nofldtxq10g;
 		iaq->nofldtxq1g = t4_nofldtxq1g;
 		iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
 		iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
 	}
 #endif
+#ifdef DEV_NETMAP
+	iaq->nnmtxq10g = t4_nnmtxq10g;
+	iaq->nnmtxq1g = t4_nnmtxq1g;
+	iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
+	iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
+#endif
 
 	for (itype = INTR_MSIX; itype; itype >>= 1) {
 
 		if ((itype & t4_intr_types) == 0)
 			continue;	/* not allowed */
 
 		if (itype == INTR_MSIX)
 			navail = pci_msix_count(sc->dev);
 		else if (itype == INTR_MSI)
 			navail = pci_msi_count(sc->dev);
 		else
 			navail = 1;
 restart:
 		if (navail == 0)
 			continue;
 
 		iaq->intr_type = itype;
-		iaq->intr_flags = 0;
+		iaq->intr_flags_10g = 0;
+		iaq->intr_flags_1g = 0;
 
 		/*
 		 * Best option: an interrupt vector for errors, one for the
-		 * firmware event queue, and one each for each rxq (NIC as well
-		 * as offload).
+		 * firmware event queue, and one for every rxq (NIC, TOE, and
+		 * netmap).
 		 */
 		iaq->nirq = T4_EXTRA_INTR;
-		iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
-		iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
+		iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
+		iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
 		if (iaq->nirq <= navail &&
 		    (itype != INTR_MSI || powerof2(iaq->nirq))) {
-			iaq->intr_flags |= INTR_DIRECT;
+			iaq->intr_flags_10g = INTR_ALL;
+			iaq->intr_flags_1g = INTR_ALL;
 			goto allocate;
 		}
 
 		/*
-		 * Second best option: an interrupt vector for errors, one for
-		 * the firmware event queue, and one each for either NIC or
-		 * offload rxq's.
+		 * Second best option: a vector for errors, one for the firmware
+		 * event queue, and vectors for either all the NIC rx queues or
+		 * all the TOE rx queues.  The queues that don't get vectors
+		 * will forward their interrupts to those that do.
+		 *
+		 * Note: netmap rx queues cannot be created early and so they
+		 * can't be setup to receive forwarded interrupts for others.
 		 */
 		iaq->nirq = T4_EXTRA_INTR;
-		iaq->nirq += n10g * max(nrxq10g, nofldrxq10g);
-		iaq->nirq += n1g * max(nrxq1g, nofldrxq1g);
+		if (nrxq10g >= nofldrxq10g) {
+			iaq->intr_flags_10g = INTR_RXQ;
+			iaq->nirq += n10g * nrxq10g;
+#ifdef DEV_NETMAP
+			iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
+#endif
+		} else {
+			iaq->intr_flags_10g = INTR_OFLD_RXQ;
+			iaq->nirq += n10g * nofldrxq10g;
+#ifdef DEV_NETMAP
+			iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
+#endif
+		}
+		if (nrxq1g >= nofldrxq1g) {
+			iaq->intr_flags_1g = INTR_RXQ;
+			iaq->nirq += n1g * nrxq1g;
+#ifdef DEV_NETMAP
+			iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
+#endif
+		} else {
+			iaq->intr_flags_1g = INTR_OFLD_RXQ;
+			iaq->nirq += n1g * nofldrxq1g;
+#ifdef DEV_NETMAP
+			iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
+#endif
+		}
 		if (iaq->nirq <= navail &&
 		    (itype != INTR_MSI || powerof2(iaq->nirq)))
 			goto allocate;
 
 		/*
 		 * Next best option: an interrupt vector for errors, one for the
 		 * firmware event queue, and at least one per port.  At this
-		 * point we know we'll have to downsize nrxq or nofldrxq to fit
-		 * what's available to us.
+		 * point we know we'll have to downsize nrxq and/or nofldrxq
+		 * and/or nnmrxq to fit what's available to us.
 		 */
 		iaq->nirq = T4_EXTRA_INTR;
 		iaq->nirq += n10g + n1g;
 		if (iaq->nirq <= navail) {
 			int leftover = navail - iaq->nirq;
 
 			if (n10g > 0) {
 				int target = max(nrxq10g, nofldrxq10g);
 
+				iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
+				    INTR_RXQ : INTR_OFLD_RXQ;
+
 				n = 1;
 				while (n < target && leftover >= n10g) {
 					leftover -= n10g;
 					iaq->nirq += n10g;
 					n++;
 				}
 				iaq->nrxq10g = min(n, nrxq10g);
 #ifdef TCP_OFFLOAD
-				if (is_offload(sc))
-					iaq->nofldrxq10g = min(n, nofldrxq10g);
+				iaq->nofldrxq10g = min(n, nofldrxq10g);
 #endif
+#ifdef DEV_NETMAP
+				iaq->nnmrxq10g = min(n, nnmrxq10g);
+#endif
 			}
 
 			if (n1g > 0) {
 				int target = max(nrxq1g, nofldrxq1g);
 
+				iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
+				    INTR_RXQ : INTR_OFLD_RXQ;
+
 				n = 1;
 				while (n < target && leftover >= n1g) {
 					leftover -= n1g;
 					iaq->nirq += n1g;
 					n++;
 				}
 				iaq->nrxq1g = min(n, nrxq1g);
 #ifdef TCP_OFFLOAD
-				if (is_offload(sc))
-					iaq->nofldrxq1g = min(n, nofldrxq1g);
+				iaq->nofldrxq1g = min(n, nofldrxq1g);
 #endif
+#ifdef DEV_NETMAP
+				iaq->nnmrxq1g = min(n, nnmrxq1g);
+#endif
 			}
 
 			if (itype != INTR_MSI || powerof2(iaq->nirq))
 				goto allocate;
 		}
 
 		/*
 		 * Least desirable option: one interrupt vector for everything.
 		 */
 		iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
+		iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
 #ifdef TCP_OFFLOAD
 		if (is_offload(sc))
 			iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
 #endif
+#ifdef DEV_NETMAP
+		iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
+#endif
 
 allocate:
 		navail = iaq->nirq;
 		rc = 0;
 		if (itype == INTR_MSIX)
 			rc = pci_alloc_msix(sc->dev, &navail);
 		else if (itype == INTR_MSI)
 			rc = pci_alloc_msi(sc->dev, &navail);
 
 		if (rc == 0) {
 			if (navail == iaq->nirq)
 				return (0);
 
 			/*
 			 * Didn't get the number requested.  Use whatever number
 			 * the kernel is willing to allocate (it's in navail).
 			 */
 			device_printf(sc->dev, "fewer vectors than requested, "
 			    "type=%d, req=%d, rcvd=%d; will downshift req.\n",
 			    itype, iaq->nirq, navail);
 			pci_release_msi(sc->dev);
 			goto restart;
 		}
 
 		device_printf(sc->dev,
 		    "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
 		    itype, rc, iaq->nirq, navail);
 	}
 
 	device_printf(sc->dev,
 	    "failed to find a usable interrupt type.  "
 	    "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
 	    pci_msix_count(sc->dev), pci_msi_count(sc->dev));
 
 	return (ENXIO);
 }
 
 #define FW_VERSION(chip) ( \
     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
 
 struct fw_info {
 	uint8_t chip;
 	char *kld_name;
 	char *fw_mod_name;
 	struct fw_hdr fw_hdr;	/* XXX: waste of space, need a sparse struct */
 } fw_info[] = {
 	{
 		.chip = CHELSIO_T4,
 		.kld_name = "t4fw_cfg",
 		.fw_mod_name = "t4fw",
 		.fw_hdr = {
 			.chip = FW_HDR_CHIP_T4,
 			.fw_ver = htobe32_const(FW_VERSION(T4)),
 			.intfver_nic = FW_INTFVER(T4, NIC),
 			.intfver_vnic = FW_INTFVER(T4, VNIC),
 			.intfver_ofld = FW_INTFVER(T4, OFLD),
 			.intfver_ri = FW_INTFVER(T4, RI),
 			.intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
 			.intfver_iscsi = FW_INTFVER(T4, ISCSI),
 			.intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
 			.intfver_fcoe = FW_INTFVER(T4, FCOE),
 		},
 	}, {
 		.chip = CHELSIO_T5,
 		.kld_name = "t5fw_cfg",
 		.fw_mod_name = "t5fw",
 		.fw_hdr = {
 			.chip = FW_HDR_CHIP_T5,
 			.fw_ver = htobe32_const(FW_VERSION(T5)),
 			.intfver_nic = FW_INTFVER(T5, NIC),
 			.intfver_vnic = FW_INTFVER(T5, VNIC),
 			.intfver_ofld = FW_INTFVER(T5, OFLD),
 			.intfver_ri = FW_INTFVER(T5, RI),
 			.intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
 			.intfver_iscsi = FW_INTFVER(T5, ISCSI),
 			.intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
 			.intfver_fcoe = FW_INTFVER(T5, FCOE),
 		},
 	}
 };
 
 static struct fw_info *
 find_fw_info(int chip)
 {
 	int i;
 
 	for (i = 0; i < nitems(fw_info); i++) {
 		if (fw_info[i].chip == chip)
 			return (&fw_info[i]);
 	}
 	return (NULL);
 }
 
 /*
  * Is the given firmware API compatible with the one the driver was compiled
  * with?
  */
 static int
 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
 {
 
 	/* short circuit if it's the exact same firmware version */
 	if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
 		return (1);
 
 	/*
 	 * XXX: Is this too conservative?  Perhaps I should limit this to the
 	 * features that are supported in the driver.
 	 */
 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
 	if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
 	    SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
 	    SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
 		return (1);
 #undef SAME_INTF
 
 	return (0);
 }
 
 /*
  * The firmware in the KLD is usable, but should it be installed?  This routine
  * explains itself in detail if it indicates the KLD firmware should be
  * installed.
  */
 static int
 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
 {
 	const char *reason;
 
 	if (!card_fw_usable) {
 		reason = "incompatible or unusable";
 		goto install;
 	}
 
 	if (k > c) {
 		reason = "older than the version bundled with this driver";
 		goto install;
 	}
 
 	if (t4_fw_install == 2 && k != c) {
 		reason = "different than the version bundled with this driver";
 		goto install;
 	}
 
 	return (0);
 
 install:
 	if (t4_fw_install == 0) {
 		device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
 		    "but the driver is prohibited from installing a different "
 		    "firmware on the card.\n",
 		    G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
 		    G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
 
 		return (0);
 	}
 
 	device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
 	    "installing firmware %u.%u.%u.%u on card.\n",
 	    G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
 	    G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
 	    G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
 	    G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
 
 	return (1);
 }
 /*
  * Establish contact with the firmware and determine if we are the master driver
  * or not, and whether we are responsible for chip initialization.
  */
 static int
 prep_firmware(struct adapter *sc)
 {
 	const struct firmware *fw = NULL, *default_cfg;
 	int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
 	enum dev_state state;
 	struct fw_info *fw_info;
 	struct fw_hdr *card_fw;		/* fw on the card */
 	const struct fw_hdr *kld_fw;	/* fw in the KLD */
 	const struct fw_hdr *drv_fw;	/* fw header the driver was compiled
 					   against */
 
 	/* Contact firmware. */
 	rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
 	if (rc < 0 || state == DEV_STATE_ERR) {
 		rc = -rc;
 		device_printf(sc->dev,
 		    "failed to connect to the firmware: %d, %d.\n", rc, state);
 		return (rc);
 	}
 	pf = rc;
 	if (pf == sc->mbox)
 		sc->flags |= MASTER_PF;
 	else if (state == DEV_STATE_UNINIT) {
 		/*
 		 * We didn't get to be the master so we definitely won't be
 		 * configuring the chip.  It's a bug if someone else hasn't
 		 * configured it already.
 		 */
 		device_printf(sc->dev, "couldn't be master(%d), "
 		    "device not already initialized either(%d).\n", rc, state);
 		return (EDOOFUS);
 	}
 
 	/* This is the firmware whose headers the driver was compiled against */
 	fw_info = find_fw_info(chip_id(sc));
 	if (fw_info == NULL) {
 		device_printf(sc->dev,
 		    "unable to look up firmware information for chip %d.\n",
 		    chip_id(sc));
 		return (EINVAL);
 	}
 	drv_fw = &fw_info->fw_hdr;
 
 	/*
 	 * The firmware KLD contains many modules.  The KLD name is also the
 	 * name of the module that contains the default config file.
 	 */
 	default_cfg = firmware_get(fw_info->kld_name);
 
 	/* Read the header of the firmware on the card */
 	card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
 	rc = -t4_read_flash(sc, FLASH_FW_START,
 	    sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
 	if (rc == 0)
 		card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
 	else {
 		device_printf(sc->dev,
 		    "Unable to read card's firmware header: %d\n", rc);
 		card_fw_usable = 0;
 	}
 
 	/* This is the firmware in the KLD */
 	fw = firmware_get(fw_info->fw_mod_name);
 	if (fw != NULL) {
 		kld_fw = (const void *)fw->data;
 		kld_fw_usable = fw_compatible(drv_fw, kld_fw);
 	} else {
 		kld_fw = NULL;
 		kld_fw_usable = 0;
 	}
 
 	if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
 	    (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
 		/*
 		 * Common case: the firmware on the card is an exact match and
 		 * the KLD is an exact match too, or the KLD is
 		 * absent/incompatible.  Note that t4_fw_install = 2 is ignored
 		 * here -- use cxgbetool loadfw if you want to reinstall the
 		 * same firmware as the one on the card.
 		 */
 	} else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
 	    should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
 	    be32toh(card_fw->fw_ver))) {
 
 		rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
 		if (rc != 0) {
 			device_printf(sc->dev,
 			    "failed to install firmware: %d\n", rc);
 			goto done;
 		}
 
 		/* Installed successfully, update the cached header too. */
 		memcpy(card_fw, kld_fw, sizeof(*card_fw));
 		card_fw_usable = 1;
 		need_fw_reset = 0;	/* already reset as part of load_fw */
 	}
 
 	if (!card_fw_usable) {
 		uint32_t d, c, k;
 
 		d = ntohl(drv_fw->fw_ver);
 		c = ntohl(card_fw->fw_ver);
 		k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
 
 		device_printf(sc->dev, "Cannot find a usable firmware: "
 		    "fw_install %d, chip state %d, "
 		    "driver compiled with %d.%d.%d.%d, "
 		    "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
 		    t4_fw_install, state,
 		    G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
 		    G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
 		    G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
 		    G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
 		    G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
 		    G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
 		rc = EINVAL;
 		goto done;
 	}
 
 	/* We're using whatever's on the card and it's known to be good. */
 	sc->params.fw_vers = ntohl(card_fw->fw_ver);
 	snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
 	    G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
 	    G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
 	    G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
 	    G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
 	t4_get_tp_version(sc, &sc->params.tp_vers);
 
 	/* Reset device */
 	if (need_fw_reset &&
 	    (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
 		device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
 		if (rc != ETIMEDOUT && rc != EIO)
 			t4_fw_bye(sc, sc->mbox);
 		goto done;
 	}
 	sc->flags |= FW_OK;
 
 	rc = get_params__pre_init(sc);
 	if (rc != 0)
 		goto done; /* error message displayed already */
 
 	/* Partition adapter resources as specified in the config file. */
 	if (state == DEV_STATE_UNINIT) {
 
 		KASSERT(sc->flags & MASTER_PF,
 		    ("%s: trying to change chip settings when not master.",
 		    __func__));
 
 		rc = partition_resources(sc, default_cfg, fw_info->kld_name);
 		if (rc != 0)
 			goto done;	/* error message displayed already */
 
 		t4_tweak_chip_settings(sc);
 
 		/* get basic stuff going */
 		rc = -t4_fw_initialize(sc, sc->mbox);
 		if (rc != 0) {
 			device_printf(sc->dev, "fw init failed: %d.\n", rc);
 			goto done;
 		}
 	} else {
 		snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
 		sc->cfcsum = 0;
 	}
 
 done:
 	free(card_fw, M_CXGBE);
 	if (fw != NULL)
 		firmware_put(fw, FIRMWARE_UNLOAD);
 	if (default_cfg != NULL)
 		firmware_put(default_cfg, FIRMWARE_UNLOAD);
 
 	return (rc);
 }
 
 #define FW_PARAM_DEV(param) \
 	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
 	 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
 #define FW_PARAM_PFVF(param) \
 	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
 	 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
 
 /*
  * Partition chip resources for use between various PFs, VFs, etc.
  */
 static int
 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
     const char *name_prefix)
 {
 	const struct firmware *cfg = NULL;
 	int rc = 0;
 	struct fw_caps_config_cmd caps;
 	uint32_t mtype, moff, finicsum, cfcsum;
 
 	/*
 	 * Figure out what configuration file to use.  Pick the default config
 	 * file for the card if the user hasn't specified one explicitly.
 	 */
 	snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
 	if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
 		/* Card specific overrides go here. */
 		if (pci_get_device(sc->dev) == 0x440a)
 			snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
 		if (is_fpga(sc))
 			snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
 	}
 
 	/*
 	 * We need to load another module if the profile is anything except
 	 * "default" or "flash".
 	 */
 	if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
 	    strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
 		char s[32];
 
 		snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
 		cfg = firmware_get(s);
 		if (cfg == NULL) {
 			if (default_cfg != NULL) {
 				device_printf(sc->dev,
 				    "unable to load module \"%s\" for "
 				    "configuration profile \"%s\", will use "
 				    "the default config file instead.\n",
 				    s, sc->cfg_file);
 				snprintf(sc->cfg_file, sizeof(sc->cfg_file),
 				    "%s", DEFAULT_CF);
 			} else {
 				device_printf(sc->dev,
 				    "unable to load module \"%s\" for "
 				    "configuration profile \"%s\", will use "
 				    "the config file on the card's flash "
 				    "instead.\n", s, sc->cfg_file);
 				snprintf(sc->cfg_file, sizeof(sc->cfg_file),
 				    "%s", FLASH_CF);
 			}
 		}
 	}
 
 	if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
 	    default_cfg == NULL) {
 		device_printf(sc->dev,
 		    "default config file not available, will use the config "
 		    "file on the card's flash instead.\n");
 		snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
 	}
 
 	if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
 		u_int cflen, i, n;
 		const uint32_t *cfdata;
 		uint32_t param, val, addr, off, mw_base, mw_aperture;
 
 		KASSERT(cfg != NULL || default_cfg != NULL,
 		    ("%s: no config to upload", __func__));
 
 		/*
 		 * Ask the firmware where it wants us to upload the config file.
 		 */
 		param = FW_PARAM_DEV(CF);
 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
 		if (rc != 0) {
 			/* No support for config file?  Shouldn't happen. */
 			device_printf(sc->dev,
 			    "failed to query config file location: %d.\n", rc);
 			goto done;
 		}
 		mtype = G_FW_PARAMS_PARAM_Y(val);
 		moff = G_FW_PARAMS_PARAM_Z(val) << 16;
 
 		/*
 		 * XXX: sheer laziness.  We deliberately added 4 bytes of
 		 * useless stuffing/comments at the end of the config file so
 		 * it's ok to simply throw away the last remaining bytes when
 		 * the config file is not an exact multiple of 4.  This also
 		 * helps with the validate_mt_off_len check.
 		 */
 		if (cfg != NULL) {
 			cflen = cfg->datasize & ~3;
 			cfdata = cfg->data;
 		} else {
 			cflen = default_cfg->datasize & ~3;
 			cfdata = default_cfg->data;
 		}
 
 		if (cflen > FLASH_CFG_MAX_SIZE) {
 			device_printf(sc->dev,
 			    "config file too long (%d, max allowed is %d).  "
 			    "Will try to use the config on the card, if any.\n",
 			    cflen, FLASH_CFG_MAX_SIZE);
 			goto use_config_on_flash;
 		}
 
 		rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
 		if (rc != 0) {
 			device_printf(sc->dev,
 			    "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
 			    "Will try to use the config on the card, if any.\n",
 			    __func__, mtype, moff, cflen, rc);
 			goto use_config_on_flash;
 		}
 
 		memwin_info(sc, 2, &mw_base, &mw_aperture);
 		while (cflen) {
 			off = position_memwin(sc, 2, addr);
 			n = min(cflen, mw_aperture - off);
 			for (i = 0; i < n; i += 4)
 				t4_write_reg(sc, mw_base + off + i, *cfdata++);
 			cflen -= n;
 			addr += n;
 		}
 	} else {
 use_config_on_flash:
 		mtype = FW_MEMTYPE_FLASH;
 		moff = t4_flash_cfg_addr(sc);
 	}
 
 	bzero(&caps, sizeof(caps));
 	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
 	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
 	caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
 	    V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
 	    V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
 	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
 	if (rc != 0) {
 		device_printf(sc->dev,
 		    "failed to pre-process config file: %d "
 		    "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
 		goto done;
 	}
 
 	finicsum = be32toh(caps.finicsum);
 	cfcsum = be32toh(caps.cfcsum);
 	if (finicsum != cfcsum) {
 		device_printf(sc->dev,
 		    "WARNING: config file checksum mismatch: %08x %08x\n",
 		    finicsum, cfcsum);
 	}
 	sc->cfcsum = cfcsum;
 
 #define LIMIT_CAPS(x) do { \
 	caps.x &= htobe16(t4_##x##_allowed); \
 } while (0)
 
 	/*
 	 * Let the firmware know what features will (not) be used so it can tune
 	 * things accordingly.
 	 */
 	LIMIT_CAPS(linkcaps);
 	LIMIT_CAPS(niccaps);
 	LIMIT_CAPS(toecaps);
 	LIMIT_CAPS(rdmacaps);
 	LIMIT_CAPS(iscsicaps);
 	LIMIT_CAPS(fcoecaps);
 #undef LIMIT_CAPS
 
 	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
 	    F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
 	caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
 	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
 	if (rc != 0) {
 		device_printf(sc->dev,
 		    "failed to process config file: %d.\n", rc);
 	}
 done:
 	if (cfg != NULL)
 		firmware_put(cfg, FIRMWARE_UNLOAD);
 	return (rc);
 }
 
 /*
  * Retrieve parameters that are needed (or nice to have) very early.
  */
 static int
 get_params__pre_init(struct adapter *sc)
 {
 	int rc;
 	uint32_t param[2], val[2];
 	struct fw_devlog_cmd cmd;
 	struct devlog_params *dlog = &sc->params.devlog;
 
 	param[0] = FW_PARAM_DEV(PORTVEC);
 	param[1] = FW_PARAM_DEV(CCLK);
 	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
 	if (rc != 0) {
 		device_printf(sc->dev,
 		    "failed to query parameters (pre_init): %d.\n", rc);
 		return (rc);
 	}
 
 	sc->params.portvec = val[0];
 	sc->params.nports = bitcount32(val[0]);
 	sc->params.vpd.cclk = val[1];
 
 	/* Read device log parameters. */
 	bzero(&cmd, sizeof(cmd));
 	cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
 	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
 	cmd.retval_len16 = htobe32(FW_LEN16(cmd));
 	rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
 	if (rc != 0) {
 		device_printf(sc->dev,
 		    "failed to get devlog parameters: %d.\n", rc);
 		bzero(dlog, sizeof (*dlog));
 		rc = 0;	/* devlog isn't critical for device operation */
 	} else {
 		val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
 		dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
 		dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
 		dlog->size = be32toh(cmd.memsize_devlog);
 	}
 
 	return (rc);
 }
 
 /*
  * Retrieve various parameters that are of interest to the driver.  The device
  * has been initialized by the firmware at this point.
  */
 static int
 get_params__post_init(struct adapter *sc)
 {
 	int rc;
 	uint32_t param[7], val[7];
 	struct fw_caps_config_cmd caps;
 
 	param[0] = FW_PARAM_PFVF(IQFLINT_START);
 	param[1] = FW_PARAM_PFVF(EQ_START);
 	param[2] = FW_PARAM_PFVF(FILTER_START);
 	param[3] = FW_PARAM_PFVF(FILTER_END);
 	param[4] = FW_PARAM_PFVF(L2T_START);
 	param[5] = FW_PARAM_PFVF(L2T_END);
 	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
 	if (rc != 0) {
 		device_printf(sc->dev,
 		    "failed to query parameters (post_init): %d.\n", rc);
 		return (rc);
 	}
 
 	sc->sge.iq_start = val[0];
 	sc->sge.eq_start = val[1];
 	sc->tids.ftid_base = val[2];
 	sc->tids.nftids = val[3] - val[2] + 1;
 	sc->params.ftid_min = val[2];
 	sc->params.ftid_max = val[3];
 	sc->vres.l2t.start = val[4];
 	sc->vres.l2t.size = val[5] - val[4] + 1;
 	KASSERT(sc->vres.l2t.size <= L2T_SIZE,
 	    ("%s: L2 table size (%u) larger than expected (%u)",
 	    __func__, sc->vres.l2t.size, L2T_SIZE));
 
 	/* get capabilites */
 	bzero(&caps, sizeof(caps));
 	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
 	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
 	caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
 	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
 	if (rc != 0) {
 		device_printf(sc->dev,
 		    "failed to get card capabilities: %d.\n", rc);
 		return (rc);
 	}
 
 #define READ_CAPS(x) do { \
 	sc->x = htobe16(caps.x); \
 } while (0)
 	READ_CAPS(linkcaps);
 	READ_CAPS(niccaps);
 	READ_CAPS(toecaps);
 	READ_CAPS(rdmacaps);
 	READ_CAPS(iscsicaps);
 	READ_CAPS(fcoecaps);
 
 	if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
 		param[0] = FW_PARAM_PFVF(ETHOFLD_START);
 		param[1] = FW_PARAM_PFVF(ETHOFLD_END);
 		param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
 		if (rc != 0) {
 			device_printf(sc->dev,
 			    "failed to query NIC parameters: %d.\n", rc);
 			return (rc);
 		}
 		sc->tids.etid_base = val[0];
 		sc->params.etid_min = val[0];
 		sc->tids.netids = val[1] - val[0] + 1;
 		sc->params.netids = sc->tids.netids;
 		sc->params.eo_wr_cred = val[2];
 		sc->params.ethoffload = 1;
 	}
 
 	if (sc->toecaps) {
 		/* query offload-related parameters */
 		param[0] = FW_PARAM_DEV(NTID);
 		param[1] = FW_PARAM_PFVF(SERVER_START);
 		param[2] = FW_PARAM_PFVF(SERVER_END);
 		param[3] = FW_PARAM_PFVF(TDDP_START);
 		param[4] = FW_PARAM_PFVF(TDDP_END);
 		param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
 		if (rc != 0) {
 			device_printf(sc->dev,
 			    "failed to query TOE parameters: %d.\n", rc);
 			return (rc);
 		}
 		sc->tids.ntids = val[0];
 		sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
 		sc->tids.stid_base = val[1];
 		sc->tids.nstids = val[2] - val[1] + 1;
 		sc->vres.ddp.start = val[3];
 		sc->vres.ddp.size = val[4] - val[3] + 1;
 		sc->params.ofldq_wr_cred = val[5];
 		sc->params.offload = 1;
 	}
 	if (sc->rdmacaps) {
 		param[0] = FW_PARAM_PFVF(STAG_START);
 		param[1] = FW_PARAM_PFVF(STAG_END);
 		param[2] = FW_PARAM_PFVF(RQ_START);
 		param[3] = FW_PARAM_PFVF(RQ_END);
 		param[4] = FW_PARAM_PFVF(PBL_START);
 		param[5] = FW_PARAM_PFVF(PBL_END);
 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
 		if (rc != 0) {
 			device_printf(sc->dev,
 			    "failed to query RDMA parameters(1): %d.\n", rc);
 			return (rc);
 		}
 		sc->vres.stag.start = val[0];
 		sc->vres.stag.size = val[1] - val[0] + 1;
 		sc->vres.rq.start = val[2];
 		sc->vres.rq.size = val[3] - val[2] + 1;
 		sc->vres.pbl.start = val[4];
 		sc->vres.pbl.size = val[5] - val[4] + 1;
 
 		param[0] = FW_PARAM_PFVF(SQRQ_START);
 		param[1] = FW_PARAM_PFVF(SQRQ_END);
 		param[2] = FW_PARAM_PFVF(CQ_START);
 		param[3] = FW_PARAM_PFVF(CQ_END);
 		param[4] = FW_PARAM_PFVF(OCQ_START);
 		param[5] = FW_PARAM_PFVF(OCQ_END);
 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
 		if (rc != 0) {
 			device_printf(sc->dev,
 			    "failed to query RDMA parameters(2): %d.\n", rc);
 			return (rc);
 		}
 		sc->vres.qp.start = val[0];
 		sc->vres.qp.size = val[1] - val[0] + 1;
 		sc->vres.cq.start = val[2];
 		sc->vres.cq.size = val[3] - val[2] + 1;
 		sc->vres.ocq.start = val[4];
 		sc->vres.ocq.size = val[5] - val[4] + 1;
 	}
 	if (sc->iscsicaps) {
 		param[0] = FW_PARAM_PFVF(ISCSI_START);
 		param[1] = FW_PARAM_PFVF(ISCSI_END);
 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
 		if (rc != 0) {
 			device_printf(sc->dev,
 			    "failed to query iSCSI parameters: %d.\n", rc);
 			return (rc);
 		}
 		sc->vres.iscsi.start = val[0];
 		sc->vres.iscsi.size = val[1] - val[0] + 1;
 	}
 
 	/*
 	 * We've got the params we wanted to query via the firmware.  Now grab
 	 * some others directly from the chip.
 	 */
 	rc = t4_read_chip_settings(sc);
 
 	return (rc);
 }
 
 static int
 set_params__post_init(struct adapter *sc)
 {
 	uint32_t param, val;
 
 	/* ask for encapsulated CPLs */
 	param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
 	val = 1;
 	(void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
 
 	return (0);
 }
 
 #undef FW_PARAM_PFVF
 #undef FW_PARAM_DEV
 
 static void
 t4_set_desc(struct adapter *sc)
 {
 	char buf[128];
 	struct adapter_params *p = &sc->params;
 
 	snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
 	    "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
 	    chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
 
 	device_set_desc_copy(sc->dev, buf);
 }
 
 static void
-build_medialist(struct port_info *pi)
+build_medialist(struct port_info *pi, struct ifmedia *media)
 {
-	struct ifmedia *media = &pi->media;
 	int data, m;
 
 	PORT_LOCK(pi);
 
 	ifmedia_removeall(media);
 
 	m = IFM_ETHER | IFM_FDX;
 	data = (pi->port_type << 8) | pi->mod_type;
 
 	switch(pi->port_type) {
 	case FW_PORT_TYPE_BT_XFI:
 		ifmedia_add(media, m | IFM_10G_T, data, NULL);
 		break;
 
 	case FW_PORT_TYPE_BT_XAUI:
 		ifmedia_add(media, m | IFM_10G_T, data, NULL);
 		/* fall through */
 
 	case FW_PORT_TYPE_BT_SGMII:
 		ifmedia_add(media, m | IFM_1000_T, data, NULL);
 		ifmedia_add(media, m | IFM_100_TX, data, NULL);
 		ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
 		ifmedia_set(media, IFM_ETHER | IFM_AUTO);
 		break;
 
 	case FW_PORT_TYPE_CX4:
 		ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
 		ifmedia_set(media, m | IFM_10G_CX4);
 		break;
 
 	case FW_PORT_TYPE_QSFP_10G:
 	case FW_PORT_TYPE_SFP:
 	case FW_PORT_TYPE_FIBER_XFI:
 	case FW_PORT_TYPE_FIBER_XAUI:
 		switch (pi->mod_type) {
 
 		case FW_PORT_MOD_TYPE_LR:
 			ifmedia_add(media, m | IFM_10G_LR, data, NULL);
 			ifmedia_set(media, m | IFM_10G_LR);
 			break;
 
 		case FW_PORT_MOD_TYPE_SR:
 			ifmedia_add(media, m | IFM_10G_SR, data, NULL);
 			ifmedia_set(media, m | IFM_10G_SR);
 			break;
 
 		case FW_PORT_MOD_TYPE_LRM:
 			ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
 			ifmedia_set(media, m | IFM_10G_LRM);
 			break;
 
 		case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
 		case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
 			ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
 			ifmedia_set(media, m | IFM_10G_TWINAX);
 			break;
 
 		case FW_PORT_MOD_TYPE_NONE:
 			m &= ~IFM_FDX;
 			ifmedia_add(media, m | IFM_NONE, data, NULL);
 			ifmedia_set(media, m | IFM_NONE);
 			break;
 
 		case FW_PORT_MOD_TYPE_NA:
 		case FW_PORT_MOD_TYPE_ER:
 		default:
 			device_printf(pi->dev,
 			    "unknown port_type (%d), mod_type (%d)\n",
 			    pi->port_type, pi->mod_type);
 			ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
 			ifmedia_set(media, m | IFM_UNKNOWN);
 			break;
 		}
 		break;
 
 	case FW_PORT_TYPE_QSFP:
 		switch (pi->mod_type) {
 
 		case FW_PORT_MOD_TYPE_LR:
 			ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
 			ifmedia_set(media, m | IFM_40G_LR4);
 			break;
 
 		case FW_PORT_MOD_TYPE_SR:
 			ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
 			ifmedia_set(media, m | IFM_40G_SR4);
 			break;
 
 		case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
 		case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
 			ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
 			ifmedia_set(media, m | IFM_40G_CR4);
 			break;
 
 		case FW_PORT_MOD_TYPE_NONE:
 			m &= ~IFM_FDX;
 			ifmedia_add(media, m | IFM_NONE, data, NULL);
 			ifmedia_set(media, m | IFM_NONE);
 			break;
 
 		default:
 			device_printf(pi->dev,
 			    "unknown port_type (%d), mod_type (%d)\n",
 			    pi->port_type, pi->mod_type);
 			ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
 			ifmedia_set(media, m | IFM_UNKNOWN);
 			break;
 		}
 		break;
 
 	default:
 		device_printf(pi->dev,
 		    "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
 		    pi->mod_type);
 		ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
 		ifmedia_set(media, m | IFM_UNKNOWN);
 		break;
 	}
 
 	PORT_UNLOCK(pi);
 }
 
 #define FW_MAC_EXACT_CHUNK	7
 
 /*
  * Program the port's XGMAC based on parameters in ifnet.  The caller also
  * indicates which parameters should be programmed (the rest are left alone).
  */
-static int
-update_mac_settings(struct port_info *pi, int flags)
+int
+update_mac_settings(struct ifnet *ifp, int flags)
 {
-	int rc;
-	struct ifnet *ifp = pi->ifp;
+	int rc = 0;
+	struct port_info *pi = ifp->if_softc;
 	struct adapter *sc = pi->adapter;
 	int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
+	uint16_t viid = 0xffff;
+	int16_t *xact_addr_filt = NULL;
 
 	ASSERT_SYNCHRONIZED_OP(sc);
 	KASSERT(flags, ("%s: not told what to update.", __func__));
 
+	if (ifp == pi->ifp) {
+		viid = pi->viid;
+		xact_addr_filt = &pi->xact_addr_filt;
+	}
+#ifdef DEV_NETMAP
+	else if (ifp == pi->nm_ifp) {
+		viid = pi->nm_viid;
+		xact_addr_filt = &pi->nm_xact_addr_filt;
+	}
+#endif
 	if (flags & XGMAC_MTU)
 		mtu = ifp->if_mtu;
 
 	if (flags & XGMAC_PROMISC)
 		promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
 
 	if (flags & XGMAC_ALLMULTI)
 		allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
 
 	if (flags & XGMAC_VLANEX)
 		vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
 
-	rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1,
-	    vlanex, false);
-	if (rc) {
-		if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc);
-		return (rc);
+	if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
+		rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
+		    1, vlanex, false);
+		if (rc) {
+			if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
+			    rc);
+			return (rc);
+		}
 	}
 
 	if (flags & XGMAC_UCADDR) {
 		uint8_t ucaddr[ETHER_ADDR_LEN];
 
 		bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
-		rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt,
-		    ucaddr, true, true);
+		rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
+		    true, true);
 		if (rc < 0) {
 			rc = -rc;
 			if_printf(ifp, "change_mac failed: %d\n", rc);
 			return (rc);
 		} else {
-			pi->xact_addr_filt = rc;
+			*xact_addr_filt = rc;
 			rc = 0;
 		}
 	}
 
 	if (flags & XGMAC_MCADDRS) {
 		const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
 		int del = 1;
 		uint64_t hash = 0;
 		struct ifmultiaddr *ifma;
 		int i = 0, j;
 
 		if_maddr_rlock(ifp);
 		TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 			if (ifma->ifma_addr->sa_family != AF_LINK)
 				continue;
 			mcaddr[i++] =
 			    LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
 
 			if (i == FW_MAC_EXACT_CHUNK) {
-				rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
-				    del, i, mcaddr, NULL, &hash, 0);
+				rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
+				    i, mcaddr, NULL, &hash, 0);
 				if (rc < 0) {
 					rc = -rc;
 					for (j = 0; j < i; j++) {
 						if_printf(ifp,
 						    "failed to add mc address"
 						    " %02x:%02x:%02x:"
 						    "%02x:%02x:%02x rc=%d\n",
 						    mcaddr[j][0], mcaddr[j][1],
 						    mcaddr[j][2], mcaddr[j][3],
 						    mcaddr[j][4], mcaddr[j][5],
 						    rc);
 					}
 					goto mcfail;
 				}
 				del = 0;
 				i = 0;
 			}
 		}
 		if (i > 0) {
-			rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
-			    del, i, mcaddr, NULL, &hash, 0);
+			rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
+			    mcaddr, NULL, &hash, 0);
 			if (rc < 0) {
 				rc = -rc;
 				for (j = 0; j < i; j++) {
 					if_printf(ifp,
 					    "failed to add mc address"
 					    " %02x:%02x:%02x:"
 					    "%02x:%02x:%02x rc=%d\n",
 					    mcaddr[j][0], mcaddr[j][1],
 					    mcaddr[j][2], mcaddr[j][3],
 					    mcaddr[j][4], mcaddr[j][5],
 					    rc);
 				}
 				goto mcfail;
 			}
 		}
 
-		rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0);
+		rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
 		if (rc != 0)
 			if_printf(ifp, "failed to set mc address hash: %d", rc);
 mcfail:
 		if_maddr_runlock(ifp);
 	}
 
 	return (rc);
 }
 
 int
 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
     char *wmesg)
 {
 	int rc, pri;
 
 #ifdef WITNESS
 	/* the caller thinks it's ok to sleep, but is it really? */
 	if (flags & SLEEP_OK)
 		pause("t4slptst", 1);
 #endif
 
 	if (INTR_OK)
 		pri = PCATCH;
 	else
 		pri = 0;
 
 	ADAPTER_LOCK(sc);
 	for (;;) {
 
 		if (pi && IS_DOOMED(pi)) {
 			rc = ENXIO;
 			goto done;
 		}
 
 		if (!IS_BUSY(sc)) {
 			rc = 0;
 			break;
 		}
 
 		if (!(flags & SLEEP_OK)) {
 			rc = EBUSY;
 			goto done;
 		}
 
 		if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
 			rc = EINTR;
 			goto done;
 		}
 	}
 
 	KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
 	SET_BUSY(sc);
 #ifdef INVARIANTS
 	sc->last_op = wmesg;
 	sc->last_op_thr = curthread;
 #endif
 
 done:
 	if (!(flags & HOLD_LOCK) || rc)
 		ADAPTER_UNLOCK(sc);
 
 	return (rc);
 }
 
 void
 end_synchronized_op(struct adapter *sc, int flags)
 {
 
 	if (flags & LOCK_HELD)
 		ADAPTER_LOCK_ASSERT_OWNED(sc);
 	else
 		ADAPTER_LOCK(sc);
 
 	KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
 	CLR_BUSY(sc);
 	wakeup(&sc->flags);
 	ADAPTER_UNLOCK(sc);
 }
 
 static int
 cxgbe_init_synchronized(struct port_info *pi)
 {
 	struct adapter *sc = pi->adapter;
 	struct ifnet *ifp = pi->ifp;
 	int rc = 0;
 
 	ASSERT_SYNCHRONIZED_OP(sc);
 
 	if (isset(&sc->open_device_map, pi->port_id)) {
 		KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
 		    ("mismatch between open_device_map and if_drv_flags"));
 		return (0);	/* already running */
 	}
 
 	if (!(sc->flags & FULL_INIT_DONE) &&
 	    ((rc = adapter_full_init(sc)) != 0))
 		return (rc);	/* error message displayed already */
 
 	if (!(pi->flags & PORT_INIT_DONE) &&
 	    ((rc = port_full_init(pi)) != 0))
 		return (rc); /* error message displayed already */
 
-	rc = update_mac_settings(pi, XGMAC_ALL);
+	rc = update_mac_settings(ifp, XGMAC_ALL);
 	if (rc)
 		goto done;	/* error message displayed already */
 
-	rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
-	if (rc != 0) {
-		if_printf(ifp, "start_link failed: %d\n", rc);
-		goto done;
-	}
-
 	rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
 	if (rc != 0) {
 		if_printf(ifp, "enable_vi failed: %d\n", rc);
 		goto done;
 	}
 
 	/*
 	 * The first iq of the first port to come up is used for tracing.
 	 */
 	if (sc->traceq < 0) {
 		sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
 		t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
 		    A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
 		    V_QUEUENUMBER(sc->traceq));
 		pi->flags |= HAS_TRACEQ;
 	}
 
 	/* all ok */
 	setbit(&sc->open_device_map, pi->port_id);
 	PORT_LOCK(pi);
 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
 	PORT_UNLOCK(pi);
 
 	callout_reset(&pi->tick, hz, cxgbe_tick, pi);
 done:
 	if (rc != 0)
 		cxgbe_uninit_synchronized(pi);
 
 	return (rc);
 }
 
 /*
  * Idempotent.
  */
 static int
 cxgbe_uninit_synchronized(struct port_info *pi)
 {
 	struct adapter *sc = pi->adapter;
 	struct ifnet *ifp = pi->ifp;
 	int rc;
 
 	ASSERT_SYNCHRONIZED_OP(sc);
 
 	/*
 	 * Disable the VI so that all its data in either direction is discarded
 	 * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
 	 * tick) intact as the TP can deliver negative advice or data that it's
 	 * holding in its RAM (for an offloaded connection) even after the VI is
 	 * disabled.
 	 */
 	rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
 	if (rc) {
 		if_printf(ifp, "disable_vi failed: %d\n", rc);
 		return (rc);
 	}
 
 	clrbit(&sc->open_device_map, pi->port_id);
 	PORT_LOCK(pi);
 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 	PORT_UNLOCK(pi);
 
 	pi->link_cfg.link_ok = 0;
 	pi->link_cfg.speed = 0;
 	pi->linkdnrc = -1;
 	t4_os_link_changed(sc, pi->port_id, 0, -1);
 
 	return (0);
 }
 
 /*
  * It is ok for this function to fail midway and return right away.  t4_detach
  * will walk the entire sc->irq list and clean up whatever is valid.
  */
 static int
 setup_intr_handlers(struct adapter *sc)
 {
 	int rc, rid, p, q;
 	char s[8];
 	struct irq *irq;
 	struct port_info *pi;
 	struct sge_rxq *rxq;
 #ifdef TCP_OFFLOAD
 	struct sge_ofld_rxq *ofld_rxq;
 #endif
+#ifdef DEV_NETMAP
+	struct sge_nm_rxq *nm_rxq;
+#endif
 
 	/*
 	 * Setup interrupts.
 	 */
 	irq = &sc->irq[0];
 	rid = sc->intr_type == INTR_INTX ? 0 : 1;
-	if (sc->intr_count == 1) {
-		KASSERT(!(sc->flags & INTR_DIRECT),
-		    ("%s: single interrupt && INTR_DIRECT?", __func__));
+	if (sc->intr_count == 1)
+		return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
 
-		rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all");
-		if (rc != 0)
-			return (rc);
-	} else {
-		/* Multiple interrupts. */
-		KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
-		    ("%s: too few intr.", __func__));
+	/* Multiple interrupts. */
+	KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
+	    ("%s: too few intr.", __func__));
 
-		/* The first one is always error intr */
-		rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
-		if (rc != 0)
-			return (rc);
-		irq++;
-		rid++;
+	/* The first one is always error intr */
+	rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
+	if (rc != 0)
+		return (rc);
+	irq++;
+	rid++;
 
-		/* The second one is always the firmware event queue */
-		rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq,
-		    "evt");
-		if (rc != 0)
-			return (rc);
-		irq++;
-		rid++;
+	/* The second one is always the firmware event queue */
+	rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
+	if (rc != 0)
+		return (rc);
+	irq++;
+	rid++;
 
-		/*
-		 * Note that if INTR_DIRECT is not set then either the NIC rx
-		 * queues or (exclusive or) the TOE rx queueus will be taking
-		 * direct interrupts.
-		 *
-		 * There is no need to check for is_offload(sc) as nofldrxq
-		 * will be 0 if offload is disabled.
-		 */
-		for_each_port(sc, p) {
-			pi = sc->port[p];
+	for_each_port(sc, p) {
+		pi = sc->port[p];
 
-#ifdef TCP_OFFLOAD
-			/*
-			 * Skip over the NIC queues if they aren't taking direct
-			 * interrupts.
-			 */
-			if (!(sc->flags & INTR_DIRECT) &&
-			    pi->nofldrxq > pi->nrxq)
-				goto ofld_queues;
-#endif
-			rxq = &sc->sge.rxq[pi->first_rxq];
-			for (q = 0; q < pi->nrxq; q++, rxq++) {
+		if (pi->flags & INTR_RXQ) {
+			for_each_rxq(pi, q, rxq) {
 				snprintf(s, sizeof(s), "%d.%d", p, q);
 				rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
 				    s);
 				if (rc != 0)
 					return (rc);
 				irq++;
 				rid++;
 			}
-
+		}
 #ifdef TCP_OFFLOAD
-			/*
-			 * Skip over the offload queues if they aren't taking
-			 * direct interrupts.
-			 */
-			if (!(sc->flags & INTR_DIRECT))
-				continue;
-ofld_queues:
-			ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq];
-			for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) {
+		if (pi->flags & INTR_OFLD_RXQ) {
+			for_each_ofld_rxq(pi, q, ofld_rxq) {
 				snprintf(s, sizeof(s), "%d,%d", p, q);
 				rc = t4_alloc_irq(sc, irq, rid, t4_intr,
 				    ofld_rxq, s);
 				if (rc != 0)
 					return (rc);
 				irq++;
 				rid++;
 			}
+		}
 #endif
+#ifdef DEV_NETMAP
+		if (pi->flags & INTR_NM_RXQ) {
+			for_each_nm_rxq(pi, q, nm_rxq) {
+				snprintf(s, sizeof(s), "%d-%d", p, q);
+				rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
+				    nm_rxq, s);
+				if (rc != 0)
+					return (rc);
+				irq++;
+				rid++;
+			}
 		}
+#endif
 	}
+	MPASS(irq == &sc->irq[sc->intr_count]);
 
 	return (0);
 }
 
-static int
+int
 adapter_full_init(struct adapter *sc)
 {
 	int rc, i;
 
 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 	KASSERT((sc->flags & FULL_INIT_DONE) == 0,
 	    ("%s: FULL_INIT_DONE already", __func__));
 
 	/*
 	 * queues that belong to the adapter (not any particular port).
 	 */
 	rc = t4_setup_adapter_queues(sc);
 	if (rc != 0)
 		goto done;
 
 	for (i = 0; i < nitems(sc->tq); i++) {
 		sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
 		    taskqueue_thread_enqueue, &sc->tq[i]);
 		if (sc->tq[i] == NULL) {
 			device_printf(sc->dev,
 			    "failed to allocate task queue %d\n", i);
 			rc = ENOMEM;
 			goto done;
 		}
 		taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
 		    device_get_nameunit(sc->dev), i);
 	}
 
 	t4_intr_enable(sc);
 	sc->flags |= FULL_INIT_DONE;
 done:
 	if (rc != 0)
 		adapter_full_uninit(sc);
 
 	return (rc);
 }
 
-static int
+int
 adapter_full_uninit(struct adapter *sc)
 {
 	int i;
 
 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 
 	t4_teardown_adapter_queues(sc);
 
 	for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
 		taskqueue_free(sc->tq[i]);
 		sc->tq[i] = NULL;
 	}
 
 	sc->flags &= ~FULL_INIT_DONE;
 
 	return (0);
 }
 
-static int
+int
 port_full_init(struct port_info *pi)
 {
 	struct adapter *sc = pi->adapter;
 	struct ifnet *ifp = pi->ifp;
 	uint16_t *rss;
 	struct sge_rxq *rxq;
 	int rc, i, j;
 
 	ASSERT_SYNCHRONIZED_OP(sc);
 	KASSERT((pi->flags & PORT_INIT_DONE) == 0,
 	    ("%s: PORT_INIT_DONE already", __func__));
 
 	sysctl_ctx_init(&pi->ctx);
 	pi->flags |= PORT_SYSCTL_CTX;
 
 	/*
 	 * Allocate tx/rx/fl queues for this port.
 	 */
 	rc = t4_setup_port_queues(pi);
 	if (rc != 0)
 		goto done;	/* error message displayed already */
 
 	/*
 	 * Setup RSS for this port.  Save a copy of the RSS table for later use.
 	 */
 	rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
 	for (i = 0; i < pi->rss_size;) {
 		for_each_rxq(pi, j, rxq) {
 			rss[i++] = rxq->iq.abs_id;
 			if (i == pi->rss_size)
 				break;
 		}
 	}
 
 	rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
 	    pi->rss_size);
 	if (rc != 0) {
 		if_printf(ifp, "rss_config failed: %d\n", rc);
 		goto done;
 	}
 
 	pi->rss = rss;
 	pi->flags |= PORT_INIT_DONE;
 done:
 	if (rc != 0)
 		port_full_uninit(pi);
 
 	return (rc);
 }
 
 /*
  * Idempotent.
  */
-static int
+int
 port_full_uninit(struct port_info *pi)
 {
 	struct adapter *sc = pi->adapter;
 	int i;
 	struct sge_rxq *rxq;
 	struct sge_txq *txq;
 #ifdef TCP_OFFLOAD
 	struct sge_ofld_rxq *ofld_rxq;
 	struct sge_wrq *ofld_txq;
 #endif
 
 	if (pi->flags & PORT_INIT_DONE) {
 
 		/* Need to quiesce queues.  XXX: ctrl queues? */
 
 		for_each_txq(pi, i, txq) {
 			quiesce_eq(sc, &txq->eq);
 		}
 
 #ifdef TCP_OFFLOAD
 		for_each_ofld_txq(pi, i, ofld_txq) {
 			quiesce_eq(sc, &ofld_txq->eq);
 		}
 #endif
 
 		for_each_rxq(pi, i, rxq) {
 			quiesce_iq(sc, &rxq->iq);
 			quiesce_fl(sc, &rxq->fl);
 		}
 
 #ifdef TCP_OFFLOAD
 		for_each_ofld_rxq(pi, i, ofld_rxq) {
 			quiesce_iq(sc, &ofld_rxq->iq);
 			quiesce_fl(sc, &ofld_rxq->fl);
 		}
 #endif
 		free(pi->rss, M_CXGBE);
 	}
 
 	t4_teardown_port_queues(pi);
 	pi->flags &= ~PORT_INIT_DONE;
 
 	return (0);
 }
 
 static void
 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
 {
 	EQ_LOCK(eq);
 	eq->flags |= EQ_DOOMED;
 
 	/*
 	 * Wait for the response to a credit flush if one's
 	 * pending.
 	 */
 	while (eq->flags & EQ_CRFLUSHED)
 		mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
 	EQ_UNLOCK(eq);
 
 	callout_drain(&eq->tx_callout);	/* XXX: iffy */
 	pause("callout", 10);		/* Still iffy */
 
 	taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
 }
 
 static void
 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
 {
 	(void) sc;	/* unused */
 
 	/* Synchronize with the interrupt handler */
 	while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
 		pause("iqfree", 1);
 }
 
 static void
 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
 {
 	mtx_lock(&sc->sfl_lock);
 	FL_LOCK(fl);
 	fl->flags |= FL_DOOMED;
 	FL_UNLOCK(fl);
 	mtx_unlock(&sc->sfl_lock);
 
 	callout_drain(&sc->sfl_callout);
 	KASSERT((fl->flags & FL_STARVING) == 0,
 	    ("%s: still starving", __func__));
 }
 
 static int
 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
     driver_intr_t *handler, void *arg, char *name)
 {
 	int rc;
 
 	irq->rid = rid;
 	irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
 	    RF_SHAREABLE | RF_ACTIVE);
 	if (irq->res == NULL) {
 		device_printf(sc->dev,
 		    "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
 		return (ENOMEM);
 	}
 
 	rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
 	    NULL, handler, arg, &irq->tag);
 	if (rc != 0) {
 		device_printf(sc->dev,
 		    "failed to setup interrupt for rid %d, name %s: %d\n",
 		    rid, name, rc);
 	} else if (name)
 		bus_describe_intr(sc->dev, irq->res, irq->tag, name);
 
 	return (rc);
 }
 
 static int
 t4_free_irq(struct adapter *sc, struct irq *irq)
 {
 	if (irq->tag)
 		bus_teardown_intr(sc->dev, irq->res, irq->tag);
 	if (irq->res)
 		bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
 
 	bzero(irq, sizeof(*irq));
 
 	return (0);
 }
 
 static void
 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
     unsigned int end)
 {
 	uint32_t *p = (uint32_t *)(buf + start);
 
 	for ( ; start <= end; start += sizeof(uint32_t))
 		*p++ = t4_read_reg(sc, start);
 }
 
 static void
 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
 {
 	int i, n;
 	const unsigned int *reg_ranges;
 	static const unsigned int t4_reg_ranges[] = {
 		0x1008, 0x1108,
 		0x1180, 0x11b4,
 		0x11fc, 0x123c,
 		0x1300, 0x173c,
 		0x1800, 0x18fc,
 		0x3000, 0x30d8,
 		0x30e0, 0x5924,
 		0x5960, 0x59d4,
 		0x5a00, 0x5af8,
 		0x6000, 0x6098,
 		0x6100, 0x6150,
 		0x6200, 0x6208,
 		0x6240, 0x6248,
 		0x6280, 0x6338,
 		0x6370, 0x638c,
 		0x6400, 0x643c,
 		0x6500, 0x6524,
 		0x6a00, 0x6a38,
 		0x6a60, 0x6a78,
 		0x6b00, 0x6b84,
 		0x6bf0, 0x6c84,
 		0x6cf0, 0x6d84,
 		0x6df0, 0x6e84,
 		0x6ef0, 0x6f84,
 		0x6ff0, 0x7084,
 		0x70f0, 0x7184,
 		0x71f0, 0x7284,
 		0x72f0, 0x7384,
 		0x73f0, 0x7450,
 		0x7500, 0x7530,
 		0x7600, 0x761c,
 		0x7680, 0x76cc,
 		0x7700, 0x7798,
 		0x77c0, 0x77fc,
 		0x7900, 0x79fc,
 		0x7b00, 0x7c38,
 		0x7d00, 0x7efc,
 		0x8dc0, 0x8e1c,
 		0x8e30, 0x8e78,
 		0x8ea0, 0x8f6c,
 		0x8fc0, 0x9074,
 		0x90fc, 0x90fc,
 		0x9400, 0x9458,
 		0x9600, 0x96bc,
 		0x9800, 0x9808,
 		0x9820, 0x983c,
 		0x9850, 0x9864,
 		0x9c00, 0x9c6c,
 		0x9c80, 0x9cec,
 		0x9d00, 0x9d6c,
 		0x9d80, 0x9dec,
 		0x9e00, 0x9e6c,
 		0x9e80, 0x9eec,
 		0x9f00, 0x9f6c,
 		0x9f80, 0x9fec,
 		0xd004, 0xd03c,
 		0xdfc0, 0xdfe0,
 		0xe000, 0xea7c,
 		0xf000, 0x11110,
 		0x11118, 0x11190,
 		0x19040, 0x1906c,
 		0x19078, 0x19080,
 		0x1908c, 0x19124,
 		0x19150, 0x191b0,
 		0x191d0, 0x191e8,
 		0x19238, 0x1924c,
 		0x193f8, 0x19474,
 		0x19490, 0x194f8,
 		0x19800, 0x19f30,
 		0x1a000, 0x1a06c,
 		0x1a0b0, 0x1a120,
 		0x1a128, 0x1a138,
 		0x1a190, 0x1a1c4,
 		0x1a1fc, 0x1a1fc,
 		0x1e040, 0x1e04c,
 		0x1e284, 0x1e28c,
 		0x1e2c0, 0x1e2c0,
 		0x1e2e0, 0x1e2e0,
 		0x1e300, 0x1e384,
 		0x1e3c0, 0x1e3c8,
 		0x1e440, 0x1e44c,
 		0x1e684, 0x1e68c,
 		0x1e6c0, 0x1e6c0,
 		0x1e6e0, 0x1e6e0,
 		0x1e700, 0x1e784,
 		0x1e7c0, 0x1e7c8,
 		0x1e840, 0x1e84c,
 		0x1ea84, 0x1ea8c,
 		0x1eac0, 0x1eac0,
 		0x1eae0, 0x1eae0,
 		0x1eb00, 0x1eb84,
 		0x1ebc0, 0x1ebc8,
 		0x1ec40, 0x1ec4c,
 		0x1ee84, 0x1ee8c,
 		0x1eec0, 0x1eec0,
 		0x1eee0, 0x1eee0,
 		0x1ef00, 0x1ef84,
 		0x1efc0, 0x1efc8,
 		0x1f040, 0x1f04c,
 		0x1f284, 0x1f28c,
 		0x1f2c0, 0x1f2c0,
 		0x1f2e0, 0x1f2e0,
 		0x1f300, 0x1f384,
 		0x1f3c0, 0x1f3c8,
 		0x1f440, 0x1f44c,
 		0x1f684, 0x1f68c,
 		0x1f6c0, 0x1f6c0,
 		0x1f6e0, 0x1f6e0,
 		0x1f700, 0x1f784,
 		0x1f7c0, 0x1f7c8,
 		0x1f840, 0x1f84c,
 		0x1fa84, 0x1fa8c,
 		0x1fac0, 0x1fac0,
 		0x1fae0, 0x1fae0,
 		0x1fb00, 0x1fb84,
 		0x1fbc0, 0x1fbc8,
 		0x1fc40, 0x1fc4c,
 		0x1fe84, 0x1fe8c,
 		0x1fec0, 0x1fec0,
 		0x1fee0, 0x1fee0,
 		0x1ff00, 0x1ff84,
 		0x1ffc0, 0x1ffc8,
 		0x20000, 0x2002c,
 		0x20100, 0x2013c,
 		0x20190, 0x201c8,
 		0x20200, 0x20318,
 		0x20400, 0x20528,
 		0x20540, 0x20614,
 		0x21000, 0x21040,
 		0x2104c, 0x21060,
 		0x210c0, 0x210ec,
 		0x21200, 0x21268,
 		0x21270, 0x21284,
 		0x212fc, 0x21388,
 		0x21400, 0x21404,
 		0x21500, 0x21518,
 		0x2152c, 0x2153c,
 		0x21550, 0x21554,
 		0x21600, 0x21600,
 		0x21608, 0x21628,
 		0x21630, 0x2163c,
 		0x21700, 0x2171c,
 		0x21780, 0x2178c,
 		0x21800, 0x21c38,
 		0x21c80, 0x21d7c,
 		0x21e00, 0x21e04,
 		0x22000, 0x2202c,
 		0x22100, 0x2213c,
 		0x22190, 0x221c8,
 		0x22200, 0x22318,
 		0x22400, 0x22528,
 		0x22540, 0x22614,
 		0x23000, 0x23040,
 		0x2304c, 0x23060,
 		0x230c0, 0x230ec,
 		0x23200, 0x23268,
 		0x23270, 0x23284,
 		0x232fc, 0x23388,
 		0x23400, 0x23404,
 		0x23500, 0x23518,
 		0x2352c, 0x2353c,
 		0x23550, 0x23554,
 		0x23600, 0x23600,
 		0x23608, 0x23628,
 		0x23630, 0x2363c,
 		0x23700, 0x2371c,
 		0x23780, 0x2378c,
 		0x23800, 0x23c38,
 		0x23c80, 0x23d7c,
 		0x23e00, 0x23e04,
 		0x24000, 0x2402c,
 		0x24100, 0x2413c,
 		0x24190, 0x241c8,
 		0x24200, 0x24318,
 		0x24400, 0x24528,
 		0x24540, 0x24614,
 		0x25000, 0x25040,
 		0x2504c, 0x25060,
 		0x250c0, 0x250ec,
 		0x25200, 0x25268,
 		0x25270, 0x25284,
 		0x252fc, 0x25388,
 		0x25400, 0x25404,
 		0x25500, 0x25518,
 		0x2552c, 0x2553c,
 		0x25550, 0x25554,
 		0x25600, 0x25600,
 		0x25608, 0x25628,
 		0x25630, 0x2563c,
 		0x25700, 0x2571c,
 		0x25780, 0x2578c,
 		0x25800, 0x25c38,
 		0x25c80, 0x25d7c,
 		0x25e00, 0x25e04,
 		0x26000, 0x2602c,
 		0x26100, 0x2613c,
 		0x26190, 0x261c8,
 		0x26200, 0x26318,
 		0x26400, 0x26528,
 		0x26540, 0x26614,
 		0x27000, 0x27040,
 		0x2704c, 0x27060,
 		0x270c0, 0x270ec,
 		0x27200, 0x27268,
 		0x27270, 0x27284,
 		0x272fc, 0x27388,
 		0x27400, 0x27404,
 		0x27500, 0x27518,
 		0x2752c, 0x2753c,
 		0x27550, 0x27554,
 		0x27600, 0x27600,
 		0x27608, 0x27628,
 		0x27630, 0x2763c,
 		0x27700, 0x2771c,
 		0x27780, 0x2778c,
 		0x27800, 0x27c38,
 		0x27c80, 0x27d7c,
 		0x27e00, 0x27e04
 	};
 	static const unsigned int t5_reg_ranges[] = {
 		0x1008, 0x1148,
 		0x1180, 0x11b4,
 		0x11fc, 0x123c,
 		0x1280, 0x173c,
 		0x1800, 0x18fc,
 		0x3000, 0x3028,
 		0x3060, 0x30d8,
 		0x30e0, 0x30fc,
 		0x3140, 0x357c,
 		0x35a8, 0x35cc,
 		0x35ec, 0x35ec,
 		0x3600, 0x5624,
 		0x56cc, 0x575c,
 		0x580c, 0x5814,
 		0x5890, 0x58bc,
 		0x5940, 0x59dc,
 		0x59fc, 0x5a18,
 		0x5a60, 0x5a9c,
 		0x5b94, 0x5bfc,
 		0x6000, 0x6040,
 		0x6058, 0x614c,
 		0x7700, 0x7798,
 		0x77c0, 0x78fc,
 		0x7b00, 0x7c54,
 		0x7d00, 0x7efc,
 		0x8dc0, 0x8de0,
 		0x8df8, 0x8e84,
 		0x8ea0, 0x8f84,
 		0x8fc0, 0x90f8,
 		0x9400, 0x9470,
 		0x9600, 0x96f4,
 		0x9800, 0x9808,
 		0x9820, 0x983c,
 		0x9850, 0x9864,
 		0x9c00, 0x9c6c,
 		0x9c80, 0x9cec,
 		0x9d00, 0x9d6c,
 		0x9d80, 0x9dec,
 		0x9e00, 0x9e6c,
 		0x9e80, 0x9eec,
 		0x9f00, 0x9f6c,
 		0x9f80, 0xa020,
 		0xd004, 0xd03c,
 		0xdfc0, 0xdfe0,
 		0xe000, 0x11088,
 		0x1109c, 0x11110,
 		0x11118, 0x1117c,
 		0x11190, 0x11204,
 		0x19040, 0x1906c,
 		0x19078, 0x19080,
 		0x1908c, 0x19124,
 		0x19150, 0x191b0,
 		0x191d0, 0x191e8,
 		0x19238, 0x19290,
 		0x193f8, 0x19474,
 		0x19490, 0x194cc,
 		0x194f0, 0x194f8,
 		0x19c00, 0x19c60,
 		0x19c94, 0x19e10,
 		0x19e50, 0x19f34,
 		0x19f40, 0x19f50,
 		0x19f90, 0x19fe4,
 		0x1a000, 0x1a06c,
 		0x1a0b0, 0x1a120,
 		0x1a128, 0x1a138,
 		0x1a190, 0x1a1c4,
 		0x1a1fc, 0x1a1fc,
 		0x1e008, 0x1e00c,
 		0x1e040, 0x1e04c,
 		0x1e284, 0x1e290,
 		0x1e2c0, 0x1e2c0,
 		0x1e2e0, 0x1e2e0,
 		0x1e300, 0x1e384,
 		0x1e3c0, 0x1e3c8,
 		0x1e408, 0x1e40c,
 		0x1e440, 0x1e44c,
 		0x1e684, 0x1e690,
 		0x1e6c0, 0x1e6c0,
 		0x1e6e0, 0x1e6e0,
 		0x1e700, 0x1e784,
 		0x1e7c0, 0x1e7c8,
 		0x1e808, 0x1e80c,
 		0x1e840, 0x1e84c,
 		0x1ea84, 0x1ea90,
 		0x1eac0, 0x1eac0,
 		0x1eae0, 0x1eae0,
 		0x1eb00, 0x1eb84,
 		0x1ebc0, 0x1ebc8,
 		0x1ec08, 0x1ec0c,
 		0x1ec40, 0x1ec4c,
 		0x1ee84, 0x1ee90,
 		0x1eec0, 0x1eec0,
 		0x1eee0, 0x1eee0,
 		0x1ef00, 0x1ef84,
 		0x1efc0, 0x1efc8,
 		0x1f008, 0x1f00c,
 		0x1f040, 0x1f04c,
 		0x1f284, 0x1f290,
 		0x1f2c0, 0x1f2c0,
 		0x1f2e0, 0x1f2e0,
 		0x1f300, 0x1f384,
 		0x1f3c0, 0x1f3c8,
 		0x1f408, 0x1f40c,
 		0x1f440, 0x1f44c,
 		0x1f684, 0x1f690,
 		0x1f6c0, 0x1f6c0,
 		0x1f6e0, 0x1f6e0,
 		0x1f700, 0x1f784,
 		0x1f7c0, 0x1f7c8,
 		0x1f808, 0x1f80c,
 		0x1f840, 0x1f84c,
 		0x1fa84, 0x1fa90,
 		0x1fac0, 0x1fac0,
 		0x1fae0, 0x1fae0,
 		0x1fb00, 0x1fb84,
 		0x1fbc0, 0x1fbc8,
 		0x1fc08, 0x1fc0c,
 		0x1fc40, 0x1fc4c,
 		0x1fe84, 0x1fe90,
 		0x1fec0, 0x1fec0,
 		0x1fee0, 0x1fee0,
 		0x1ff00, 0x1ff84,
 		0x1ffc0, 0x1ffc8,
 		0x30000, 0x30030,
 		0x30100, 0x30144,
 		0x30190, 0x301d0,
 		0x30200, 0x30318,
 		0x30400, 0x3052c,
 		0x30540, 0x3061c,
 		0x30800, 0x30834,
 		0x308c0, 0x30908,
 		0x30910, 0x309ac,
 		0x30a00, 0x30a2c,
 		0x30a44, 0x30a50,
 		0x30a74, 0x30c24,
 		0x30d00, 0x30d00,
 		0x30d08, 0x30d14,
 		0x30d1c, 0x30d20,
 		0x30d3c, 0x30d50,
 		0x31200, 0x3120c,
 		0x31220, 0x31220,
 		0x31240, 0x31240,
 		0x31600, 0x3160c,
 		0x31a00, 0x31a1c,
 		0x31e00, 0x31e20,
 		0x31e38, 0x31e3c,
 		0x31e80, 0x31e80,
 		0x31e88, 0x31ea8,
 		0x31eb0, 0x31eb4,
 		0x31ec8, 0x31ed4,
 		0x31fb8, 0x32004,
 		0x32200, 0x32200,
 		0x32208, 0x32240,
 		0x32248, 0x32280,
 		0x32288, 0x322c0,
 		0x322c8, 0x322fc,
 		0x32600, 0x32630,
 		0x32a00, 0x32abc,
 		0x32b00, 0x32b70,
 		0x33000, 0x33048,
 		0x33060, 0x3309c,
 		0x330f0, 0x33148,
 		0x33160, 0x3319c,
 		0x331f0, 0x332e4,
 		0x332f8, 0x333e4,
 		0x333f8, 0x33448,
 		0x33460, 0x3349c,
 		0x334f0, 0x33548,
 		0x33560, 0x3359c,
 		0x335f0, 0x336e4,
 		0x336f8, 0x337e4,
 		0x337f8, 0x337fc,
 		0x33814, 0x33814,
 		0x3382c, 0x3382c,
 		0x33880, 0x3388c,
 		0x338e8, 0x338ec,
 		0x33900, 0x33948,
 		0x33960, 0x3399c,
 		0x339f0, 0x33ae4,
 		0x33af8, 0x33b10,
 		0x33b28, 0x33b28,
 		0x33b3c, 0x33b50,
 		0x33bf0, 0x33c10,
 		0x33c28, 0x33c28,
 		0x33c3c, 0x33c50,
 		0x33cf0, 0x33cfc,
 		0x34000, 0x34030,
 		0x34100, 0x34144,
 		0x34190, 0x341d0,
 		0x34200, 0x34318,
 		0x34400, 0x3452c,
 		0x34540, 0x3461c,
 		0x34800, 0x34834,
 		0x348c0, 0x34908,
 		0x34910, 0x349ac,
 		0x34a00, 0x34a2c,
 		0x34a44, 0x34a50,
 		0x34a74, 0x34c24,
 		0x34d00, 0x34d00,
 		0x34d08, 0x34d14,
 		0x34d1c, 0x34d20,
 		0x34d3c, 0x34d50,
 		0x35200, 0x3520c,
 		0x35220, 0x35220,
 		0x35240, 0x35240,
 		0x35600, 0x3560c,
 		0x35a00, 0x35a1c,
 		0x35e00, 0x35e20,
 		0x35e38, 0x35e3c,
 		0x35e80, 0x35e80,
 		0x35e88, 0x35ea8,
 		0x35eb0, 0x35eb4,
 		0x35ec8, 0x35ed4,
 		0x35fb8, 0x36004,
 		0x36200, 0x36200,
 		0x36208, 0x36240,
 		0x36248, 0x36280,
 		0x36288, 0x362c0,
 		0x362c8, 0x362fc,
 		0x36600, 0x36630,
 		0x36a00, 0x36abc,
 		0x36b00, 0x36b70,
 		0x37000, 0x37048,
 		0x37060, 0x3709c,
 		0x370f0, 0x37148,
 		0x37160, 0x3719c,
 		0x371f0, 0x372e4,
 		0x372f8, 0x373e4,
 		0x373f8, 0x37448,
 		0x37460, 0x3749c,
 		0x374f0, 0x37548,
 		0x37560, 0x3759c,
 		0x375f0, 0x376e4,
 		0x376f8, 0x377e4,
 		0x377f8, 0x377fc,
 		0x37814, 0x37814,
 		0x3782c, 0x3782c,
 		0x37880, 0x3788c,
 		0x378e8, 0x378ec,
 		0x37900, 0x37948,
 		0x37960, 0x3799c,
 		0x379f0, 0x37ae4,
 		0x37af8, 0x37b10,
 		0x37b28, 0x37b28,
 		0x37b3c, 0x37b50,
 		0x37bf0, 0x37c10,
 		0x37c28, 0x37c28,
 		0x37c3c, 0x37c50,
 		0x37cf0, 0x37cfc,
 		0x38000, 0x38030,
 		0x38100, 0x38144,
 		0x38190, 0x381d0,
 		0x38200, 0x38318,
 		0x38400, 0x3852c,
 		0x38540, 0x3861c,
 		0x38800, 0x38834,
 		0x388c0, 0x38908,
 		0x38910, 0x389ac,
 		0x38a00, 0x38a2c,
 		0x38a44, 0x38a50,
 		0x38a74, 0x38c24,
 		0x38d00, 0x38d00,
 		0x38d08, 0x38d14,
 		0x38d1c, 0x38d20,
 		0x38d3c, 0x38d50,
 		0x39200, 0x3920c,
 		0x39220, 0x39220,
 		0x39240, 0x39240,
 		0x39600, 0x3960c,
 		0x39a00, 0x39a1c,
 		0x39e00, 0x39e20,
 		0x39e38, 0x39e3c,
 		0x39e80, 0x39e80,
 		0x39e88, 0x39ea8,
 		0x39eb0, 0x39eb4,
 		0x39ec8, 0x39ed4,
 		0x39fb8, 0x3a004,
 		0x3a200, 0x3a200,
 		0x3a208, 0x3a240,
 		0x3a248, 0x3a280,
 		0x3a288, 0x3a2c0,
 		0x3a2c8, 0x3a2fc,
 		0x3a600, 0x3a630,
 		0x3aa00, 0x3aabc,
 		0x3ab00, 0x3ab70,
 		0x3b000, 0x3b048,
 		0x3b060, 0x3b09c,
 		0x3b0f0, 0x3b148,
 		0x3b160, 0x3b19c,
 		0x3b1f0, 0x3b2e4,
 		0x3b2f8, 0x3b3e4,
 		0x3b3f8, 0x3b448,
 		0x3b460, 0x3b49c,
 		0x3b4f0, 0x3b548,
 		0x3b560, 0x3b59c,
 		0x3b5f0, 0x3b6e4,
 		0x3b6f8, 0x3b7e4,
 		0x3b7f8, 0x3b7fc,
 		0x3b814, 0x3b814,
 		0x3b82c, 0x3b82c,
 		0x3b880, 0x3b88c,
 		0x3b8e8, 0x3b8ec,
 		0x3b900, 0x3b948,
 		0x3b960, 0x3b99c,
 		0x3b9f0, 0x3bae4,
 		0x3baf8, 0x3bb10,
 		0x3bb28, 0x3bb28,
 		0x3bb3c, 0x3bb50,
 		0x3bbf0, 0x3bc10,
 		0x3bc28, 0x3bc28,
 		0x3bc3c, 0x3bc50,
 		0x3bcf0, 0x3bcfc,
 		0x3c000, 0x3c030,
 		0x3c100, 0x3c144,
 		0x3c190, 0x3c1d0,
 		0x3c200, 0x3c318,
 		0x3c400, 0x3c52c,
 		0x3c540, 0x3c61c,
 		0x3c800, 0x3c834,
 		0x3c8c0, 0x3c908,
 		0x3c910, 0x3c9ac,
 		0x3ca00, 0x3ca2c,
 		0x3ca44, 0x3ca50,
 		0x3ca74, 0x3cc24,
 		0x3cd00, 0x3cd00,
 		0x3cd08, 0x3cd14,
 		0x3cd1c, 0x3cd20,
 		0x3cd3c, 0x3cd50,
 		0x3d200, 0x3d20c,
 		0x3d220, 0x3d220,
 		0x3d240, 0x3d240,
 		0x3d600, 0x3d60c,
 		0x3da00, 0x3da1c,
 		0x3de00, 0x3de20,
 		0x3de38, 0x3de3c,
 		0x3de80, 0x3de80,
 		0x3de88, 0x3dea8,
 		0x3deb0, 0x3deb4,
 		0x3dec8, 0x3ded4,
 		0x3dfb8, 0x3e004,
 		0x3e200, 0x3e200,
 		0x3e208, 0x3e240,
 		0x3e248, 0x3e280,
 		0x3e288, 0x3e2c0,
 		0x3e2c8, 0x3e2fc,
 		0x3e600, 0x3e630,
 		0x3ea00, 0x3eabc,
 		0x3eb00, 0x3eb70,
 		0x3f000, 0x3f048,
 		0x3f060, 0x3f09c,
 		0x3f0f0, 0x3f148,
 		0x3f160, 0x3f19c,
 		0x3f1f0, 0x3f2e4,
 		0x3f2f8, 0x3f3e4,
 		0x3f3f8, 0x3f448,
 		0x3f460, 0x3f49c,
 		0x3f4f0, 0x3f548,
 		0x3f560, 0x3f59c,
 		0x3f5f0, 0x3f6e4,
 		0x3f6f8, 0x3f7e4,
 		0x3f7f8, 0x3f7fc,
 		0x3f814, 0x3f814,
 		0x3f82c, 0x3f82c,
 		0x3f880, 0x3f88c,
 		0x3f8e8, 0x3f8ec,
 		0x3f900, 0x3f948,
 		0x3f960, 0x3f99c,
 		0x3f9f0, 0x3fae4,
 		0x3faf8, 0x3fb10,
 		0x3fb28, 0x3fb28,
 		0x3fb3c, 0x3fb50,
 		0x3fbf0, 0x3fc10,
 		0x3fc28, 0x3fc28,
 		0x3fc3c, 0x3fc50,
 		0x3fcf0, 0x3fcfc,
 		0x40000, 0x4000c,
 		0x40040, 0x40068,
 		0x4007c, 0x40144,
 		0x40180, 0x4018c,
 		0x40200, 0x40298,
 		0x402ac, 0x4033c,
 		0x403f8, 0x403fc,
 		0x41304, 0x413c4,
 		0x41400, 0x4141c,
 		0x41480, 0x414d0,
 		0x44000, 0x44078,
 		0x440c0, 0x44278,
 		0x442c0, 0x44478,
 		0x444c0, 0x44678,
 		0x446c0, 0x44878,
 		0x448c0, 0x449fc,
 		0x45000, 0x45068,
 		0x45080, 0x45084,
 		0x450a0, 0x450b0,
 		0x45200, 0x45268,
 		0x45280, 0x45284,
 		0x452a0, 0x452b0,
 		0x460c0, 0x460e4,
 		0x47000, 0x4708c,
 		0x47200, 0x47250,
 		0x47400, 0x47420,
 		0x47600, 0x47618,
 		0x47800, 0x47814,
 		0x48000, 0x4800c,
 		0x48040, 0x48068,
 		0x4807c, 0x48144,
 		0x48180, 0x4818c,
 		0x48200, 0x48298,
 		0x482ac, 0x4833c,
 		0x483f8, 0x483fc,
 		0x49304, 0x493c4,
 		0x49400, 0x4941c,
 		0x49480, 0x494d0,
 		0x4c000, 0x4c078,
 		0x4c0c0, 0x4c278,
 		0x4c2c0, 0x4c478,
 		0x4c4c0, 0x4c678,
 		0x4c6c0, 0x4c878,
 		0x4c8c0, 0x4c9fc,
 		0x4d000, 0x4d068,
 		0x4d080, 0x4d084,
 		0x4d0a0, 0x4d0b0,
 		0x4d200, 0x4d268,
 		0x4d280, 0x4d284,
 		0x4d2a0, 0x4d2b0,
 		0x4e0c0, 0x4e0e4,
 		0x4f000, 0x4f08c,
 		0x4f200, 0x4f250,
 		0x4f400, 0x4f420,
 		0x4f600, 0x4f618,
 		0x4f800, 0x4f814,
 		0x50000, 0x500cc,
 		0x50400, 0x50400,
 		0x50800, 0x508cc,
 		0x50c00, 0x50c00,
 		0x51000, 0x5101c,
 		0x51300, 0x51308,
 	};
 
 	if (is_t4(sc)) {
 		reg_ranges = &t4_reg_ranges[0];
 		n = nitems(t4_reg_ranges);
 	} else {
 		reg_ranges = &t5_reg_ranges[0];
 		n = nitems(t5_reg_ranges);
 	}
 
 	regs->version = chip_id(sc) | chip_rev(sc) << 10;
 	for (i = 0; i < n; i += 2)
 		reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
 }
 
 static void
 cxgbe_tick(void *arg)
 {
 	struct port_info *pi = arg;
 	struct adapter *sc = pi->adapter;
 	struct ifnet *ifp = pi->ifp;
 	struct sge_txq *txq;
 	int i, drops;
 	struct port_stats *s = &pi->stats;
 
 	PORT_LOCK(pi);
 	if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 		PORT_UNLOCK(pi);
 		return;	/* without scheduling another callout */
 	}
 
 	t4_get_port_stats(sc, pi->tx_chan, s);
 
 	ifp->if_opackets = s->tx_frames - s->tx_pause;
 	ifp->if_ipackets = s->rx_frames - s->rx_pause;
 	ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
 	ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
 	ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
 	ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
 	ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
 	    s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
 	    s->rx_trunc3;
 	for (i = 0; i < 4; i++) {
 		if (pi->rx_chan_map & (1 << i)) {
 			uint32_t v;
 
 			/*
 			 * XXX: indirect reads from the same ADDR/DATA pair can
 			 * race with each other.
 			 */
 			t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
 			    1, A_TP_MIB_TNL_CNG_DROP_0 + i);
 			ifp->if_iqdrops += v;
 		}
 	}
 
 	drops = s->tx_drop;
 	for_each_txq(pi, i, txq)
 		drops += txq->br->br_drops;
 	ifp->if_snd.ifq_drops = drops;
 
 	ifp->if_oerrors = s->tx_error_frames;
 	ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
 	    s->rx_fcs_err + s->rx_len_err;
 
 	callout_schedule(&pi->tick, hz);
 	PORT_UNLOCK(pi);
 }
 
 static void
 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
 {
 	struct ifnet *vlan;
 
 	if (arg != ifp || ifp->if_type != IFT_ETHER)
 		return;
 
 	vlan = VLAN_DEVAT(ifp, vid);
 	VLAN_SETCOOKIE(vlan, ifp);
 }
 
 static int
 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 {
 
 #ifdef INVARIANTS
 	panic("%s: opcode 0x%02x on iq %p with payload %p",
 	    __func__, rss->opcode, iq, m);
 #else
 	log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
 	    __func__, rss->opcode, iq, m);
 	m_freem(m);
 #endif
 	return (EDOOFUS);
 }
 
 int
 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
 {
 	uintptr_t *loc, new;
 
 	if (opcode >= nitems(sc->cpl_handler))
 		return (EINVAL);
 
 	new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
 	loc = (uintptr_t *) &sc->cpl_handler[opcode];
 	atomic_store_rel_ptr(loc, new);
 
 	return (0);
 }
 
 static int
 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
 {
 
 #ifdef INVARIANTS
 	panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
 #else
 	log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
 	    __func__, iq, ctrl);
 #endif
 	return (EDOOFUS);
 }
 
 int
 t4_register_an_handler(struct adapter *sc, an_handler_t h)
 {
 	uintptr_t *loc, new;
 
 	new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
 	loc = (uintptr_t *) &sc->an_handler;
 	atomic_store_rel_ptr(loc, new);
 
 	return (0);
 }
 
 static int
 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
 {
 	const struct cpl_fw6_msg *cpl =
 	    __containerof(rpl, struct cpl_fw6_msg, data[0]);
 
 #ifdef INVARIANTS
 	panic("%s: fw_msg type %d", __func__, cpl->type);
 #else
 	log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
 #endif
 	return (EDOOFUS);
 }
 
 int
 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
 {
 	uintptr_t *loc, new;
 
 	if (type >= nitems(sc->fw_msg_handler))
 		return (EINVAL);
 
 	/*
 	 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
 	 * handler dispatch table.  Reject any attempt to install a handler for
 	 * this subtype.
 	 */
 	if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
 		return (EINVAL);
 
 	new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
 	loc = (uintptr_t *) &sc->fw_msg_handler[type];
 	atomic_store_rel_ptr(loc, new);
 
 	return (0);
 }
 
 static int
 t4_sysctls(struct adapter *sc)
 {
 	struct sysctl_ctx_list *ctx;
 	struct sysctl_oid *oid;
 	struct sysctl_oid_list *children, *c0;
 	static char *caps[] = {
 		"\20\1PPP\2QFC\3DCBX",			/* caps[0] linkcaps */
 		"\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"	/* caps[1] niccaps */
 		    "\6HASHFILTER\7ETHOFLD",
 		"\20\1TOE",				/* caps[2] toecaps */
 		"\20\1RDDP\2RDMAC",			/* caps[3] rdmacaps */
 		"\20\1INITIATOR_PDU\2TARGET_PDU"	/* caps[4] iscsicaps */
 		    "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
 		    "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
 		"\20\1INITIATOR\2TARGET\3CTRL_OFLD"	/* caps[5] fcoecaps */
 		    "\4PO_INITIAOR\5PO_TARGET"
 	};
 	static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
 
 	ctx = device_get_sysctl_ctx(sc->dev);
 
 	/*
 	 * dev.t4nex.X.
 	 */
 	oid = device_get_sysctl_tree(sc->dev);
 	c0 = children = SYSCTL_CHILDREN(oid);
 
 	sc->sc_do_rxcopy = 1;
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
 	    &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
 
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
 	    sc->params.nports, "# of ports");
 
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
 	    NULL, chip_rev(sc), "chip hardware revision");
 
 	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
 	    CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
 
 	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
 	    CTLFLAG_RD, &sc->cfg_file, 0, "configuration file");
 
 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
 	    sc->cfcsum, "config file checksum");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
 	    CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
 	    sysctl_bitfield, "A", "available doorbells");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
 	    CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
 	    sysctl_bitfield, "A", "available link capabilities");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
 	    CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
 	    sysctl_bitfield, "A", "available NIC capabilities");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
 	    CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
 	    sysctl_bitfield, "A", "available TCP offload capabilities");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
 	    CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
 	    sysctl_bitfield, "A", "available RDMA capabilities");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
 	    CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
 	    sysctl_bitfield, "A", "available iSCSI capabilities");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
 	    CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
 	    sysctl_bitfield, "A", "available FCoE capabilities");
 
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
 	    sc->params.vpd.cclk, "core clock frequency (in KHz)");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
 	    sizeof(sc->sge.timer_val), sysctl_int_array, "A",
 	    "interrupt holdoff timer values (us)");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
 	    sizeof(sc->sge.counter_val), sysctl_int_array, "A",
 	    "interrupt holdoff packet counter values");
 
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
 	    NULL, sc->tids.nftids, "number of filters");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
 	    CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
 	    "chip temperature (in Celsius)");
 
 	t4_sge_sysctls(sc, ctx, children);
 
 	sc->lro_timeout = 100;
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
 	    &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
 
 #ifdef SBUF_DRAIN
 	/*
 	 * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
 	 */
 	oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
 	    CTLFLAG_RD | CTLFLAG_SKIP, NULL,
 	    "logs and miscellaneous information");
 	children = SYSCTL_CHILDREN(oid);
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_cctrl, "A", "congestion control");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_cim_la, "A", "CIM logic analyzer");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
 
 	if (is_t5(sc)) {
 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
 		    CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
 		    sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
 
 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
 		    CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
 		    sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
 	}
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_cim_qcfg, "A", "CIM queue configuration");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_cpl_stats, "A", "CPL statistics");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_ddp_stats, "A", "DDP statistics");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_devlog, "A", "firmware's device log");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_fcoe_stats, "A", "FCoE statistics");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_hw_sched, "A", "hardware scheduler ");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_l2t, "A", "hardware L2 table");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_lb_stats, "A", "loopback statistics");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_meminfo, "A", "memory regions");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_mps_tcam, "A", "MPS TCAM entries");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_path_mtus, "A", "path MTUs");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_pm_stats, "A", "PM statistics");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_rdma_stats, "A", "RDMA statistics");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_tcp_stats, "A", "TCP statistics");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_tids, "A", "TID information");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_tp_err_stats, "A", "TP error statistics");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_tp_la, "A", "TP logic analyzer");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_tx_rate, "A", "Tx rate");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 	    sysctl_ulprx_la, "A", "ULPRX logic analyzer");
 
 	if (is_t5(sc)) {
 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
 		    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
 		    sysctl_wcwr_stats, "A", "write combined work requests");
 	}
 #endif
 
 #ifdef TCP_OFFLOAD
 	if (is_offload(sc)) {
 		/*
 		 * dev.t4nex.X.toe.
 		 */
 		oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
 		    NULL, "TOE parameters");
 		children = SYSCTL_CHILDREN(oid);
 
 		sc->tt.sndbuf = 256 * 1024;
 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
 		    &sc->tt.sndbuf, 0, "max hardware send buffer size");
 
 		sc->tt.ddp = 0;
 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
 		    &sc->tt.ddp, 0, "DDP allowed");
 
 		sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
 		    &sc->tt.indsz, 0, "DDP max indicate size allowed");
 
 		sc->tt.ddp_thres =
 		    G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
 		    &sc->tt.ddp_thres, 0, "DDP threshold");
 
 		sc->tt.rx_coalesce = 1;
 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
 		    CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
 	}
 #endif
 
 
 	return (0);
 }
 
 static int
 cxgbe_sysctls(struct port_info *pi)
 {
 	struct sysctl_ctx_list *ctx;
 	struct sysctl_oid *oid;
 	struct sysctl_oid_list *children;
 	struct adapter *sc = pi->adapter;
 
 	ctx = device_get_sysctl_ctx(pi->dev);
 
 	/*
 	 * dev.cxgbe.X.
 	 */
 	oid = device_get_sysctl_tree(pi->dev);
 	children = SYSCTL_CHILDREN(oid);
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
 	   CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
 	if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
 		    CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
 		    "PHY temperature (in Celsius)");
 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
 		    CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
 		    "PHY firmware version");
 	}
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
 	    &pi->nrxq, 0, "# of rx queues");
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
 	    &pi->ntxq, 0, "# of tx queues");
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
 	    &pi->first_rxq, 0, "index of first rx queue");
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
 	    &pi->first_txq, 0, "index of first tx queue");
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
 	    CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
 	    "Reserve queue 0 for non-flowid packets");
 
 #ifdef TCP_OFFLOAD
 	if (is_offload(sc)) {
 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
 		    &pi->nofldrxq, 0,
 		    "# of rx queues for offloaded TCP connections");
 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
 		    &pi->nofldtxq, 0,
 		    "# of tx queues for offloaded TCP connections");
 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
 		    CTLFLAG_RD, &pi->first_ofld_rxq, 0,
 		    "index of first TOE rx queue");
 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
 		    CTLFLAG_RD, &pi->first_ofld_txq, 0,
 		    "index of first TOE tx queue");
 	}
 #endif
+#ifdef DEV_NETMAP
+	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
+	    &pi->nnmrxq, 0, "# of rx queues for netmap");
+	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
+	    &pi->nnmtxq, 0, "# of tx queues for netmap");
+	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
+	    CTLFLAG_RD, &pi->first_nm_rxq, 0,
+	    "index of first netmap rx queue");
+	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
+	    CTLFLAG_RD, &pi->first_nm_txq, 0,
+	    "index of first netmap tx queue");
+#endif
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
 	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
 	    "holdoff timer index");
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
 	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
 	    "holdoff packet counter index");
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
 	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
 	    "rx queue size");
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
 	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
 	    "tx queue size");
 
 	/*
 	 * dev.cxgbe.X.stats.
 	 */
 	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
 	    NULL, "port statistics");
 	children = SYSCTL_CHILDREN(oid);
 
 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
 	SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
 	    CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
 	    sysctl_handle_t4_reg64, "QU", desc)
 
 	SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
 	    "# of tx frames in this range",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
 	    "# of tx frames in this range",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
 	    "# of tx frames in this range",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
 	    "# of tx frames in this range",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
 	    "# of tx frames in this range",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
 	    "# of tx frames in this range",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
 	    "# of tx frames in this range",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
 
 	SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
 	    "# of frames received with bad FCS",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
 	    "# of frames received with length error",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
 	    "# of rx frames in this range",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
 	    "# of rx frames in this range",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
 	    "# of rx frames in this range",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
 	    "# of rx frames in this range",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
 	    "# of rx frames in this range",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
 	    "# of rx frames in this range",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
 	    "# of rx frames in this range",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
 
 #undef SYSCTL_ADD_T4_REG64
 
 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
 	    &pi->stats.name, desc)
 
 	/* We get these from port_stats and they may be stale by upto 1s */
 	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
 	    "# drops due to buffer-group 0 overflows");
 	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
 	    "# drops due to buffer-group 1 overflows");
 	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
 	    "# drops due to buffer-group 2 overflows");
 	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
 	    "# drops due to buffer-group 3 overflows");
 	SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
 	    "# of buffer-group 0 truncated packets");
 	SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
 	    "# of buffer-group 1 truncated packets");
 	SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
 	    "# of buffer-group 2 truncated packets");
 	SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
 	    "# of buffer-group 3 truncated packets");
 
 #undef SYSCTL_ADD_T4_PORTSTAT
 
 	return (0);
 }
 
 static int
 sysctl_int_array(SYSCTL_HANDLER_ARGS)
 {
 	int rc, *i;
 	struct sbuf sb;
 
 	sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
 	for (i = arg1; arg2; arg2 -= sizeof(int), i++)
 		sbuf_printf(&sb, "%d ", *i);
 	sbuf_trim(&sb);
 	sbuf_finish(&sb);
 	rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
 	sbuf_delete(&sb);
 	return (rc);
 }
 
 static int
 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
 {
 	int rc;
 	struct sbuf *sb;
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return(rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 static int
 sysctl_btphy(SYSCTL_HANDLER_ARGS)
 {
 	struct port_info *pi = arg1;
 	int op = arg2;
 	struct adapter *sc = pi->adapter;
 	u_int v;
 	int rc;
 
 	rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
 	if (rc)
 		return (rc);
 	/* XXX: magic numbers */
 	rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
 	    &v);
 	end_synchronized_op(sc, 0);
 	if (rc)
 		return (rc);
 	if (op == 0)
 		v /= 256;
 
 	rc = sysctl_handle_int(oidp, &v, 0, req);
 	return (rc);
 }
 
 static int
 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
 {
 	struct port_info *pi = arg1;
 	int rc, val;
 
 	val = pi->rsrv_noflowq;
 	rc = sysctl_handle_int(oidp, &val, 0, req);
 	if (rc != 0 || req->newptr == NULL)
 		return (rc);
 
 	if ((val >= 1) && (pi->ntxq > 1))
 		pi->rsrv_noflowq = 1;
 	else
 		pi->rsrv_noflowq = 0;
 
 	return (rc);
 }
 
 static int
 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
 {
 	struct port_info *pi = arg1;
 	struct adapter *sc = pi->adapter;
 	int idx, rc, i;
 	struct sge_rxq *rxq;
 #ifdef TCP_OFFLOAD
 	struct sge_ofld_rxq *ofld_rxq;
 #endif
 	uint8_t v;
 
 	idx = pi->tmr_idx;
 
 	rc = sysctl_handle_int(oidp, &idx, 0, req);
 	if (rc != 0 || req->newptr == NULL)
 		return (rc);
 
 	if (idx < 0 || idx >= SGE_NTIMERS)
 		return (EINVAL);
 
 	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
 	    "t4tmr");
 	if (rc)
 		return (rc);
 
 	v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
 	for_each_rxq(pi, i, rxq) {
 #ifdef atomic_store_rel_8
 		atomic_store_rel_8(&rxq->iq.intr_params, v);
 #else
 		rxq->iq.intr_params = v;
 #endif
 	}
 #ifdef TCP_OFFLOAD
 	for_each_ofld_rxq(pi, i, ofld_rxq) {
 #ifdef atomic_store_rel_8
 		atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
 #else
 		ofld_rxq->iq.intr_params = v;
 #endif
 	}
 #endif
 	pi->tmr_idx = idx;
 
 	end_synchronized_op(sc, LOCK_HELD);
 	return (0);
 }
 
 static int
 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
 {
 	struct port_info *pi = arg1;
 	struct adapter *sc = pi->adapter;
 	int idx, rc;
 
 	idx = pi->pktc_idx;
 
 	rc = sysctl_handle_int(oidp, &idx, 0, req);
 	if (rc != 0 || req->newptr == NULL)
 		return (rc);
 
 	if (idx < -1 || idx >= SGE_NCOUNTERS)
 		return (EINVAL);
 
 	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
 	    "t4pktc");
 	if (rc)
 		return (rc);
 
 	if (pi->flags & PORT_INIT_DONE)
 		rc = EBUSY; /* cannot be changed once the queues are created */
 	else
 		pi->pktc_idx = idx;
 
 	end_synchronized_op(sc, LOCK_HELD);
 	return (rc);
 }
 
 static int
 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
 {
 	struct port_info *pi = arg1;
 	struct adapter *sc = pi->adapter;
 	int qsize, rc;
 
 	qsize = pi->qsize_rxq;
 
 	rc = sysctl_handle_int(oidp, &qsize, 0, req);
 	if (rc != 0 || req->newptr == NULL)
 		return (rc);
 
 	if (qsize < 128 || (qsize & 7))
 		return (EINVAL);
 
 	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
 	    "t4rxqs");
 	if (rc)
 		return (rc);
 
 	if (pi->flags & PORT_INIT_DONE)
 		rc = EBUSY; /* cannot be changed once the queues are created */
 	else
 		pi->qsize_rxq = qsize;
 
 	end_synchronized_op(sc, LOCK_HELD);
 	return (rc);
 }
 
 static int
 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
 {
 	struct port_info *pi = arg1;
 	struct adapter *sc = pi->adapter;
 	int qsize, rc;
 
 	qsize = pi->qsize_txq;
 
 	rc = sysctl_handle_int(oidp, &qsize, 0, req);
 	if (rc != 0 || req->newptr == NULL)
 		return (rc);
 
 	/* bufring size must be powerof2 */
 	if (qsize < 128 || !powerof2(qsize))
 		return (EINVAL);
 
 	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
 	    "t4txqs");
 	if (rc)
 		return (rc);
 
 	if (pi->flags & PORT_INIT_DONE)
 		rc = EBUSY; /* cannot be changed once the queues are created */
 	else
 		pi->qsize_txq = qsize;
 
 	end_synchronized_op(sc, LOCK_HELD);
 	return (rc);
 }
 
 static int
 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	int reg = arg2;
 	uint64_t val;
 
 	val = t4_read_reg64(sc, reg);
 
 	return (sysctl_handle_64(oidp, &val, 0, req));
 }
 
 static int
 sysctl_temperature(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	int rc, t;
 	uint32_t param, val;
 
 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
 	if (rc)
 		return (rc);
 	param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
 	    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
 	    V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
 	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
 	end_synchronized_op(sc, 0);
 	if (rc)
 		return (rc);
 
 	/* unknown is returned as 0 but we display -1 in that case */
 	t = val == 0 ? -1 : val;
 
 	rc = sysctl_handle_int(oidp, &t, 0, req);
 	return (rc);
 }
 
 #ifdef SBUF_DRAIN
 static int
 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc, i;
 	uint16_t incr[NMTUS][NCCTRL_WIN];
 	static const char *dec_fac[] = {
 		"0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
 		"0.9375"
 	};
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	t4_read_cong_tbl(sc, incr);
 
 	for (i = 0; i < NCCTRL_WIN; ++i) {
 		sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
 		    incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
 		    incr[5][i], incr[6][i], incr[7][i]);
 		sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
 		    incr[8][i], incr[9][i], incr[10][i], incr[11][i],
 		    incr[12][i], incr[13][i], incr[14][i], incr[15][i],
 		    sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
 	}
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
 	"TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",	/* ibq's */
 	"ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI",	/* obq's */
 	"SGE0-RX", "SGE1-RX"	/* additional obq's (T5 onwards) */
 };
 
 static int
 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc, i, n, qid = arg2;
 	uint32_t *buf, *p;
 	char *qtype;
 	u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
 
 	KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
 	    ("%s: bad qid %d\n", __func__, qid));
 
 	if (qid < CIM_NUM_IBQ) {
 		/* inbound queue */
 		qtype = "IBQ";
 		n = 4 * CIM_IBQ_SIZE;
 		buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
 		rc = t4_read_cim_ibq(sc, qid, buf, n);
 	} else {
 		/* outbound queue */
 		qtype = "OBQ";
 		qid -= CIM_NUM_IBQ;
 		n = 4 * cim_num_obq * CIM_OBQ_SIZE;
 		buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
 		rc = t4_read_cim_obq(sc, qid, buf, n);
 	}
 
 	if (rc < 0) {
 		rc = -rc;
 		goto done;
 	}
 	n = rc * sizeof(uint32_t);	/* rc has # of words actually read */
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		goto done;
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
 	if (sb == NULL) {
 		rc = ENOMEM;
 		goto done;
 	}
 
 	sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
 	for (i = 0, p = buf; i < n; i += 16, p += 4)
 		sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
 		    p[2], p[3]);
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 done:
 	free(buf, M_CXGBE);
 	return (rc);
 }
 
 static int
 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	u_int cfg;
 	struct sbuf *sb;
 	uint32_t *buf, *p;
 	int rc;
 
 	rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
 	if (rc != 0)
 		return (rc);
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
 	    M_ZERO | M_WAITOK);
 
 	rc = -t4_cim_read_la(sc, buf, NULL);
 	if (rc != 0)
 		goto done;
 
 	sbuf_printf(sb, "Status   Data      PC%s",
 	    cfg & F_UPDBGLACAPTPCONLY ? "" :
 	    "     LS0Stat  LS0Addr             LS0Data");
 
 	KASSERT((sc->params.cim_la_size & 7) == 0,
 	    ("%s: p will walk off the end of buf", __func__));
 
 	for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
 		if (cfg & F_UPDBGLACAPTPCONLY) {
 			sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
 			    p[6], p[7]);
 			sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
 			    (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
 			    p[4] & 0xff, p[5] >> 8);
 			sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
 			    (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
 			    p[1] & 0xf, p[2] >> 4);
 		} else {
 			sbuf_printf(sb,
 			    "\n  %02x   %x%07x %x%07x %08x %08x "
 			    "%08x%08x%08x%08x",
 			    (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
 			    p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
 			    p[6], p[7]);
 		}
 	}
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 done:
 	free(buf, M_CXGBE);
 	return (rc);
 }
 
 static int
 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	u_int i;
 	struct sbuf *sb;
 	uint32_t *buf, *p;
 	int rc;
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
 	    M_ZERO | M_WAITOK);
 
 	t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
 	p = buf;
 
 	for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
 		sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
 		    p[1], p[0]);
 	}
 
 	sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
 	for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
 		sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
 		    (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
 		    (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
 		    (p[1] >> 2) | ((p[2] & 3) << 30),
 		    (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
 		    p[0] & 1);
 	}
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 	free(buf, M_CXGBE);
 	return (rc);
 }
 
 static int
 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	u_int i;
 	struct sbuf *sb;
 	uint32_t *buf, *p;
 	int rc;
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
 	    M_ZERO | M_WAITOK);
 
 	t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
 	p = buf;
 
 	sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
 	for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
 		sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
 		    (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
 		    p[4], p[3], p[2], p[1], p[0]);
 	}
 
 	sbuf_printf(sb, "\n\nCntl ID               Data");
 	for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
 		sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
 		    (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
 	}
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 	free(buf, M_CXGBE);
 	return (rc);
 }
 
 static int
 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc, i;
 	uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
 	uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
 	uint16_t thres[CIM_NUM_IBQ];
 	uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
 	uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
 	u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
 
 	if (is_t4(sc)) {
 		cim_num_obq = CIM_NUM_OBQ;
 		ibq_rdaddr = A_UP_IBQ_0_RDADDR;
 		obq_rdaddr = A_UP_OBQ_0_REALADDR;
 	} else {
 		cim_num_obq = CIM_NUM_OBQ_T5;
 		ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
 		obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
 	}
 	nq = CIM_NUM_IBQ + cim_num_obq;
 
 	rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
 	if (rc == 0)
 		rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
 	if (rc != 0)
 		return (rc);
 
 	t4_read_cimq_cfg(sc, base, size, thres);
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
 
 	for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
 		sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
 		    qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
 		    G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
 		    G_QUEREMFLITS(p[2]) * 16);
 	for ( ; i < nq; i++, p += 4, wr += 2)
 		sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
 		    base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
 		    wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
 		    G_QUEREMFLITS(p[2]) * 16);
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 static int
 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc;
 	struct tp_cpl_stats stats;
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	t4_tp_get_cpl_stats(sc, &stats);
 
 	sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
 	    "channel 3\n");
 	sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
 		   stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
 	sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
 		   stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 static int
 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc;
 	struct tp_usm_stats stats;
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return(rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	t4_get_usm_stats(sc, &stats);
 
 	sbuf_printf(sb, "Frames: %u\n", stats.frames);
 	sbuf_printf(sb, "Octets: %ju\n", stats.octets);
 	sbuf_printf(sb, "Drops:  %u", stats.drops);
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 const char *devlog_level_strings[] = {
 	[FW_DEVLOG_LEVEL_EMERG]		= "EMERG",
 	[FW_DEVLOG_LEVEL_CRIT]		= "CRIT",
 	[FW_DEVLOG_LEVEL_ERR]		= "ERR",
 	[FW_DEVLOG_LEVEL_NOTICE]	= "NOTICE",
 	[FW_DEVLOG_LEVEL_INFO]		= "INFO",
 	[FW_DEVLOG_LEVEL_DEBUG]		= "DEBUG"
 };
 
 const char *devlog_facility_strings[] = {
 	[FW_DEVLOG_FACILITY_CORE]	= "CORE",
 	[FW_DEVLOG_FACILITY_CF]		= "CF",
 	[FW_DEVLOG_FACILITY_SCHED]	= "SCHED",
 	[FW_DEVLOG_FACILITY_TIMER]	= "TIMER",
 	[FW_DEVLOG_FACILITY_RES]	= "RES",
 	[FW_DEVLOG_FACILITY_HW]		= "HW",
 	[FW_DEVLOG_FACILITY_FLR]	= "FLR",
 	[FW_DEVLOG_FACILITY_DMAQ]	= "DMAQ",
 	[FW_DEVLOG_FACILITY_PHY]	= "PHY",
 	[FW_DEVLOG_FACILITY_MAC]	= "MAC",
 	[FW_DEVLOG_FACILITY_PORT]	= "PORT",
 	[FW_DEVLOG_FACILITY_VI]		= "VI",
 	[FW_DEVLOG_FACILITY_FILTER]	= "FILTER",
 	[FW_DEVLOG_FACILITY_ACL]	= "ACL",
 	[FW_DEVLOG_FACILITY_TM]		= "TM",
 	[FW_DEVLOG_FACILITY_QFC]	= "QFC",
 	[FW_DEVLOG_FACILITY_DCB]	= "DCB",
 	[FW_DEVLOG_FACILITY_ETH]	= "ETH",
 	[FW_DEVLOG_FACILITY_OFLD]	= "OFLD",
 	[FW_DEVLOG_FACILITY_RI]		= "RI",
 	[FW_DEVLOG_FACILITY_ISCSI]	= "ISCSI",
 	[FW_DEVLOG_FACILITY_FCOE]	= "FCOE",
 	[FW_DEVLOG_FACILITY_FOISCSI]	= "FOISCSI",
 	[FW_DEVLOG_FACILITY_FOFCOE]	= "FOFCOE"
 };
 
 static int
 sysctl_devlog(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct devlog_params *dparams = &sc->params.devlog;
 	struct fw_devlog_e *buf, *e;
 	int i, j, rc, nentries, first = 0, m;
 	struct sbuf *sb;
 	uint64_t ftstamp = UINT64_MAX;
 
 	if (dparams->start == 0) {
 		dparams->memtype = FW_MEMTYPE_EDC0;
 		dparams->start = 0x84000;
 		dparams->size = 32768;
 	}
 
 	nentries = dparams->size / sizeof(struct fw_devlog_e);
 
 	buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
 	if (buf == NULL)
 		return (ENOMEM);
 
 	m = fwmtype_to_hwmtype(dparams->memtype);
 	rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
 	if (rc != 0)
 		goto done;
 
 	for (i = 0; i < nentries; i++) {
 		e = &buf[i];
 
 		if (e->timestamp == 0)
 			break;	/* end */
 
 		e->timestamp = be64toh(e->timestamp);
 		e->seqno = be32toh(e->seqno);
 		for (j = 0; j < 8; j++)
 			e->params[j] = be32toh(e->params[j]);
 
 		if (e->timestamp < ftstamp) {
 			ftstamp = e->timestamp;
 			first = i;
 		}
 	}
 
 	if (buf[first].timestamp == 0)
 		goto done;	/* nothing in the log */
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		goto done;
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
 	if (sb == NULL) {
 		rc = ENOMEM;
 		goto done;
 	}
 	sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
 	    "Seq#", "Tstamp", "Level", "Facility", "Message");
 
 	i = first;
 	do {
 		e = &buf[i];
 		if (e->timestamp == 0)
 			break;	/* end */
 
 		sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
 		    e->seqno, e->timestamp,
 		    (e->level < nitems(devlog_level_strings) ?
 			devlog_level_strings[e->level] : "UNKNOWN"),
 		    (e->facility < nitems(devlog_facility_strings) ?
 			devlog_facility_strings[e->facility] : "UNKNOWN"));
 		sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
 		    e->params[2], e->params[3], e->params[4],
 		    e->params[5], e->params[6], e->params[7]);
 
 		if (++i == nentries)
 			i = 0;
 	} while (i != first);
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 done:
 	free(buf, M_CXGBE);
 	return (rc);
 }
 
 static int
 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc;
 	struct tp_fcoe_stats stats[4];
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	t4_get_fcoe_stats(sc, 0, &stats[0]);
 	t4_get_fcoe_stats(sc, 1, &stats[1]);
 	t4_get_fcoe_stats(sc, 2, &stats[2]);
 	t4_get_fcoe_stats(sc, 3, &stats[3]);
 
 	sbuf_printf(sb, "                   channel 0        channel 1        "
 	    "channel 2        channel 3\n");
 	sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
 	    stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
 	    stats[3].octetsDDP);
 	sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
 	    stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
 	sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
 	    stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
 	    stats[3].framesDrop);
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 static int
 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc, i;
 	unsigned int map, kbps, ipg, mode;
 	unsigned int pace_tab[NTX_SCHED];
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
 	mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
 	t4_read_pace_tbl(sc, pace_tab);
 
 	sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
 	    "Class IPG (0.1 ns)   Flow IPG (us)");
 
 	for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
 		t4_get_tx_sched(sc, i, &kbps, &ipg);
 		sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
 		    (mode & (1 << i)) ? "flow" : "class", map & 3);
 		if (kbps)
 			sbuf_printf(sb, "%9u     ", kbps);
 		else
 			sbuf_printf(sb, " disabled     ");
 
 		if (ipg)
 			sbuf_printf(sb, "%13u        ", ipg);
 		else
 			sbuf_printf(sb, "     disabled        ");
 
 		if (pace_tab[i])
 			sbuf_printf(sb, "%10u", pace_tab[i]);
 		else
 			sbuf_printf(sb, "  disabled");
 	}
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 static int
 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc, i, j;
 	uint64_t *p0, *p1;
 	struct lb_port_stats s[2];
 	static const char *stat_name[] = {
 		"OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
 		"UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
 		"Frames128To255:", "Frames256To511:", "Frames512To1023:",
 		"Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
 		"BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
 		"BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
 		"BG2FramesTrunc:", "BG3FramesTrunc:"
 	};
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	memset(s, 0, sizeof(s));
 
 	for (i = 0; i < 4; i += 2) {
 		t4_get_lb_stats(sc, i, &s[0]);
 		t4_get_lb_stats(sc, i + 1, &s[1]);
 
 		p0 = &s[0].octets;
 		p1 = &s[1].octets;
 		sbuf_printf(sb, "%s                       Loopback %u"
 		    "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
 
 		for (j = 0; j < nitems(stat_name); j++)
 			sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
 				   *p0++, *p1++);
 	}
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 static int
 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
 {
 	int rc = 0;
 	struct port_info *pi = arg1;
 	struct sbuf *sb;
 	static const char *linkdnreasons[] = {
 		"non-specific", "remote fault", "autoneg failed", "reserved3",
 		"PHY overheated", "unknown", "rx los", "reserved7"
 	};
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return(rc);
 	sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	if (pi->linkdnrc < 0)
 		sbuf_printf(sb, "n/a");
 	else if (pi->linkdnrc < nitems(linkdnreasons))
 		sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
 	else
 		sbuf_printf(sb, "%d", pi->linkdnrc);
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 struct mem_desc {
 	unsigned int base;
 	unsigned int limit;
 	unsigned int idx;
 };
 
 static int
 mem_desc_cmp(const void *a, const void *b)
 {
 	return ((const struct mem_desc *)a)->base -
 	       ((const struct mem_desc *)b)->base;
 }
 
 static void
 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
     unsigned int to)
 {
 	unsigned int size;
 
 	size = to - from + 1;
 	if (size == 0)
 		return;
 
 	/* XXX: need humanize_number(3) in libkern for a more readable 'size' */
 	sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
 }
 
 static int
 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc, i, n;
 	uint32_t lo, hi, used, alloc;
 	static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
 	static const char *region[] = {
 		"DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
 		"Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
 		"Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
 		"TDDP region:", "TPT region:", "STAG region:", "RQ region:",
 		"RQUDP region:", "PBL region:", "TXPBL region:",
 		"DBVFIFO region:", "ULPRX state:", "ULPTX state:",
 		"On-chip queues:"
 	};
 	struct mem_desc avail[4];
 	struct mem_desc mem[nitems(region) + 3];	/* up to 3 holes */
 	struct mem_desc *md = mem;
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	for (i = 0; i < nitems(mem); i++) {
 		mem[i].limit = 0;
 		mem[i].idx = i;
 	}
 
 	/* Find and sort the populated memory ranges */
 	i = 0;
 	lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
 	if (lo & F_EDRAM0_ENABLE) {
 		hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
 		avail[i].base = G_EDRAM0_BASE(hi) << 20;
 		avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
 		avail[i].idx = 0;
 		i++;
 	}
 	if (lo & F_EDRAM1_ENABLE) {
 		hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
 		avail[i].base = G_EDRAM1_BASE(hi) << 20;
 		avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
 		avail[i].idx = 1;
 		i++;
 	}
 	if (lo & F_EXT_MEM_ENABLE) {
 		hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
 		avail[i].base = G_EXT_MEM_BASE(hi) << 20;
 		avail[i].limit = avail[i].base +
 		    (G_EXT_MEM_SIZE(hi) << 20);
 		avail[i].idx = is_t4(sc) ? 2 : 3;	/* Call it MC for T4 */
 		i++;
 	}
 	if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
 		hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
 		avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
 		avail[i].limit = avail[i].base +
 		    (G_EXT_MEM1_SIZE(hi) << 20);
 		avail[i].idx = 4;
 		i++;
 	}
 	if (!i)                                    /* no memory available */
 		return 0;
 	qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
 
 	(md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
 	(md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
 	(md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
 	(md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
 	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
 	(md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
 	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
 	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
 	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
 
 	/* the next few have explicit upper bounds */
 	md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
 	md->limit = md->base - 1 +
 		    t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
 		    G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
 	md++;
 
 	md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
 	md->limit = md->base - 1 +
 		    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
 		    G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
 	md++;
 
 	if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
 		hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
 		md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
 		md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
 	} else {
 		md->base = 0;
 		md->idx = nitems(region);  /* hide it */
 	}
 	md++;
 
 #define ulp_region(reg) \
 	md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
 	(md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
 
 	ulp_region(RX_ISCSI);
 	ulp_region(RX_TDDP);
 	ulp_region(TX_TPT);
 	ulp_region(RX_STAG);
 	ulp_region(RX_RQ);
 	ulp_region(RX_RQUDP);
 	ulp_region(RX_PBL);
 	ulp_region(TX_PBL);
 #undef ulp_region
 
 	md->base = 0;
 	md->idx = nitems(region);
 	if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
 		md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
 		md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
 		    A_SGE_DBVFIFO_SIZE))) << 2) - 1;
 	}
 	md++;
 
 	md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
 	md->limit = md->base + sc->tids.ntids - 1;
 	md++;
 	md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
 	md->limit = md->base + sc->tids.ntids - 1;
 	md++;
 
 	md->base = sc->vres.ocq.start;
 	if (sc->vres.ocq.size)
 		md->limit = md->base + sc->vres.ocq.size - 1;
 	else
 		md->idx = nitems(region);  /* hide it */
 	md++;
 
 	/* add any address-space holes, there can be up to 3 */
 	for (n = 0; n < i - 1; n++)
 		if (avail[n].limit < avail[n + 1].base)
 			(md++)->base = avail[n].limit;
 	if (avail[n].limit)
 		(md++)->base = avail[n].limit;
 
 	n = md - mem;
 	qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
 
 	for (lo = 0; lo < i; lo++)
 		mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
 				avail[lo].limit - 1);
 
 	sbuf_printf(sb, "\n");
 	for (i = 0; i < n; i++) {
 		if (mem[i].idx >= nitems(region))
 			continue;                        /* skip holes */
 		if (!mem[i].limit)
 			mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
 		mem_region_show(sb, region[mem[i].idx], mem[i].base,
 				mem[i].limit);
 	}
 
 	sbuf_printf(sb, "\n");
 	lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
 	hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
 	mem_region_show(sb, "uP RAM:", lo, hi);
 
 	lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
 	hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
 	mem_region_show(sb, "uP Extmem2:", lo, hi);
 
 	lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
 	sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
 		   G_PMRXMAXPAGE(lo),
 		   t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
 		   (lo & F_PMRXNUMCHN) ? 2 : 1);
 
 	lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
 	hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
 	sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
 		   G_PMTXMAXPAGE(lo),
 		   hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
 		   hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
 	sbuf_printf(sb, "%u p-structs\n",
 		   t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
 
 	for (i = 0; i < 4; i++) {
 		lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
 		if (is_t4(sc)) {
 			used = G_USED(lo);
 			alloc = G_ALLOC(lo);
 		} else {
 			used = G_T5_USED(lo);
 			alloc = G_T5_ALLOC(lo);
 		}
 		sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
 			   i, used, alloc);
 	}
 	for (i = 0; i < 4; i++) {
 		lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
 		if (is_t4(sc)) {
 			used = G_USED(lo);
 			alloc = G_ALLOC(lo);
 		} else {
 			used = G_T5_USED(lo);
 			alloc = G_T5_ALLOC(lo);
 		}
 		sbuf_printf(sb,
 			   "\nLoopback %d using %u pages out of %u allocated",
 			   i, used, alloc);
 	}
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 static inline void
 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
 {
 	*mask = x | y;
 	y = htobe64(y);
 	memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
 }
 
 static int
 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc, i, n;
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	sbuf_printf(sb,
 	    "Idx  Ethernet address     Mask     Vld Ports PF"
 	    "  VF              Replication             P0 P1 P2 P3  ML");
 	n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
 	    NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
 	for (i = 0; i < n; i++) {
 		uint64_t tcamx, tcamy, mask;
 		uint32_t cls_lo, cls_hi;
 		uint8_t addr[ETHER_ADDR_LEN];
 
 		tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
 		tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
 		cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
 		cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
 
 		if (tcamx & tcamy)
 			continue;
 
 		tcamxy2valmask(tcamx, tcamy, addr, &mask);
 		sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
 			   "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
 			   addr[3], addr[4], addr[5], (uintmax_t)mask,
 			   (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
 			   G_PORTMAP(cls_hi), G_PF(cls_lo),
 			   (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
 
 		if (cls_lo & F_REPLICATE) {
 			struct fw_ldst_cmd ldst_cmd;
 
 			memset(&ldst_cmd, 0, sizeof(ldst_cmd));
 			ldst_cmd.op_to_addrspace =
 			    htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
 				F_FW_CMD_REQUEST | F_FW_CMD_READ |
 				V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
 			ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
 			ldst_cmd.u.mps.fid_ctl =
 			    htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
 				V_FW_LDST_CMD_CTL(i));
 
 			rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
 			    "t4mps");
 			if (rc)
 				break;
 			rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
 			    sizeof(ldst_cmd), &ldst_cmd);
 			end_synchronized_op(sc, 0);
 
 			if (rc != 0) {
 				sbuf_printf(sb,
 				    " ------------ error %3u ------------", rc);
 				rc = 0;
 			} else {
 				sbuf_printf(sb, " %08x %08x %08x %08x",
 				    be32toh(ldst_cmd.u.mps.rplc127_96),
 				    be32toh(ldst_cmd.u.mps.rplc95_64),
 				    be32toh(ldst_cmd.u.mps.rplc63_32),
 				    be32toh(ldst_cmd.u.mps.rplc31_0));
 			}
 		} else
 			sbuf_printf(sb, "%36s", "");
 
 		sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
 		    G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
 		    G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
 	}
 
 	if (rc)
 		(void) sbuf_finish(sb);
 	else
 		rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 static int
 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc;
 	uint16_t mtus[NMTUS];
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	t4_read_mtu_tbl(sc, mtus, NULL);
 
 	sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
 	    mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
 	    mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
 	    mtus[14], mtus[15]);
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 static int
 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc, i;
 	uint32_t cnt[PM_NSTATS];
 	uint64_t cyc[PM_NSTATS];
 	static const char *rx_stats[] = {
 		"Read:", "Write bypass:", "Write mem:", "Flush:"
 	};
 	static const char *tx_stats[] = {
 		"Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
 	};
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	t4_pmtx_get_stats(sc, cnt, cyc);
 	sbuf_printf(sb, "                Tx pcmds             Tx bytes");
 	for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
 		sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
 		    cyc[i]);
 
 	t4_pmrx_get_stats(sc, cnt, cyc);
 	sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
 	for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
 		sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
 		    cyc[i]);
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 static int
 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc;
 	struct tp_rdma_stats stats;
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	t4_tp_get_rdma_stats(sc, &stats);
 	sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
 	sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 static int
 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc;
 	struct tp_tcp_stats v4, v6;
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	t4_tp_get_tcp_stats(sc, &v4, &v6);
 	sbuf_printf(sb,
 	    "                                IP                 IPv6\n");
 	sbuf_printf(sb, "OutRsts:      %20u %20u\n",
 	    v4.tcpOutRsts, v6.tcpOutRsts);
 	sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
 	    v4.tcpInSegs, v6.tcpInSegs);
 	sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
 	    v4.tcpOutSegs, v6.tcpOutSegs);
 	sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
 	    v4.tcpRetransSegs, v6.tcpRetransSegs);
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 static int
 sysctl_tids(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc;
 	struct tid_info *t = &sc->tids;
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	if (t->natids) {
 		sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
 		    t->atids_in_use);
 	}
 
 	if (t->ntids) {
 		if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
 			uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
 
 			if (b) {
 				sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
 				    t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
 				    t->ntids - 1);
 			} else {
 				sbuf_printf(sb, "TID range: %u-%u",
 				    t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
 				    t->ntids - 1);
 			}
 		} else
 			sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
 		sbuf_printf(sb, ", in use: %u\n",
 		    atomic_load_acq_int(&t->tids_in_use));
 	}
 
 	if (t->nstids) {
 		sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
 		    t->stid_base + t->nstids - 1, t->stids_in_use);
 	}
 
 	if (t->nftids) {
 		sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
 		    t->ftid_base + t->nftids - 1);
 	}
 
 	if (t->netids) {
 		sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
 		    t->etid_base + t->netids - 1);
 	}
 
 	sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
 	    t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
 	    t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 static int
 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc;
 	struct tp_err_stats stats;
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	t4_tp_get_err_stats(sc, &stats);
 
 	sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
 		      "channel 3\n");
 	sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
 	    stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
 	    stats.macInErrs[3]);
 	sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
 	    stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
 	    stats.hdrInErrs[3]);
 	sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
 	    stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
 	    stats.tcpInErrs[3]);
 	sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
 	    stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
 	    stats.tcp6InErrs[3]);
 	sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
 	    stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
 	    stats.tnlCongDrops[3]);
 	sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
 	    stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
 	    stats.tnlTxDrops[3]);
 	sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
 	    stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
 	    stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
 	sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
 	    stats.ofldChanDrops[0], stats.ofldChanDrops[1],
 	    stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
 	sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
 	    stats.ofldNoNeigh, stats.ofldCongDefer);
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 struct field_desc {
 	const char *name;
 	u_int start;
 	u_int width;
 };
 
 static void
 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
 {
 	char buf[32];
 	int line_size = 0;
 
 	while (f->name) {
 		uint64_t mask = (1ULL << f->width) - 1;
 		int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
 		    ((uintmax_t)v >> f->start) & mask);
 
 		if (line_size + len >= 79) {
 			line_size = 8;
 			sbuf_printf(sb, "\n        ");
 		}
 		sbuf_printf(sb, "%s ", buf);
 		line_size += len + 1;
 		f++;
 	}
 	sbuf_printf(sb, "\n");
 }
 
 static struct field_desc tp_la0[] = {
 	{ "RcfOpCodeOut", 60, 4 },
 	{ "State", 56, 4 },
 	{ "WcfState", 52, 4 },
 	{ "RcfOpcSrcOut", 50, 2 },
 	{ "CRxError", 49, 1 },
 	{ "ERxError", 48, 1 },
 	{ "SanityFailed", 47, 1 },
 	{ "SpuriousMsg", 46, 1 },
 	{ "FlushInputMsg", 45, 1 },
 	{ "FlushInputCpl", 44, 1 },
 	{ "RssUpBit", 43, 1 },
 	{ "RssFilterHit", 42, 1 },
 	{ "Tid", 32, 10 },
 	{ "InitTcb", 31, 1 },
 	{ "LineNumber", 24, 7 },
 	{ "Emsg", 23, 1 },
 	{ "EdataOut", 22, 1 },
 	{ "Cmsg", 21, 1 },
 	{ "CdataOut", 20, 1 },
 	{ "EreadPdu", 19, 1 },
 	{ "CreadPdu", 18, 1 },
 	{ "TunnelPkt", 17, 1 },
 	{ "RcfPeerFin", 16, 1 },
 	{ "RcfReasonOut", 12, 4 },
 	{ "TxCchannel", 10, 2 },
 	{ "RcfTxChannel", 8, 2 },
 	{ "RxEchannel", 6, 2 },
 	{ "RcfRxChannel", 5, 1 },
 	{ "RcfDataOutSrdy", 4, 1 },
 	{ "RxDvld", 3, 1 },
 	{ "RxOoDvld", 2, 1 },
 	{ "RxCongestion", 1, 1 },
 	{ "TxCongestion", 0, 1 },
 	{ NULL }
 };
 
 static struct field_desc tp_la1[] = {
 	{ "CplCmdIn", 56, 8 },
 	{ "CplCmdOut", 48, 8 },
 	{ "ESynOut", 47, 1 },
 	{ "EAckOut", 46, 1 },
 	{ "EFinOut", 45, 1 },
 	{ "ERstOut", 44, 1 },
 	{ "SynIn", 43, 1 },
 	{ "AckIn", 42, 1 },
 	{ "FinIn", 41, 1 },
 	{ "RstIn", 40, 1 },
 	{ "DataIn", 39, 1 },
 	{ "DataInVld", 38, 1 },
 	{ "PadIn", 37, 1 },
 	{ "RxBufEmpty", 36, 1 },
 	{ "RxDdp", 35, 1 },
 	{ "RxFbCongestion", 34, 1 },
 	{ "TxFbCongestion", 33, 1 },
 	{ "TxPktSumSrdy", 32, 1 },
 	{ "RcfUlpType", 28, 4 },
 	{ "Eread", 27, 1 },
 	{ "Ebypass", 26, 1 },
 	{ "Esave", 25, 1 },
 	{ "Static0", 24, 1 },
 	{ "Cread", 23, 1 },
 	{ "Cbypass", 22, 1 },
 	{ "Csave", 21, 1 },
 	{ "CPktOut", 20, 1 },
 	{ "RxPagePoolFull", 18, 2 },
 	{ "RxLpbkPkt", 17, 1 },
 	{ "TxLpbkPkt", 16, 1 },
 	{ "RxVfValid", 15, 1 },
 	{ "SynLearned", 14, 1 },
 	{ "SetDelEntry", 13, 1 },
 	{ "SetInvEntry", 12, 1 },
 	{ "CpcmdDvld", 11, 1 },
 	{ "CpcmdSave", 10, 1 },
 	{ "RxPstructsFull", 8, 2 },
 	{ "EpcmdDvld", 7, 1 },
 	{ "EpcmdFlush", 6, 1 },
 	{ "EpcmdTrimPrefix", 5, 1 },
 	{ "EpcmdTrimPostfix", 4, 1 },
 	{ "ERssIp4Pkt", 3, 1 },
 	{ "ERssIp6Pkt", 2, 1 },
 	{ "ERssTcpUdpPkt", 1, 1 },
 	{ "ERssFceFipPkt", 0, 1 },
 	{ NULL }
 };
 
 static struct field_desc tp_la2[] = {
 	{ "CplCmdIn", 56, 8 },
 	{ "MpsVfVld", 55, 1 },
 	{ "MpsPf", 52, 3 },
 	{ "MpsVf", 44, 8 },
 	{ "SynIn", 43, 1 },
 	{ "AckIn", 42, 1 },
 	{ "FinIn", 41, 1 },
 	{ "RstIn", 40, 1 },
 	{ "DataIn", 39, 1 },
 	{ "DataInVld", 38, 1 },
 	{ "PadIn", 37, 1 },
 	{ "RxBufEmpty", 36, 1 },
 	{ "RxDdp", 35, 1 },
 	{ "RxFbCongestion", 34, 1 },
 	{ "TxFbCongestion", 33, 1 },
 	{ "TxPktSumSrdy", 32, 1 },
 	{ "RcfUlpType", 28, 4 },
 	{ "Eread", 27, 1 },
 	{ "Ebypass", 26, 1 },
 	{ "Esave", 25, 1 },
 	{ "Static0", 24, 1 },
 	{ "Cread", 23, 1 },
 	{ "Cbypass", 22, 1 },
 	{ "Csave", 21, 1 },
 	{ "CPktOut", 20, 1 },
 	{ "RxPagePoolFull", 18, 2 },
 	{ "RxLpbkPkt", 17, 1 },
 	{ "TxLpbkPkt", 16, 1 },
 	{ "RxVfValid", 15, 1 },
 	{ "SynLearned", 14, 1 },
 	{ "SetDelEntry", 13, 1 },
 	{ "SetInvEntry", 12, 1 },
 	{ "CpcmdDvld", 11, 1 },
 	{ "CpcmdSave", 10, 1 },
 	{ "RxPstructsFull", 8, 2 },
 	{ "EpcmdDvld", 7, 1 },
 	{ "EpcmdFlush", 6, 1 },
 	{ "EpcmdTrimPrefix", 5, 1 },
 	{ "EpcmdTrimPostfix", 4, 1 },
 	{ "ERssIp4Pkt", 3, 1 },
 	{ "ERssIp6Pkt", 2, 1 },
 	{ "ERssTcpUdpPkt", 1, 1 },
 	{ "ERssFceFipPkt", 0, 1 },
 	{ NULL }
 };
 
 static void
 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
 {
 
 	field_desc_show(sb, *p, tp_la0);
 }
 
 static void
 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
 {
 
 	if (idx)
 		sbuf_printf(sb, "\n");
 	field_desc_show(sb, p[0], tp_la0);
 	if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
 		field_desc_show(sb, p[1], tp_la0);
 }
 
 static void
 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
 {
 
 	if (idx)
 		sbuf_printf(sb, "\n");
 	field_desc_show(sb, p[0], tp_la0);
 	if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
 		field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
 }
 
 static int
 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	uint64_t *buf, *p;
 	int rc;
 	u_int i, inc;
 	void (*show_func)(struct sbuf *, uint64_t *, int);
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
 
 	t4_tp_read_la(sc, buf, NULL);
 	p = buf;
 
 	switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
 	case 2:
 		inc = 2;
 		show_func = tp_la_show2;
 		break;
 	case 3:
 		inc = 2;
 		show_func = tp_la_show3;
 		break;
 	default:
 		inc = 1;
 		show_func = tp_la_show;
 	}
 
 	for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
 		(*show_func)(sb, p, i);
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 	free(buf, M_CXGBE);
 	return (rc);
 }
 
 static int
 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc;
 	u64 nrate[NCHAN], orate[NCHAN];
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	t4_get_chan_txrate(sc, nrate, orate);
 	sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
 		 "channel 3\n");
 	sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
 	    nrate[0], nrate[1], nrate[2], nrate[3]);
 	sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
 	    orate[0], orate[1], orate[2], orate[3]);
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 
 static int
 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	uint32_t *buf, *p;
 	int rc, i;
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
 	    M_ZERO | M_WAITOK);
 
 	t4_ulprx_read_la(sc, buf);
 	p = buf;
 
 	sbuf_printf(sb, "      Pcmd        Type   Message"
 	    "                Data");
 	for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
 		sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
 		    p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
 	}
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 	free(buf, M_CXGBE);
 	return (rc);
 }
 
 static int
 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;
 	struct sbuf *sb;
 	int rc, v;
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	if (rc != 0)
 		return (rc);
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
 	if (sb == NULL)
 		return (ENOMEM);
 
 	v = t4_read_reg(sc, A_SGE_STAT_CFG);
 	if (G_STATSOURCE_T5(v) == 7) {
 		if (G_STATMODE(v) == 0) {
 			sbuf_printf(sb, "total %d, incomplete %d",
 			    t4_read_reg(sc, A_SGE_STAT_TOTAL),
 			    t4_read_reg(sc, A_SGE_STAT_MATCH));
 		} else if (G_STATMODE(v) == 1) {
 			sbuf_printf(sb, "total %d, data overflow %d",
 			    t4_read_reg(sc, A_SGE_STAT_TOTAL),
 			    t4_read_reg(sc, A_SGE_STAT_MATCH));
 		}
 	}
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 
 	return (rc);
 }
 #endif
 
 static inline void
 txq_start(struct ifnet *ifp, struct sge_txq *txq)
 {
 	struct buf_ring *br;
 	struct mbuf *m;
 
 	TXQ_LOCK_ASSERT_OWNED(txq);
 
 	br = txq->br;
 	m = txq->m ? txq->m : drbr_dequeue(ifp, br);
 	if (m)
 		t4_eth_tx(ifp, txq, m);
 }
 
 void
 t4_tx_callout(void *arg)
 {
 	struct sge_eq *eq = arg;
 	struct adapter *sc;
 
 	if (EQ_TRYLOCK(eq) == 0)
 		goto reschedule;
 
 	if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
 		EQ_UNLOCK(eq);
 reschedule:
 		if (__predict_true(!(eq->flags && EQ_DOOMED)))
 			callout_schedule(&eq->tx_callout, 1);
 		return;
 	}
 
 	EQ_LOCK_ASSERT_OWNED(eq);
 
 	if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
 
 		if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
 			struct sge_txq *txq = arg;
 			struct port_info *pi = txq->ifp->if_softc;
 
 			sc = pi->adapter;
 		} else {
 			struct sge_wrq *wrq = arg;
 
 			sc = wrq->adapter;
 		}
 
 		taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
 	}
 
 	EQ_UNLOCK(eq);
 }
 
 void
 t4_tx_task(void *arg, int count)
 {
 	struct sge_eq *eq = arg;
 
 	EQ_LOCK(eq);
 	if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
 		struct sge_txq *txq = arg;
 		txq_start(txq->ifp, txq);
 	} else {
 		struct sge_wrq *wrq = arg;
 		t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
 	}
 	EQ_UNLOCK(eq);
 }
 
 static uint32_t
 fconf_to_mode(uint32_t fconf)
 {
 	uint32_t mode;
 
 	mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
 	    T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
 
 	if (fconf & F_FRAGMENTATION)
 		mode |= T4_FILTER_IP_FRAGMENT;
 
 	if (fconf & F_MPSHITTYPE)
 		mode |= T4_FILTER_MPS_HIT_TYPE;
 
 	if (fconf & F_MACMATCH)
 		mode |= T4_FILTER_MAC_IDX;
 
 	if (fconf & F_ETHERTYPE)
 		mode |= T4_FILTER_ETH_TYPE;
 
 	if (fconf & F_PROTOCOL)
 		mode |= T4_FILTER_IP_PROTO;
 
 	if (fconf & F_TOS)
 		mode |= T4_FILTER_IP_TOS;
 
 	if (fconf & F_VLAN)
 		mode |= T4_FILTER_VLAN;
 
 	if (fconf & F_VNIC_ID)
 		mode |= T4_FILTER_VNIC;
 
 	if (fconf & F_PORT)
 		mode |= T4_FILTER_PORT;
 
 	if (fconf & F_FCOE)
 		mode |= T4_FILTER_FCoE;
 
 	return (mode);
 }
 
 static uint32_t
 mode_to_fconf(uint32_t mode)
 {
 	uint32_t fconf = 0;
 
 	if (mode & T4_FILTER_IP_FRAGMENT)
 		fconf |= F_FRAGMENTATION;
 
 	if (mode & T4_FILTER_MPS_HIT_TYPE)
 		fconf |= F_MPSHITTYPE;
 
 	if (mode & T4_FILTER_MAC_IDX)
 		fconf |= F_MACMATCH;
 
 	if (mode & T4_FILTER_ETH_TYPE)
 		fconf |= F_ETHERTYPE;
 
 	if (mode & T4_FILTER_IP_PROTO)
 		fconf |= F_PROTOCOL;
 
 	if (mode & T4_FILTER_IP_TOS)
 		fconf |= F_TOS;
 
 	if (mode & T4_FILTER_VLAN)
 		fconf |= F_VLAN;
 
 	if (mode & T4_FILTER_VNIC)
 		fconf |= F_VNIC_ID;
 
 	if (mode & T4_FILTER_PORT)
 		fconf |= F_PORT;
 
 	if (mode & T4_FILTER_FCoE)
 		fconf |= F_FCOE;
 
 	return (fconf);
 }
 
 static uint32_t
 fspec_to_fconf(struct t4_filter_specification *fs)
 {
 	uint32_t fconf = 0;
 
 	if (fs->val.frag || fs->mask.frag)
 		fconf |= F_FRAGMENTATION;
 
 	if (fs->val.matchtype || fs->mask.matchtype)
 		fconf |= F_MPSHITTYPE;
 
 	if (fs->val.macidx || fs->mask.macidx)
 		fconf |= F_MACMATCH;
 
 	if (fs->val.ethtype || fs->mask.ethtype)
 		fconf |= F_ETHERTYPE;
 
 	if (fs->val.proto || fs->mask.proto)
 		fconf |= F_PROTOCOL;
 
 	if (fs->val.tos || fs->mask.tos)
 		fconf |= F_TOS;
 
 	if (fs->val.vlan_vld || fs->mask.vlan_vld)
 		fconf |= F_VLAN;
 
 	if (fs->val.vnic_vld || fs->mask.vnic_vld)
 		fconf |= F_VNIC_ID;
 
 	if (fs->val.iport || fs->mask.iport)
 		fconf |= F_PORT;
 
 	if (fs->val.fcoe || fs->mask.fcoe)
 		fconf |= F_FCOE;
 
 	return (fconf);
 }
 
 static int
 get_filter_mode(struct adapter *sc, uint32_t *mode)
 {
 	int rc;
 	uint32_t fconf;
 
 	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
 	    "t4getfm");
 	if (rc)
 		return (rc);
 
 	t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
 	    A_TP_VLAN_PRI_MAP);
 
 	if (sc->params.tp.vlan_pri_map != fconf) {
 		log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
 		    device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
 		    fconf);
 		sc->params.tp.vlan_pri_map = fconf;
 	}
 
 	*mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
 
 	end_synchronized_op(sc, LOCK_HELD);
 	return (0);
 }
 
 static int
 set_filter_mode(struct adapter *sc, uint32_t mode)
 {
 	uint32_t fconf;
 	int rc;
 
 	fconf = mode_to_fconf(mode);
 
 	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
 	    "t4setfm");
 	if (rc)
 		return (rc);
 
 	if (sc->tids.ftids_in_use > 0) {
 		rc = EBUSY;
 		goto done;
 	}
 
 #ifdef TCP_OFFLOAD
 	if (sc->offload_map) {
 		rc = EBUSY;
 		goto done;
 	}
 #endif
 
 #ifdef notyet
 	rc = -t4_set_filter_mode(sc, fconf);
 	if (rc == 0)
 		sc->filter_mode = fconf;
 #else
 	rc = ENOTSUP;
 #endif
 
 done:
 	end_synchronized_op(sc, LOCK_HELD);
 	return (rc);
 }
 
 static inline uint64_t
 get_filter_hits(struct adapter *sc, uint32_t fid)
 {
 	uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
 	uint64_t hits;
 
 	memwin_info(sc, 0, &mw_base, NULL);
 	off = position_memwin(sc, 0,
 	    tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
 	if (is_t4(sc)) {
 		hits = t4_read_reg64(sc, mw_base + off + 16);
 		hits = be64toh(hits);
 	} else {
 		hits = t4_read_reg(sc, mw_base + off + 24);
 		hits = be32toh(hits);
 	}
 
 	return (hits);
 }
 
 static int
 get_filter(struct adapter *sc, struct t4_filter *t)
 {
 	int i, rc, nfilters = sc->tids.nftids;
 	struct filter_entry *f;
 
 	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
 	    "t4getf");
 	if (rc)
 		return (rc);
 
 	if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
 	    t->idx >= nfilters) {
 		t->idx = 0xffffffff;
 		goto done;
 	}
 
 	f = &sc->tids.ftid_tab[t->idx];
 	for (i = t->idx; i < nfilters; i++, f++) {
 		if (f->valid) {
 			t->idx = i;
 			t->l2tidx = f->l2t ? f->l2t->idx : 0;
 			t->smtidx = f->smtidx;
 			if (f->fs.hitcnts)
 				t->hits = get_filter_hits(sc, t->idx);
 			else
 				t->hits = UINT64_MAX;
 			t->fs = f->fs;
 
 			goto done;
 		}
 	}
 
 	t->idx = 0xffffffff;
 done:
 	end_synchronized_op(sc, LOCK_HELD);
 	return (0);
 }
 
 static int
 set_filter(struct adapter *sc, struct t4_filter *t)
 {
 	unsigned int nfilters, nports;
 	struct filter_entry *f;
 	int i, rc;
 
 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
 	if (rc)
 		return (rc);
 
 	nfilters = sc->tids.nftids;
 	nports = sc->params.nports;
 
 	if (nfilters == 0) {
 		rc = ENOTSUP;
 		goto done;
 	}
 
 	if (!(sc->flags & FULL_INIT_DONE)) {
 		rc = EAGAIN;
 		goto done;
 	}
 
 	if (t->idx >= nfilters) {
 		rc = EINVAL;
 		goto done;
 	}
 
 	/* Validate against the global filter mode */
 	if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
 	    sc->params.tp.vlan_pri_map) {
 		rc = E2BIG;
 		goto done;
 	}
 
 	if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
 		rc = EINVAL;
 		goto done;
 	}
 
 	if (t->fs.val.iport >= nports) {
 		rc = EINVAL;
 		goto done;
 	}
 
 	/* Can't specify an iq if not steering to it */
 	if (!t->fs.dirsteer && t->fs.iq) {
 		rc = EINVAL;
 		goto done;
 	}
 
 	/* IPv6 filter idx must be 4 aligned */
 	if (t->fs.type == 1 &&
 	    ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
 		rc = EINVAL;
 		goto done;
 	}
 
 	if (sc->tids.ftid_tab == NULL) {
 		KASSERT(sc->tids.ftids_in_use == 0,
 		    ("%s: no memory allocated but filters_in_use > 0",
 		    __func__));
 
 		sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
 		    nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
 		if (sc->tids.ftid_tab == NULL) {
 			rc = ENOMEM;
 			goto done;
 		}
 		mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
 	}
 
 	for (i = 0; i < 4; i++) {
 		f = &sc->tids.ftid_tab[t->idx + i];
 
 		if (f->pending || f->valid) {
 			rc = EBUSY;
 			goto done;
 		}
 		if (f->locked) {
 			rc = EPERM;
 			goto done;
 		}
 
 		if (t->fs.type == 0)
 			break;
 	}
 
 	f = &sc->tids.ftid_tab[t->idx];
 	f->fs = t->fs;
 
 	rc = set_filter_wr(sc, t->idx);
 done:
 	end_synchronized_op(sc, 0);
 
 	if (rc == 0) {
 		mtx_lock(&sc->tids.ftid_lock);
 		for (;;) {
 			if (f->pending == 0) {
 				rc = f->valid ? 0 : EIO;
 				break;
 			}
 
 			if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
 			    PCATCH, "t4setfw", 0)) {
 				rc = EINPROGRESS;
 				break;
 			}
 		}
 		mtx_unlock(&sc->tids.ftid_lock);
 	}
 	return (rc);
 }
 
 static int
 del_filter(struct adapter *sc, struct t4_filter *t)
 {
 	unsigned int nfilters;
 	struct filter_entry *f;
 	int rc;
 
 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
 	if (rc)
 		return (rc);
 
 	nfilters = sc->tids.nftids;
 
 	if (nfilters == 0) {
 		rc = ENOTSUP;
 		goto done;
 	}
 
 	if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
 	    t->idx >= nfilters) {
 		rc = EINVAL;
 		goto done;
 	}
 
 	if (!(sc->flags & FULL_INIT_DONE)) {
 		rc = EAGAIN;
 		goto done;
 	}
 
 	f = &sc->tids.ftid_tab[t->idx];
 
 	if (f->pending) {
 		rc = EBUSY;
 		goto done;
 	}
 	if (f->locked) {
 		rc = EPERM;
 		goto done;
 	}
 
 	if (f->valid) {
 		t->fs = f->fs;	/* extra info for the caller */
 		rc = del_filter_wr(sc, t->idx);
 	}
 
 done:
 	end_synchronized_op(sc, 0);
 
 	if (rc == 0) {
 		mtx_lock(&sc->tids.ftid_lock);
 		for (;;) {
 			if (f->pending == 0) {
 				rc = f->valid ? EIO : 0;
 				break;
 			}
 
 			if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
 			    PCATCH, "t4delfw", 0)) {
 				rc = EINPROGRESS;
 				break;
 			}
 		}
 		mtx_unlock(&sc->tids.ftid_lock);
 	}
 
 	return (rc);
 }
 
 static void
 clear_filter(struct filter_entry *f)
 {
 	if (f->l2t)
 		t4_l2t_release(f->l2t);
 
 	bzero(f, sizeof (*f));
 }
 
 static int
 set_filter_wr(struct adapter *sc, int fidx)
 {
 	struct filter_entry *f = &sc->tids.ftid_tab[fidx];
 	struct wrqe *wr;
 	struct fw_filter_wr *fwr;
 	unsigned int ftid;
 
 	ASSERT_SYNCHRONIZED_OP(sc);
 
 	if (f->fs.newdmac || f->fs.newvlan) {
 		/* This filter needs an L2T entry; allocate one. */
 		f->l2t = t4_l2t_alloc_switching(sc->l2t);
 		if (f->l2t == NULL)
 			return (EAGAIN);
 		if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
 		    f->fs.dmac)) {
 			t4_l2t_release(f->l2t);
 			f->l2t = NULL;
 			return (ENOMEM);
 		}
 	}
 
 	ftid = sc->tids.ftid_base + fidx;
 
 	wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
 	if (wr == NULL)
 		return (ENOMEM);
 
 	fwr = wrtod(wr);
 	bzero(fwr, sizeof (*fwr));
 
 	fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
 	fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
 	fwr->tid_to_iq =
 	    htobe32(V_FW_FILTER_WR_TID(ftid) |
 		V_FW_FILTER_WR_RQTYPE(f->fs.type) |
 		V_FW_FILTER_WR_NOREPLY(0) |
 		V_FW_FILTER_WR_IQ(f->fs.iq));
 	fwr->del_filter_to_l2tix =
 	    htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
 		V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
 		V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
 		V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
 		V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
 		V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
 		V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
 		V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
 		V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
 		    f->fs.newvlan == VLAN_REWRITE) |
 		V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
 		    f->fs.newvlan == VLAN_REWRITE) |
 		V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
 		V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
 		V_FW_FILTER_WR_PRIO(f->fs.prio) |
 		V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
 	fwr->ethtype = htobe16(f->fs.val.ethtype);
 	fwr->ethtypem = htobe16(f->fs.mask.ethtype);
 	fwr->frag_to_ovlan_vldm =
 	    (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
 		V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
 		V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
 		V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
 		V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
 		V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
 	fwr->smac_sel = 0;
 	fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
 	    V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
 	fwr->maci_to_matchtypem =
 	    htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
 		V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
 		V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
 		V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
 		V_FW_FILTER_WR_PORT(f->fs.val.iport) |
 		V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
 		V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
 		V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
 	fwr->ptcl = f->fs.val.proto;
 	fwr->ptclm = f->fs.mask.proto;
 	fwr->ttyp = f->fs.val.tos;
 	fwr->ttypm = f->fs.mask.tos;
 	fwr->ivlan = htobe16(f->fs.val.vlan);
 	fwr->ivlanm = htobe16(f->fs.mask.vlan);
 	fwr->ovlan = htobe16(f->fs.val.vnic);
 	fwr->ovlanm = htobe16(f->fs.mask.vnic);
 	bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
 	bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
 	bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
 	bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
 	fwr->lp = htobe16(f->fs.val.dport);
 	fwr->lpm = htobe16(f->fs.mask.dport);
 	fwr->fp = htobe16(f->fs.val.sport);
 	fwr->fpm = htobe16(f->fs.mask.sport);
 	if (f->fs.newsmac)
 		bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
 
 	f->pending = 1;
 	sc->tids.ftids_in_use++;
 
 	t4_wrq_tx(sc, wr);
 	return (0);
 }
 
 static int
 del_filter_wr(struct adapter *sc, int fidx)
 {
 	struct filter_entry *f = &sc->tids.ftid_tab[fidx];
 	struct wrqe *wr;
 	struct fw_filter_wr *fwr;
 	unsigned int ftid;
 
 	ftid = sc->tids.ftid_base + fidx;
 
 	wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
 	if (wr == NULL)
 		return (ENOMEM);
 	fwr = wrtod(wr);
 	bzero(fwr, sizeof (*fwr));
 
 	t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
 
 	f->pending = 1;
 	t4_wrq_tx(sc, wr);
 	return (0);
 }
 
 int
 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
 	const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
 	unsigned int idx = GET_TID(rpl);
 	unsigned int rc;
 	struct filter_entry *f;
 
 	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
 	    rss->opcode));
 
 	if (is_ftid(sc, idx)) {
 
 		idx -= sc->tids.ftid_base;
 		f = &sc->tids.ftid_tab[idx];
 		rc = G_COOKIE(rpl->cookie);
 
 		mtx_lock(&sc->tids.ftid_lock);
 		if (rc == FW_FILTER_WR_FLT_ADDED) {
 			KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
 			    __func__, idx));
 			f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
 			f->pending = 0;  /* asynchronous setup completed */
 			f->valid = 1;
 		} else {
 			if (rc != FW_FILTER_WR_FLT_DELETED) {
 				/* Add or delete failed, display an error */
 				log(LOG_ERR,
 				    "filter %u setup failed with error %u\n",
 				    idx, rc);
 			}
 
 			clear_filter(f);
 			sc->tids.ftids_in_use--;
 		}
 		wakeup(&sc->tids.ftid_tab);
 		mtx_unlock(&sc->tids.ftid_lock);
 	}
 
 	return (0);
 }
 
 static int
 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
 {
 	int rc;
 
 	if (cntxt->cid > M_CTXTQID)
 		return (EINVAL);
 
 	if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
 	    cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
 		return (EINVAL);
 
 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
 	if (rc)
 		return (rc);
 
 	if (sc->flags & FW_OK) {
 		rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
 		    &cntxt->data[0]);
 		if (rc == 0)
 			goto done;
 	}
 
 	/*
 	 * Read via firmware failed or wasn't even attempted.  Read directly via
 	 * the backdoor.
 	 */
 	rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
 done:
 	end_synchronized_op(sc, 0);
 	return (rc);
 }
 
 static int
 load_fw(struct adapter *sc, struct t4_data *fw)
 {
 	int rc;
 	uint8_t *fw_data;
 
 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
 	if (rc)
 		return (rc);
 
 	if (sc->flags & FULL_INIT_DONE) {
 		rc = EBUSY;
 		goto done;
 	}
 
 	fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
 	if (fw_data == NULL) {
 		rc = ENOMEM;
 		goto done;
 	}
 
 	rc = copyin(fw->data, fw_data, fw->len);
 	if (rc == 0)
 		rc = -t4_load_fw(sc, fw_data, fw->len);
 
 	free(fw_data, M_CXGBE);
 done:
 	end_synchronized_op(sc, 0);
 	return (rc);
 }
 
 static int
 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
 {
 	uint32_t addr, off, remaining, i, n;
 	uint32_t *buf, *b;
 	uint32_t mw_base, mw_aperture;
 	int rc;
 	uint8_t *dst;
 
 	rc = validate_mem_range(sc, mr->addr, mr->len);
 	if (rc != 0)
 		return (rc);
 
 	memwin_info(sc, win, &mw_base, &mw_aperture);
 	buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
 	addr = mr->addr;
 	remaining = mr->len;
 	dst = (void *)mr->data;
 
 	while (remaining) {
 		off = position_memwin(sc, win, addr);
 
 		/* number of bytes that we'll copy in the inner loop */
 		n = min(remaining, mw_aperture - off);
 		for (i = 0; i < n; i += 4)
 			*b++ = t4_read_reg(sc, mw_base + off + i);
 
 		rc = copyout(buf, dst, n);
 		if (rc != 0)
 			break;
 
 		b = buf;
 		dst += n;
 		remaining -= n;
 		addr += n;
 	}
 
 	free(buf, M_CXGBE);
 	return (rc);
 }
 
 static int
 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
 {
 	int rc;
 
 	if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
 		return (EINVAL);
 
 	if (i2cd->len > sizeof(i2cd->data))
 		return (EFBIG);
 
 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
 	if (rc)
 		return (rc);
 	rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
 	    i2cd->offset, i2cd->len, &i2cd->data[0]);
 	end_synchronized_op(sc, 0);
 
 	return (rc);
 }
 
 static int
 in_range(int val, int lo, int hi)
 {
 
 	return (val < 0 || (val <= hi && val >= lo));
 }
 
 static int
 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
 {
 	int fw_subcmd, fw_type, rc;
 
 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
 	if (rc)
 		return (rc);
 
 	if (!(sc->flags & FULL_INIT_DONE)) {
 		rc = EAGAIN;
 		goto done;
 	}
 
 	/*
 	 * Translate the cxgbetool parameters into T4 firmware parameters.  (The
 	 * sub-command and type are in common locations.)
 	 */
 	if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
 		fw_subcmd = FW_SCHED_SC_CONFIG;
 	else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
 		fw_subcmd = FW_SCHED_SC_PARAMS;
 	else {
 		rc = EINVAL;
 		goto done;
 	}
 	if (p->type == SCHED_CLASS_TYPE_PACKET)
 		fw_type = FW_SCHED_TYPE_PKTSCHED;
 	else {
 		rc = EINVAL;
 		goto done;
 	}
 
 	if (fw_subcmd == FW_SCHED_SC_CONFIG) {
 		/* Vet our parameters ..*/
 		if (p->u.config.minmax < 0) {
 			rc = EINVAL;
 			goto done;
 		}
 
 		/* And pass the request to the firmware ...*/
-		rc = -t4_sched_config(sc, fw_type, p->u.config.minmax);
+		rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
 		goto done;
 	}
 
 	if (fw_subcmd == FW_SCHED_SC_PARAMS) {
 		int fw_level;
 		int fw_mode;
 		int fw_rateunit;
 		int fw_ratemode;
 
 		if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
 			fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
 		else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
 			fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
 		else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
 			fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
 		else {
 			rc = EINVAL;
 			goto done;
 		}
 
 		if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
 			fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
 		else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
 			fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
 		else {
 			rc = EINVAL;
 			goto done;
 		}
 
 		if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
 			fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
 		else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
 			fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
 		else {
 			rc = EINVAL;
 			goto done;
 		}
 
 		if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
 			fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
 		else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
 			fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
 		else {
 			rc = EINVAL;
 			goto done;
 		}
 
 		/* Vet our parameters ... */
 		if (!in_range(p->u.params.channel, 0, 3) ||
 		    !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
 		    !in_range(p->u.params.minrate, 0, 10000000) ||
 		    !in_range(p->u.params.maxrate, 0, 10000000) ||
 		    !in_range(p->u.params.weight, 0, 100)) {
 			rc = ERANGE;
 			goto done;
 		}
 
 		/*
 		 * Translate any unset parameters into the firmware's
 		 * nomenclature and/or fail the call if the parameters
 		 * are required ...
 		 */
 		if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
 		    p->u.params.channel < 0 || p->u.params.cl < 0) {
 			rc = EINVAL;
 			goto done;
 		}
 		if (p->u.params.minrate < 0)
 			p->u.params.minrate = 0;
 		if (p->u.params.maxrate < 0) {
 			if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
 			    p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
 				rc = EINVAL;
 				goto done;
 			} else
 				p->u.params.maxrate = 0;
 		}
 		if (p->u.params.weight < 0) {
 			if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
 				rc = EINVAL;
 				goto done;
 			} else
 				p->u.params.weight = 0;
 		}
 		if (p->u.params.pktsize < 0) {
 			if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
 			    p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
 				rc = EINVAL;
 				goto done;
 			} else
 				p->u.params.pktsize = 0;
 		}
 
 		/* See what the firmware thinks of the request ... */
 		rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
 		    fw_rateunit, fw_ratemode, p->u.params.channel,
 		    p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
-		    p->u.params.weight, p->u.params.pktsize);
+		    p->u.params.weight, p->u.params.pktsize, 1);
 		goto done;
 	}
 
 	rc = EINVAL;
 done:
 	end_synchronized_op(sc, 0);
 	return (rc);
 }
 
 static int
 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
 {
 	struct port_info *pi = NULL;
 	struct sge_txq *txq;
 	uint32_t fw_mnem, fw_queue, fw_class;
 	int i, rc;
 
 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
 	if (rc)
 		return (rc);
 
 	if (!(sc->flags & FULL_INIT_DONE)) {
 		rc = EAGAIN;
 		goto done;
 	}
 
 	if (p->port >= sc->params.nports) {
 		rc = EINVAL;
 		goto done;
 	}
 
 	pi = sc->port[p->port];
 	if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
 		rc = EINVAL;
 		goto done;
 	}
 
 	/*
 	 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
 	 * Scheduling Class in this case).
 	 */
 	fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
 	    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
 	fw_class = p->cl < 0 ? 0xffffffff : p->cl;
 
 	/*
 	 * If op.queue is non-negative, then we're only changing the scheduling
 	 * on a single specified TX queue.
 	 */
 	if (p->queue >= 0) {
 		txq = &sc->sge.txq[pi->first_txq + p->queue];
 		fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
 		rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
 		    &fw_class);
 		goto done;
 	}
 
 	/*
 	 * Change the scheduling on all the TX queues for the
 	 * interface.
 	 */
 	for_each_txq(pi, i, txq) {
 		fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
 		rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
 		    &fw_class);
 		if (rc)
 			goto done;
 	}
 
 	rc = 0;
 done:
 	end_synchronized_op(sc, 0);
 	return (rc);
 }
 
 int
 t4_os_find_pci_capability(struct adapter *sc, int cap)
 {
 	int i;
 
 	return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
 }
 
 int
 t4_os_pci_save_state(struct adapter *sc)
 {
 	device_t dev;
 	struct pci_devinfo *dinfo;
 
 	dev = sc->dev;
 	dinfo = device_get_ivars(dev);
 
 	pci_cfg_save(dev, dinfo, 0);
 	return (0);
 }
 
 int
 t4_os_pci_restore_state(struct adapter *sc)
 {
 	device_t dev;
 	struct pci_devinfo *dinfo;
 
 	dev = sc->dev;
 	dinfo = device_get_ivars(dev);
 
 	pci_cfg_restore(dev, dinfo);
 	return (0);
 }
 
 void
 t4_os_portmod_changed(const struct adapter *sc, int idx)
 {
 	struct port_info *pi = sc->port[idx];
 	static const char *mod_str[] = {
 		NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
 	};
 
 	if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
 		if_printf(pi->ifp, "transceiver unplugged.\n");
 	else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
 		if_printf(pi->ifp, "unknown transceiver inserted.\n");
 	else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
 		if_printf(pi->ifp, "unsupported transceiver inserted.\n");
 	else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
 		if_printf(pi->ifp, "%s transceiver inserted.\n",
 		    mod_str[pi->mod_type]);
 	} else {
 		if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
 		    pi->mod_type);
 	}
 }
 
 void
 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
 {
 	struct port_info *pi = sc->port[idx];
 	struct ifnet *ifp = pi->ifp;
 
 	if (link_stat) {
 		pi->linkdnrc = -1;
 		ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
 		if_link_state_change(ifp, LINK_STATE_UP);
 	} else {
 		if (reason >= 0)
 			pi->linkdnrc = reason;
 		if_link_state_change(ifp, LINK_STATE_DOWN);
 	}
 }
 
 void
 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
 {
 	struct adapter *sc;
 
 	sx_slock(&t4_list_lock);
 	SLIST_FOREACH(sc, &t4_list, link) {
 		/*
 		 * func should not make any assumptions about what state sc is
 		 * in - the only guarantee is that sc->sc_lock is a valid lock.
 		 */
 		func(sc, arg);
 	}
 	sx_sunlock(&t4_list_lock);
 }
 
 static int
 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
 {
        return (0);
 }
 
 static int
 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
 {
        return (0);
 }
 
 static int
 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
     struct thread *td)
 {
 	int rc;
 	struct adapter *sc = dev->si_drv1;
 
 	rc = priv_check(td, PRIV_DRIVER);
 	if (rc != 0)
 		return (rc);
 
 	switch (cmd) {
 	case CHELSIO_T4_GETREG: {
 		struct t4_reg *edata = (struct t4_reg *)data;
 
 		if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
 			return (EFAULT);
 
 		if (edata->size == 4)
 			edata->val = t4_read_reg(sc, edata->addr);
 		else if (edata->size == 8)
 			edata->val = t4_read_reg64(sc, edata->addr);
 		else
 			return (EINVAL);
 
 		break;
 	}
 	case CHELSIO_T4_SETREG: {
 		struct t4_reg *edata = (struct t4_reg *)data;
 
 		if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
 			return (EFAULT);
 
 		if (edata->size == 4) {
 			if (edata->val & 0xffffffff00000000)
 				return (EINVAL);
 			t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
 		} else if (edata->size == 8)
 			t4_write_reg64(sc, edata->addr, edata->val);
 		else
 			return (EINVAL);
 		break;
 	}
 	case CHELSIO_T4_REGDUMP: {
 		struct t4_regdump *regs = (struct t4_regdump *)data;
 		int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
 		uint8_t *buf;
 
 		if (regs->len < reglen) {
 			regs->len = reglen; /* hint to the caller */
 			return (ENOBUFS);
 		}
 
 		regs->len = reglen;
 		buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
 		t4_get_regs(sc, regs, buf);
 		rc = copyout(buf, regs->data, reglen);
 		free(buf, M_CXGBE);
 		break;
 	}
 	case CHELSIO_T4_GET_FILTER_MODE:
 		rc = get_filter_mode(sc, (uint32_t *)data);
 		break;
 	case CHELSIO_T4_SET_FILTER_MODE:
 		rc = set_filter_mode(sc, *(uint32_t *)data);
 		break;
 	case CHELSIO_T4_GET_FILTER:
 		rc = get_filter(sc, (struct t4_filter *)data);
 		break;
 	case CHELSIO_T4_SET_FILTER:
 		rc = set_filter(sc, (struct t4_filter *)data);
 		break;
 	case CHELSIO_T4_DEL_FILTER:
 		rc = del_filter(sc, (struct t4_filter *)data);
 		break;
 	case CHELSIO_T4_GET_SGE_CONTEXT:
 		rc = get_sge_context(sc, (struct t4_sge_context *)data);
 		break;
 	case CHELSIO_T4_LOAD_FW:
 		rc = load_fw(sc, (struct t4_data *)data);
 		break;
 	case CHELSIO_T4_GET_MEM:
 		rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
 		break;
 	case CHELSIO_T4_GET_I2C:
 		rc = read_i2c(sc, (struct t4_i2c_data *)data);
 		break;
 	case CHELSIO_T4_CLEAR_STATS: {
 		int i;
 		u_int port_id = *(uint32_t *)data;
 		struct port_info *pi;
 
 		if (port_id >= sc->params.nports)
 			return (EINVAL);
 		pi = sc->port[port_id];
 
 		/* MAC stats */
 		t4_clr_port_stats(sc, pi->tx_chan);
 
 		if (pi->flags & PORT_INIT_DONE) {
 			struct sge_rxq *rxq;
 			struct sge_txq *txq;
 			struct sge_wrq *wrq;
 
 			for_each_rxq(pi, i, rxq) {
 #if defined(INET) || defined(INET6)
 				rxq->lro.lro_queued = 0;
 				rxq->lro.lro_flushed = 0;
 #endif
 				rxq->rxcsum = 0;
 				rxq->vlan_extraction = 0;
 			}
 
 			for_each_txq(pi, i, txq) {
 				txq->txcsum = 0;
 				txq->tso_wrs = 0;
 				txq->vlan_insertion = 0;
 				txq->imm_wrs = 0;
 				txq->sgl_wrs = 0;
 				txq->txpkt_wrs = 0;
 				txq->txpkts_wrs = 0;
 				txq->txpkts_pkts = 0;
 				txq->br->br_drops = 0;
 				txq->no_dmamap = 0;
 				txq->no_desc = 0;
 			}
 
 #ifdef TCP_OFFLOAD
 			/* nothing to clear for each ofld_rxq */
 
 			for_each_ofld_txq(pi, i, wrq) {
 				wrq->tx_wrs = 0;
 				wrq->no_desc = 0;
 			}
 #endif
 			wrq = &sc->sge.ctrlq[pi->port_id];
 			wrq->tx_wrs = 0;
 			wrq->no_desc = 0;
 		}
 		break;
 	}
 	case CHELSIO_T4_SCHED_CLASS:
 		rc = set_sched_class(sc, (struct t4_sched_params *)data);
 		break;
 	case CHELSIO_T4_SCHED_QUEUE:
 		rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
 		break;
 	case CHELSIO_T4_GET_TRACER:
 		rc = t4_get_tracer(sc, (struct t4_tracer *)data);
 		break;
 	case CHELSIO_T4_SET_TRACER:
 		rc = t4_set_tracer(sc, (struct t4_tracer *)data);
 		break;
 	default:
 		rc = EINVAL;
 	}
 
 	return (rc);
 }
 
 #ifdef TCP_OFFLOAD
+void
+t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
+    const unsigned int *pgsz_order)
+{
+	struct port_info *pi = ifp->if_softc;
+	struct adapter *sc = pi->adapter;
+
+	t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
+	t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
+		V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
+		V_HPZ3(pgsz_order[3]));
+}
+
 static int
 toe_capability(struct port_info *pi, int enable)
 {
 	int rc;
 	struct adapter *sc = pi->adapter;
 
 	ASSERT_SYNCHRONIZED_OP(sc);
 
 	if (!is_offload(sc))
 		return (ENODEV);
 
 	if (enable) {
 		if (!(sc->flags & FULL_INIT_DONE)) {
 			rc = cxgbe_init_synchronized(pi);
 			if (rc)
 				return (rc);
 		}
 
 		if (isset(&sc->offload_map, pi->port_id))
 			return (0);
 
 		if (!(sc->flags & TOM_INIT_DONE)) {
 			rc = t4_activate_uld(sc, ULD_TOM);
 			if (rc == EAGAIN) {
 				log(LOG_WARNING,
 				    "You must kldload t4_tom.ko before trying "
 				    "to enable TOE on a cxgbe interface.\n");
 			}
 			if (rc != 0)
 				return (rc);
 			KASSERT(sc->tom_softc != NULL,
 			    ("%s: TOM activated but softc NULL", __func__));
 			KASSERT(sc->flags & TOM_INIT_DONE,
 			    ("%s: TOM activated but flag not set", __func__));
 		}
 
 		setbit(&sc->offload_map, pi->port_id);
 	} else {
 		if (!isset(&sc->offload_map, pi->port_id))
 			return (0);
 
 		KASSERT(sc->flags & TOM_INIT_DONE,
 		    ("%s: TOM never initialized?", __func__));
 		clrbit(&sc->offload_map, pi->port_id);
 	}
 
 	return (0);
 }
 
 /*
  * Add an upper layer driver to the global list.
  */
 int
 t4_register_uld(struct uld_info *ui)
 {
 	int rc = 0;
 	struct uld_info *u;
 
 	sx_xlock(&t4_uld_list_lock);
 	SLIST_FOREACH(u, &t4_uld_list, link) {
 	    if (u->uld_id == ui->uld_id) {
 		    rc = EEXIST;
 		    goto done;
 	    }
 	}
 
 	SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
 	ui->refcount = 0;
 done:
 	sx_xunlock(&t4_uld_list_lock);
 	return (rc);
 }
 
 int
 t4_unregister_uld(struct uld_info *ui)
 {
 	int rc = EINVAL;
 	struct uld_info *u;
 
 	sx_xlock(&t4_uld_list_lock);
 
 	SLIST_FOREACH(u, &t4_uld_list, link) {
 	    if (u == ui) {
 		    if (ui->refcount > 0) {
 			    rc = EBUSY;
 			    goto done;
 		    }
 
 		    SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
 		    rc = 0;
 		    goto done;
 	    }
 	}
 done:
 	sx_xunlock(&t4_uld_list_lock);
 	return (rc);
 }
 
 int
 t4_activate_uld(struct adapter *sc, int id)
 {
 	int rc = EAGAIN;
 	struct uld_info *ui;
 
 	ASSERT_SYNCHRONIZED_OP(sc);
 
 	sx_slock(&t4_uld_list_lock);
 
 	SLIST_FOREACH(ui, &t4_uld_list, link) {
 		if (ui->uld_id == id) {
 			rc = ui->activate(sc);
 			if (rc == 0)
 				ui->refcount++;
 			goto done;
 		}
 	}
 done:
 	sx_sunlock(&t4_uld_list_lock);
 
 	return (rc);
 }
 
 int
 t4_deactivate_uld(struct adapter *sc, int id)
 {
 	int rc = EINVAL;
 	struct uld_info *ui;
 
 	ASSERT_SYNCHRONIZED_OP(sc);
 
 	sx_slock(&t4_uld_list_lock);
 
 	SLIST_FOREACH(ui, &t4_uld_list, link) {
 		if (ui->uld_id == id) {
 			rc = ui->deactivate(sc);
 			if (rc == 0)
 				ui->refcount--;
 			goto done;
 		}
 	}
 done:
 	sx_sunlock(&t4_uld_list_lock);
 
 	return (rc);
 }
 #endif
 
 /*
  * Come up with reasonable defaults for some of the tunables, provided they're
  * not set by the user (in which case we'll use the values as is).
  */
 static void
 tweak_tunables(void)
 {
 	int nc = mp_ncpus;	/* our snapshot of the number of CPUs */
 
 	if (t4_ntxq10g < 1)
 		t4_ntxq10g = min(nc, NTXQ_10G);
 
 	if (t4_ntxq1g < 1)
 		t4_ntxq1g = min(nc, NTXQ_1G);
 
 	if (t4_nrxq10g < 1)
 		t4_nrxq10g = min(nc, NRXQ_10G);
 
 	if (t4_nrxq1g < 1)
 		t4_nrxq1g = min(nc, NRXQ_1G);
 
 #ifdef TCP_OFFLOAD
 	if (t4_nofldtxq10g < 1)
 		t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
 
 	if (t4_nofldtxq1g < 1)
 		t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
 
 	if (t4_nofldrxq10g < 1)
 		t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
 
 	if (t4_nofldrxq1g < 1)
 		t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
 
 	if (t4_toecaps_allowed == -1)
 		t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
 #else
 	if (t4_toecaps_allowed == -1)
 		t4_toecaps_allowed = 0;
+#endif
+
+#ifdef DEV_NETMAP
+	if (t4_nnmtxq10g < 1)
+		t4_nnmtxq10g = min(nc, NNMTXQ_10G);
+
+	if (t4_nnmtxq1g < 1)
+		t4_nnmtxq1g = min(nc, NNMTXQ_1G);
+
+	if (t4_nnmrxq10g < 1)
+		t4_nnmrxq10g = min(nc, NNMRXQ_10G);
+
+	if (t4_nnmrxq1g < 1)
+		t4_nnmrxq1g = min(nc, NNMRXQ_1G);
 #endif
 
 	if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
 		t4_tmr_idx_10g = TMR_IDX_10G;
 
 	if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
 		t4_pktc_idx_10g = PKTC_IDX_10G;
 
 	if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
 		t4_tmr_idx_1g = TMR_IDX_1G;
 
 	if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
 		t4_pktc_idx_1g = PKTC_IDX_1G;
 
 	if (t4_qsize_txq < 128)
 		t4_qsize_txq = 128;
 
 	if (t4_qsize_rxq < 128)
 		t4_qsize_rxq = 128;
 	while (t4_qsize_rxq & 7)
 		t4_qsize_rxq++;
 
 	t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
 }
 
 static struct sx mlu;	/* mod load unload */
 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
 
 static int
 mod_event(module_t mod, int cmd, void *arg)
 {
 	int rc = 0;
 	static int loaded = 0;
 
 	switch (cmd) {
 	case MOD_LOAD:
 		sx_xlock(&mlu);
 		if (loaded++ == 0) {
 			t4_sge_modload();
 			sx_init(&t4_list_lock, "T4/T5 adapters");
 			SLIST_INIT(&t4_list);
 #ifdef TCP_OFFLOAD
 			sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
 			SLIST_INIT(&t4_uld_list);
 #endif
 			t4_tracer_modload();
 			tweak_tunables();
 		}
 		sx_xunlock(&mlu);
 		break;
 
 	case MOD_UNLOAD:
 		sx_xlock(&mlu);
 		if (--loaded == 0) {
 			int tries;
 
 			sx_slock(&t4_list_lock);
 			if (!SLIST_EMPTY(&t4_list)) {
 				rc = EBUSY;
 				sx_sunlock(&t4_list_lock);
 				goto done_unload;
 			}
 #ifdef TCP_OFFLOAD
 			sx_slock(&t4_uld_list_lock);
 			if (!SLIST_EMPTY(&t4_uld_list)) {
 				rc = EBUSY;
 				sx_sunlock(&t4_uld_list_lock);
 				sx_sunlock(&t4_list_lock);
 				goto done_unload;
 			}
 #endif
 			tries = 0;
 			while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
 				uprintf("%ju clusters with custom free routine "
 				    "still is use.\n", t4_sge_extfree_refs());
 				pause("t4unload", 2 * hz);
 			}
 #ifdef TCP_OFFLOAD
 			sx_sunlock(&t4_uld_list_lock);
 #endif
 			sx_sunlock(&t4_list_lock);
 
 			if (t4_sge_extfree_refs() == 0) {
 				t4_tracer_modunload();
 #ifdef TCP_OFFLOAD
 				sx_destroy(&t4_uld_list_lock);
 #endif
 				sx_destroy(&t4_list_lock);
 				t4_sge_modunload();
 				loaded = 0;
 			} else {
 				rc = EBUSY;
 				loaded++;	/* undo earlier decrement */
 			}
 		}
 done_unload:
 		sx_xunlock(&mlu);
 		break;
 	}
 
 	return (rc);
 }
 
 static devclass_t t4_devclass, t5_devclass;
 static devclass_t cxgbe_devclass, cxl_devclass;
 
 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
 MODULE_VERSION(t4nex, 1);
 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
 
 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
 MODULE_VERSION(t5nex, 1);
 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
 
 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
 MODULE_VERSION(cxgbe, 1);
 
 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
 MODULE_VERSION(cxl, 1);
Index: stable/10/sys/dev/cxgbe/t4_netmap.c
===================================================================
--- stable/10/sys/dev/cxgbe/t4_netmap.c	(nonexistent)
+++ stable/10/sys/dev/cxgbe/t4_netmap.c	(revision 270297)
@@ -0,0 +1,1138 @@
+/*-
+ * Copyright (c) 2014 Chelsio Communications, Inc.
+ * All rights reserved.
+ * Written by: Navdeep Parhar <np@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_inet.h"
+#include "opt_inet6.h"
+
+#ifdef DEV_NETMAP
+#include <sys/param.h>
+#include <sys/eventhandler.h>
+#include <sys/lock.h>
+#include <sys/types.h>
+#include <sys/mbuf.h>
+#include <sys/selinfo.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <machine/bus.h>
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_media.h>
+#include <net/if_var.h>
+#include <net/if_clone.h>
+#include <net/if_types.h>
+#include <net/netmap.h>
+#include <dev/netmap/netmap_kern.h>
+
+#include "common/common.h"
+#include "common/t4_regs.h"
+#include "common/t4_regs_values.h"
+
+extern int fl_pad;	/* XXXNM */
+extern int spg_len;	/* XXXNM */
+extern int fl_pktshift;	/* XXXNM */
+
+/* netmap ifnet routines */
+static void cxgbe_nm_init(void *);
+static int cxgbe_nm_ioctl(struct ifnet *, unsigned long, caddr_t);
+static int cxgbe_nm_transmit(struct ifnet *, struct mbuf *);
+static void cxgbe_nm_qflush(struct ifnet *);
+
+static int cxgbe_nm_init_synchronized(struct port_info *);
+static int cxgbe_nm_uninit_synchronized(struct port_info *);
+
+static void
+cxgbe_nm_init(void *arg)
+{
+	struct port_info *pi = arg;
+	struct adapter *sc = pi->adapter;
+
+	if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4nminit") != 0)
+		return;
+	cxgbe_nm_init_synchronized(pi);
+	end_synchronized_op(sc, 0);
+
+	return;
+}
+
+static int
+cxgbe_nm_init_synchronized(struct port_info *pi)
+{
+	struct adapter *sc = pi->adapter;
+	struct ifnet *ifp = pi->nm_ifp;
+	int rc = 0;
+
+	ASSERT_SYNCHRONIZED_OP(sc);
+
+	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+		return (0);	/* already running */
+
+	if (!(sc->flags & FULL_INIT_DONE) &&
+	    ((rc = adapter_full_init(sc)) != 0))
+		return (rc);	/* error message displayed already */
+
+	if (!(pi->flags & PORT_INIT_DONE) &&
+	    ((rc = port_full_init(pi)) != 0))
+		return (rc);	/* error message displayed already */
+
+	rc = update_mac_settings(ifp, XGMAC_ALL);
+	if (rc)
+		return (rc);	/* error message displayed already */
+
+	ifp->if_drv_flags |= IFF_DRV_RUNNING;
+
+	return (rc);
+}
+
+static int
+cxgbe_nm_uninit_synchronized(struct port_info *pi)
+{
+#ifdef INVARIANTS
+	struct adapter *sc = pi->adapter;
+#endif
+	struct ifnet *ifp = pi->nm_ifp;
+
+	ASSERT_SYNCHRONIZED_OP(sc);
+
+	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+
+	return (0);
+}
+
+static int
+cxgbe_nm_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
+{
+	int rc = 0, mtu, flags;
+	struct port_info *pi = ifp->if_softc;
+	struct adapter *sc = pi->adapter;
+	struct ifreq *ifr = (struct ifreq *)data;
+	uint32_t mask;
+
+	MPASS(pi->nm_ifp == ifp);
+
+	switch (cmd) {
+	case SIOCSIFMTU:
+		mtu = ifr->ifr_mtu;
+		if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
+			return (EINVAL);
+
+		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4nmtu");
+		if (rc)
+			return (rc);
+		ifp->if_mtu = mtu;
+		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+			rc = update_mac_settings(ifp, XGMAC_MTU);
+		end_synchronized_op(sc, 0);
+		break;
+
+	case SIOCSIFFLAGS:
+		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4nflg");
+		if (rc)
+			return (rc);
+
+		if (ifp->if_flags & IFF_UP) {
+			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+				flags = pi->nmif_flags;
+				if ((ifp->if_flags ^ flags) &
+				    (IFF_PROMISC | IFF_ALLMULTI)) {
+					rc = update_mac_settings(ifp,
+					    XGMAC_PROMISC | XGMAC_ALLMULTI);
+				}
+			} else
+				rc = cxgbe_nm_init_synchronized(pi);
+			pi->nmif_flags = ifp->if_flags;
+		} else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+			rc = cxgbe_nm_uninit_synchronized(pi);
+		end_synchronized_op(sc, 0);
+		break;
+
+	case SIOCADDMULTI:
+	case SIOCDELMULTI: /* these two are called with a mutex held :-( */
+		rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4nmulti");
+		if (rc)
+			return (rc);
+		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+			rc = update_mac_settings(ifp, XGMAC_MCADDRS);
+		end_synchronized_op(sc, LOCK_HELD);
+		break;
+
+	case SIOCSIFCAP:
+		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
+		if (mask & IFCAP_TXCSUM) {
+			ifp->if_capenable ^= IFCAP_TXCSUM;
+			ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
+		}
+		if (mask & IFCAP_TXCSUM_IPV6) {
+			ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
+			ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
+		}
+		if (mask & IFCAP_RXCSUM)
+			ifp->if_capenable ^= IFCAP_RXCSUM;
+		if (mask & IFCAP_RXCSUM_IPV6)
+			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
+		break;
+
+	case SIOCSIFMEDIA:
+	case SIOCGIFMEDIA:
+		ifmedia_ioctl(ifp, ifr, &pi->nm_media, cmd);
+		break;
+
+	default:
+		rc = ether_ioctl(ifp, cmd, data);
+	}
+
+	return (rc);
+}
+
+static int
+cxgbe_nm_transmit(struct ifnet *ifp, struct mbuf *m)
+{
+
+	m_freem(m);
+	return (0);
+}
+
+static void
+cxgbe_nm_qflush(struct ifnet *ifp)
+{
+
+	return;
+}
+
+static int
+alloc_nm_rxq_hwq(struct port_info *pi, struct sge_nm_rxq *nm_rxq)
+{
+	int rc, cntxt_id;
+	__be32 v;
+	struct adapter *sc = pi->adapter;
+	struct netmap_adapter *na = NA(pi->nm_ifp);
+	struct fw_iq_cmd c;
+
+	MPASS(na != NULL);
+	MPASS(nm_rxq->iq_desc != NULL);
+	MPASS(nm_rxq->fl_desc != NULL);
+
+	bzero(nm_rxq->iq_desc, pi->qsize_rxq * IQ_ESIZE);
+	bzero(nm_rxq->fl_desc, na->num_rx_desc * EQ_ESIZE + spg_len);
+
+	bzero(&c, sizeof(c));
+	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
+	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
+	    V_FW_IQ_CMD_VFN(0));
+	c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
+	    FW_LEN16(c));
+	if (pi->flags & INTR_NM_RXQ) {
+		KASSERT(nm_rxq->intr_idx < sc->intr_count,
+		    ("%s: invalid direct intr_idx %d", __func__,
+		    nm_rxq->intr_idx));
+		v = V_FW_IQ_CMD_IQANDSTINDEX(nm_rxq->intr_idx);
+	} else {
+		CXGBE_UNIMPLEMENTED(__func__);	/* XXXNM: needs review */
+		v = V_FW_IQ_CMD_IQANDSTINDEX(nm_rxq->intr_idx) |
+		    F_FW_IQ_CMD_IQANDST;
+	}
+	c.type_to_iqandstindex = htobe32(v |
+	    V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
+	    V_FW_IQ_CMD_VIID(pi->nm_viid) |
+	    V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
+	c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
+	    F_FW_IQ_CMD_IQGTSMODE |
+	    V_FW_IQ_CMD_IQINTCNTTHRESH(0) |
+	    V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4));
+	c.iqsize = htobe16(pi->qsize_rxq);
+	c.iqaddr = htobe64(nm_rxq->iq_ba);
+	c.iqns_to_fl0congen |=
+	    htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
+		F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
+		(fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0));
+	c.fl0dcaen_to_fl0cidxfthresh =
+	    htobe16(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_64B) |
+		V_FW_IQ_CMD_FL0FBMAX(X_FETCHBURSTMAX_512B));
+	c.fl0size = htobe16(na->num_rx_desc + spg_len / EQ_ESIZE);
+	c.fl0addr = htobe64(nm_rxq->fl_ba);
+
+	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
+	if (rc != 0) {
+		device_printf(sc->dev,
+		    "failed to create netmap ingress queue: %d\n", rc);
+		return (rc);
+	}
+
+	nm_rxq->iq_cidx = 0;
+	MPASS(nm_rxq->iq_sidx == pi->qsize_rxq - spg_len / IQ_ESIZE);
+	nm_rxq->iq_gen = F_RSPD_GEN;
+	nm_rxq->iq_cntxt_id = be16toh(c.iqid);
+	nm_rxq->iq_abs_id = be16toh(c.physiqid);
+	cntxt_id = nm_rxq->iq_cntxt_id - sc->sge.iq_start;
+	if (cntxt_id >= sc->sge.niq) {
+		panic ("%s: nm_rxq->iq_cntxt_id (%d) more than the max (%d)",
+		    __func__, cntxt_id, sc->sge.niq - 1);
+	}
+	sc->sge.iqmap[cntxt_id] = (void *)nm_rxq;
+
+	nm_rxq->fl_cntxt_id = be16toh(c.fl0id);
+	nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0;
+	MPASS(nm_rxq->fl_sidx == na->num_rx_desc);
+	cntxt_id = nm_rxq->fl_cntxt_id - sc->sge.eq_start;
+	if (cntxt_id >= sc->sge.neq) {
+		panic("%s: nm_rxq->fl_cntxt_id (%d) more than the max (%d)",
+		    __func__, cntxt_id, sc->sge.neq - 1);
+	}
+	sc->sge.eqmap[cntxt_id] = (void *)nm_rxq;
+
+	nm_rxq->fl_db_val = F_DBPRIO | V_QID(nm_rxq->fl_cntxt_id) | V_PIDX(0);
+	if (is_t5(sc))
+		nm_rxq->fl_db_val |= F_DBTYPE;
+
+	t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(F_QINTR_CNT_EN) |
+	    V_INGRESSQID(nm_rxq->iq_cntxt_id));
+
+	return (rc);
+}
+
+static int
+free_nm_rxq_hwq(struct port_info *pi, struct sge_nm_rxq *nm_rxq)
+{
+	struct adapter *sc = pi->adapter;
+	int rc;
+
+	rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, FW_IQ_TYPE_FL_INT_CAP,
+	    nm_rxq->iq_cntxt_id, nm_rxq->fl_cntxt_id, 0xffff);
+	if (rc != 0)
+		device_printf(sc->dev, "%s: failed for iq %d, fl %d: %d\n",
+		    __func__, nm_rxq->iq_cntxt_id, nm_rxq->fl_cntxt_id, rc);
+	return (rc);
+}
+
+static int
+alloc_nm_txq_hwq(struct port_info *pi, struct sge_nm_txq *nm_txq)
+{
+	int rc, cntxt_id;
+	size_t len;
+	struct adapter *sc = pi->adapter;
+	struct netmap_adapter *na = NA(pi->nm_ifp);
+	struct fw_eq_eth_cmd c;
+
+	MPASS(na != NULL);
+	MPASS(nm_txq->desc != NULL);
+
+	len = na->num_tx_desc * EQ_ESIZE + spg_len;
+	bzero(nm_txq->desc, len);
+
+	bzero(&c, sizeof(c));
+	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
+	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
+	    V_FW_EQ_ETH_CMD_VFN(0));
+	c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
+	    F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
+	c.autoequiqe_to_viid = htobe32(V_FW_EQ_ETH_CMD_VIID(pi->nm_viid));
+	c.fetchszm_to_iqid =
+	    htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
+		V_FW_EQ_ETH_CMD_PCIECHN(pi->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
+		V_FW_EQ_ETH_CMD_IQID(sc->sge.nm_rxq[nm_txq->iqidx].iq_cntxt_id));
+	c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
+		      V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
+		      V_FW_EQ_ETH_CMD_EQSIZE(len / EQ_ESIZE));
+	c.eqaddr = htobe64(nm_txq->ba);
+
+	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
+	if (rc != 0) {
+		device_printf(pi->dev,
+		    "failed to create netmap egress queue: %d\n", rc);
+		return (rc);
+	}
+
+	nm_txq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd));
+	cntxt_id = nm_txq->cntxt_id - sc->sge.eq_start;
+	if (cntxt_id >= sc->sge.neq)
+	    panic("%s: nm_txq->cntxt_id (%d) more than the max (%d)", __func__,
+		cntxt_id, sc->sge.neq - 1);
+	sc->sge.eqmap[cntxt_id] = (void *)nm_txq;
+
+	nm_txq->pidx = nm_txq->cidx = 0;
+	MPASS(nm_txq->sidx == na->num_tx_desc);
+	nm_txq->equiqidx = nm_txq-> equeqidx = nm_txq->dbidx = 0;
+
+	nm_txq->doorbells = sc->doorbells;
+	if (isset(&nm_txq->doorbells, DOORBELL_UDB) ||
+	    isset(&nm_txq->doorbells, DOORBELL_UDBWC) ||
+	    isset(&nm_txq->doorbells, DOORBELL_WCWR)) {
+		uint32_t s_qpp = sc->sge.eq_s_qpp;
+		uint32_t mask = (1 << s_qpp) - 1;
+		volatile uint8_t *udb;
+
+		udb = sc->udbs_base + UDBS_DB_OFFSET;
+		udb += (nm_txq->cntxt_id >> s_qpp) << PAGE_SHIFT;
+		nm_txq->udb_qid = nm_txq->cntxt_id & mask;
+		if (nm_txq->udb_qid >= PAGE_SIZE / UDBS_SEG_SIZE)
+	    		clrbit(&nm_txq->doorbells, DOORBELL_WCWR);
+		else {
+			udb += nm_txq->udb_qid << UDBS_SEG_SHIFT;
+			nm_txq->udb_qid = 0;
+		}
+		nm_txq->udb = (volatile void *)udb;
+	}
+
+	return (rc);
+}
+
+static int
+free_nm_txq_hwq(struct port_info *pi, struct sge_nm_txq *nm_txq)
+{
+	struct adapter *sc = pi->adapter;
+	int rc;
+
+	rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, nm_txq->cntxt_id);
+	if (rc != 0)
+		device_printf(sc->dev, "%s: failed for eq %d: %d\n", __func__,
+		    nm_txq->cntxt_id, rc);
+	return (rc);
+}
+
+static int
+cxgbe_netmap_on(struct adapter *sc, struct port_info *pi, struct ifnet *ifp,
+    struct netmap_adapter *na)
+{
+	struct netmap_slot *slot;
+	struct sge_nm_rxq *nm_rxq;
+	struct sge_nm_txq *nm_txq;
+	int rc, i, j, hwidx;
+	struct hw_buf_info *hwb;
+	uint16_t *rss;
+
+	ASSERT_SYNCHRONIZED_OP(sc);
+
+	if ((pi->flags & PORT_INIT_DONE) == 0 ||
+	    (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
+		return (EAGAIN);
+
+	hwb = &sc->sge.hw_buf_info[0];
+	for (i = 0; i < SGE_FLBUF_SIZES; i++, hwb++) {
+		if (hwb->size == NETMAP_BUF_SIZE(na))
+			break;
+	}
+	if (i >= SGE_FLBUF_SIZES) {
+		if_printf(ifp, "no hwidx for netmap buffer size %d.\n",
+		    NETMAP_BUF_SIZE(na));
+		return (ENXIO);
+	}
+	hwidx = i;
+
+	/* Must set caps before calling netmap_reset */
+	nm_set_native_flags(na);
+
+	for_each_nm_rxq(pi, i, nm_rxq) {
+		alloc_nm_rxq_hwq(pi, nm_rxq);
+		nm_rxq->fl_hwidx = hwidx;
+		slot = netmap_reset(na, NR_RX, i, 0);
+		MPASS(slot != NULL);	/* XXXNM: error check, not assert */
+
+		/* We deal with 8 bufs at a time */
+		MPASS((na->num_rx_desc & 7) == 0);
+		MPASS(na->num_rx_desc == nm_rxq->fl_sidx);
+		for (j = 0; j < nm_rxq->fl_sidx - 8; j++) {
+			uint64_t ba;
+
+			PNMB(na, &slot[j], &ba);
+			nm_rxq->fl_desc[j] = htobe64(ba | hwidx);
+		}
+		nm_rxq->fl_pidx = j;
+		MPASS((j & 7) == 0);
+		j /= 8;	/* driver pidx to hardware pidx */
+		wmb();
+		t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
+		    nm_rxq->fl_db_val | V_PIDX(j));
+	}
+
+	for_each_nm_txq(pi, i, nm_txq) {
+		alloc_nm_txq_hwq(pi, nm_txq);
+		slot = netmap_reset(na, NR_TX, i, 0);
+		MPASS(slot != NULL);	/* XXXNM: error check, not assert */
+	}
+
+	rss = malloc(pi->nm_rss_size * sizeof (*rss), M_CXGBE, M_ZERO |
+	    M_WAITOK);
+	for (i = 0; i < pi->nm_rss_size;) {
+		for_each_nm_rxq(pi, j, nm_rxq) {
+			rss[i++] = nm_rxq->iq_abs_id;
+			if (i == pi->nm_rss_size)
+				break;
+		}
+	}
+	rc = -t4_config_rss_range(sc, sc->mbox, pi->nm_viid, 0, pi->nm_rss_size,
+	    rss, pi->nm_rss_size);
+	if (rc != 0)
+		if_printf(ifp, "netmap rss_config failed: %d\n", rc);
+	free(rss, M_CXGBE);
+
+	rc = -t4_enable_vi(sc, sc->mbox, pi->nm_viid, true, true);
+	if (rc != 0)
+		if_printf(ifp, "netmap enable_vi failed: %d\n", rc);
+
+	return (rc);
+}
+
+static int
+cxgbe_netmap_off(struct adapter *sc, struct port_info *pi, struct ifnet *ifp,
+    struct netmap_adapter *na)
+{
+	int rc, i;
+	struct sge_nm_txq *nm_txq;
+	struct sge_nm_rxq *nm_rxq;
+
+	ASSERT_SYNCHRONIZED_OP(sc);
+
+	rc = -t4_enable_vi(sc, sc->mbox, pi->nm_viid, false, false);
+	if (rc != 0)
+		if_printf(ifp, "netmap disable_vi failed: %d\n", rc);
+	nm_clear_native_flags(na);
+
+	/*
+	 * XXXNM: We need to make sure that the tx queues are quiet and won't
+	 * request any more SGE_EGR_UPDATEs.
+	 */
+
+	for_each_nm_txq(pi, i, nm_txq) {
+		free_nm_txq_hwq(pi, nm_txq);
+	}
+	for_each_nm_rxq(pi, i, nm_rxq) {
+		free_nm_rxq_hwq(pi, nm_rxq);
+	}
+
+	return (rc);
+}
+
+static int
+cxgbe_netmap_reg(struct netmap_adapter *na, int on)
+{
+	struct ifnet *ifp = na->ifp;
+	struct port_info *pi = ifp->if_softc;
+	struct adapter *sc = pi->adapter;
+	int rc;
+
+	rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4nmreg");
+	if (rc != 0)
+		return (rc);
+	if (on)
+		rc = cxgbe_netmap_on(sc, pi, ifp, na);
+	else
+		rc = cxgbe_netmap_off(sc, pi, ifp, na);
+	end_synchronized_op(sc, 0);
+
+	return (rc);
+}
+
+/* How many packets can a single type1 WR carry in n descriptors */
+static inline int
+ndesc_to_npkt(const int n)
+{
+
+	MPASS(n > 0 && n <= SGE_MAX_WR_NDESC);
+
+	return (n * 2 - 1);
+}
+#define MAX_NPKT_IN_TYPE1_WR	(ndesc_to_npkt(SGE_MAX_WR_NDESC))
+
+/* Space (in descriptors) needed for a type1 WR that carries n packets */
+static inline int
+npkt_to_ndesc(const int n)
+{
+
+	MPASS(n > 0 && n <= MAX_NPKT_IN_TYPE1_WR);
+
+	return ((n + 2) / 2);
+}
+
+/* Space (in 16B units) needed for a type1 WR that carries n packets */
+static inline int
+npkt_to_len16(const int n)
+{
+
+	MPASS(n > 0 && n <= MAX_NPKT_IN_TYPE1_WR);
+
+	return (n * 2 + 1);
+}
+
+#define NMIDXDIFF(q, idx) IDXDIFF((q)->pidx, (q)->idx, (q)->sidx)
+
+static void
+ring_nm_txq_db(struct adapter *sc, struct sge_nm_txq *nm_txq)
+{
+	int n;
+	u_int db = nm_txq->doorbells;
+
+	MPASS(nm_txq->pidx != nm_txq->dbidx);
+
+	n = NMIDXDIFF(nm_txq, dbidx);
+	if (n > 1)
+		clrbit(&db, DOORBELL_WCWR);
+	wmb();
+
+	switch (ffs(db) - 1) {
+	case DOORBELL_UDB:
+		*nm_txq->udb = htole32(V_QID(nm_txq->udb_qid) | V_PIDX(n));
+		break;
+
+	case DOORBELL_WCWR: {
+		volatile uint64_t *dst, *src;
+
+		/*
+		 * Queues whose 128B doorbell segment fits in the page do not
+		 * use relative qid (udb_qid is always 0).  Only queues with
+		 * doorbell segments can do WCWR.
+		 */
+		KASSERT(nm_txq->udb_qid == 0 && n == 1,
+		    ("%s: inappropriate doorbell (0x%x, %d, %d) for nm_txq %p",
+		    __func__, nm_txq->doorbells, n, nm_txq->pidx, nm_txq));
+
+		dst = (volatile void *)((uintptr_t)nm_txq->udb +
+		    UDBS_WR_OFFSET - UDBS_DB_OFFSET);
+		src = (void *)&nm_txq->desc[nm_txq->dbidx];
+		while (src != (void *)&nm_txq->desc[nm_txq->dbidx + 1])
+			*dst++ = *src++;
+		wmb();
+		break;
+	}
+
+	case DOORBELL_UDBWC:
+		*nm_txq->udb = htole32(V_QID(nm_txq->udb_qid) | V_PIDX(n));
+		wmb();
+		break;
+
+	case DOORBELL_KDB:
+		t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
+		    V_QID(nm_txq->cntxt_id) | V_PIDX(n));
+		break;
+	}
+	nm_txq->dbidx = nm_txq->pidx;
+}
+
+int lazy_tx_credit_flush = 1;
+
+/*
+ * Write work requests to send 'npkt' frames and ring the doorbell to send them
+ * on their way.  No need to check for wraparound.
+ */
+static void
+cxgbe_nm_tx(struct adapter *sc, struct sge_nm_txq *nm_txq,
+    struct netmap_kring *kring, int npkt, int npkt_remaining)
+{
+	struct netmap_ring *ring = kring->ring;
+	struct netmap_slot *slot;
+	const u_int lim = kring->nkr_num_slots - 1;
+	struct fw_eth_tx_pkts_wr *wr = (void *)&nm_txq->desc[nm_txq->pidx];
+	uint16_t len;
+	uint64_t ba;
+	struct cpl_tx_pkt_core *cpl;
+	struct ulptx_sgl *usgl;
+	int i, n;
+
+	while (npkt) {
+		n = min(npkt, MAX_NPKT_IN_TYPE1_WR);
+		len = 0;
+
+		wr = (void *)&nm_txq->desc[nm_txq->pidx];
+		wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR));
+		wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(npkt_to_len16(n)));
+		wr->npkt = n;
+		wr->r3 = 0;
+		wr->type = 1;
+		cpl = (void *)(wr + 1);
+
+		for (i = 0; i < n; i++) {
+			slot = &ring->slot[kring->nr_hwcur];
+			PNMB(kring->na, slot, &ba);
+
+			cpl->ctrl0 = nm_txq->cpl_ctrl0;
+			cpl->pack = 0;
+			cpl->len = htobe16(slot->len);
+			/*
+			 * netmap(4) says "netmap does not use features such as
+			 * checksum offloading, TCP segmentation offloading,
+			 * encryption, VLAN encapsulation/decapsulation, etc."
+			 *
+			 * XXXNM: it makes sense to enable checksum offload.
+			 */
+			cpl->ctrl1 = htobe64(F_TXPKT_IPCSUM_DIS |
+			    F_TXPKT_L4CSUM_DIS);
+
+			usgl = (void *)(cpl + 1);
+			usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
+			    V_ULPTX_NSGE(1));
+			usgl->len0 = htobe32(slot->len);
+			usgl->addr0 = htobe64(ba);
+
+			slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
+			cpl = (void *)(usgl + 1);
+			MPASS(slot->len + len <= UINT16_MAX);
+			len += slot->len;
+			kring->nr_hwcur = nm_next(kring->nr_hwcur, lim);
+		}
+		wr->plen = htobe16(len);
+
+		npkt -= n;
+		nm_txq->pidx += npkt_to_ndesc(n);
+		MPASS(nm_txq->pidx <= nm_txq->sidx);
+		if (__predict_false(nm_txq->pidx == nm_txq->sidx)) {
+			/*
+			 * This routine doesn't know how to write WRs that wrap
+			 * around.  Make sure it wasn't asked to.
+			 */
+			MPASS(npkt == 0);
+			nm_txq->pidx = 0;
+		}
+
+		if (npkt == 0 && npkt_remaining == 0) {
+			/* All done. */
+			if (lazy_tx_credit_flush == 0) {
+				wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ |
+				    F_FW_WR_EQUIQ);
+				nm_txq->equeqidx = nm_txq->pidx;
+				nm_txq->equiqidx = nm_txq->pidx;
+			}
+			ring_nm_txq_db(sc, nm_txq);
+			return;
+		}
+
+		if (NMIDXDIFF(nm_txq, equiqidx) >= nm_txq->sidx / 2) {
+			wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ |
+			    F_FW_WR_EQUIQ);
+			nm_txq->equeqidx = nm_txq->pidx;
+			nm_txq->equiqidx = nm_txq->pidx;
+		} else if (NMIDXDIFF(nm_txq, equeqidx) >= 64) {
+			wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ);
+			nm_txq->equeqidx = nm_txq->pidx;
+		}
+		if (NMIDXDIFF(nm_txq, dbidx) >= 2 * SGE_MAX_WR_NDESC)
+			ring_nm_txq_db(sc, nm_txq);
+	}
+
+	/* Will get called again. */
+	MPASS(npkt_remaining);
+}
+
+/* How many contiguous free descriptors starting at pidx */
+static inline int
+contiguous_ndesc_available(struct sge_nm_txq *nm_txq)
+{
+
+	if (nm_txq->cidx > nm_txq->pidx)
+		return (nm_txq->cidx - nm_txq->pidx - 1);
+	else if (nm_txq->cidx > 0)
+		return (nm_txq->sidx - nm_txq->pidx);
+	else
+		return (nm_txq->sidx - nm_txq->pidx - 1);
+}
+
+static int
+reclaim_nm_tx_desc(struct sge_nm_txq *nm_txq)
+{
+	struct sge_qstat *spg = (void *)&nm_txq->desc[nm_txq->sidx];
+	uint16_t hw_cidx = spg->cidx;	/* snapshot */
+	struct fw_eth_tx_pkts_wr *wr;
+	int n = 0;
+
+	hw_cidx = be16toh(hw_cidx);
+
+	while (nm_txq->cidx != hw_cidx) {
+		wr = (void *)&nm_txq->desc[nm_txq->cidx];
+
+		MPASS(wr->op_pkd == htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)));
+		MPASS(wr->type == 1);
+		MPASS(wr->npkt > 0 && wr->npkt <= MAX_NPKT_IN_TYPE1_WR);
+
+		n += wr->npkt;
+		nm_txq->cidx += npkt_to_ndesc(wr->npkt);
+
+		/*
+		 * We never sent a WR that wrapped around so the credits coming
+		 * back, WR by WR, should never cause the cidx to wrap around
+		 * either.
+		 */
+		MPASS(nm_txq->cidx <= nm_txq->sidx);
+		if (__predict_false(nm_txq->cidx == nm_txq->sidx))
+			nm_txq->cidx = 0;
+	}
+
+	return (n);
+}
+
+static int
+cxgbe_netmap_txsync(struct netmap_kring *kring, int flags)
+{
+	struct netmap_adapter *na = kring->na;
+	struct ifnet *ifp = na->ifp;
+	struct port_info *pi = ifp->if_softc;
+	struct adapter *sc = pi->adapter;
+	struct sge_nm_txq *nm_txq = &sc->sge.nm_txq[pi->first_nm_txq + kring->ring_id];
+	const u_int head = kring->rhead;
+	u_int reclaimed = 0;
+	int n, d, npkt_remaining, ndesc_remaining;
+
+	/*
+	 * Tx was at kring->nr_hwcur last time around and now we need to advance
+	 * to kring->rhead.  Note that the driver's pidx moves independent of
+	 * netmap's kring->nr_hwcur (pidx counts descriptors and the relation
+	 * between descriptors and frames isn't 1:1).
+	 */
+
+	npkt_remaining = head >= kring->nr_hwcur ? head - kring->nr_hwcur :
+	    kring->nkr_num_slots - kring->nr_hwcur + head;
+	while (npkt_remaining) {
+		reclaimed += reclaim_nm_tx_desc(nm_txq);
+		ndesc_remaining = contiguous_ndesc_available(nm_txq);
+		/* Can't run out of descriptors with packets still remaining */
+		MPASS(ndesc_remaining > 0);
+
+		/* # of desc needed to tx all remaining packets */
+		d = (npkt_remaining / MAX_NPKT_IN_TYPE1_WR) * SGE_MAX_WR_NDESC;
+		if (npkt_remaining % MAX_NPKT_IN_TYPE1_WR)
+			d += npkt_to_ndesc(npkt_remaining % MAX_NPKT_IN_TYPE1_WR);
+
+		if (d <= ndesc_remaining)
+			n = npkt_remaining;
+		else {
+			/* Can't send all, calculate how many can be sent */
+			n = (ndesc_remaining / SGE_MAX_WR_NDESC) *
+			    MAX_NPKT_IN_TYPE1_WR;
+			if (ndesc_remaining % SGE_MAX_WR_NDESC)
+				n += ndesc_to_npkt(ndesc_remaining % SGE_MAX_WR_NDESC);
+		}
+
+		/* Send n packets and update nm_txq->pidx and kring->nr_hwcur */
+		npkt_remaining -= n;
+		cxgbe_nm_tx(sc, nm_txq, kring, n, npkt_remaining);
+	}
+	MPASS(npkt_remaining == 0);
+	MPASS(kring->nr_hwcur == head);
+	MPASS(nm_txq->dbidx == nm_txq->pidx);
+
+	/*
+	 * Second part: reclaim buffers for completed transmissions.
+	 */
+	if (reclaimed || flags & NAF_FORCE_RECLAIM || nm_kr_txempty(kring)) {
+		reclaimed += reclaim_nm_tx_desc(nm_txq);
+		kring->nr_hwtail += reclaimed;
+		if (kring->nr_hwtail >= kring->nkr_num_slots)
+			kring->nr_hwtail -= kring->nkr_num_slots;
+	}
+
+	nm_txsync_finalize(kring);
+
+	return (0);
+}
+
+static int
+cxgbe_netmap_rxsync(struct netmap_kring *kring, int flags)
+{
+	struct netmap_adapter *na = kring->na;
+	struct netmap_ring *ring = kring->ring;
+	struct ifnet *ifp = na->ifp;
+	struct port_info *pi = ifp->if_softc;
+	struct adapter *sc = pi->adapter;
+	struct sge_nm_rxq *nm_rxq = &sc->sge.nm_rxq[pi->first_nm_rxq + kring->ring_id];
+	u_int const head = nm_rxsync_prologue(kring);
+	u_int n;
+	int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
+
+	if (netmap_no_pendintr || force_update) {
+		kring->nr_hwtail = atomic_load_acq_32(&nm_rxq->fl_cidx);
+		kring->nr_kflags &= ~NKR_PENDINTR;
+	}
+
+	/* Userspace done with buffers from kring->nr_hwcur to head */
+	n = head >= kring->nr_hwcur ? head - kring->nr_hwcur :
+	    kring->nkr_num_slots - kring->nr_hwcur + head;
+	n &= ~7U;
+	if (n > 0) {
+		u_int fl_pidx = nm_rxq->fl_pidx;
+		struct netmap_slot *slot = &ring->slot[fl_pidx];
+		uint64_t ba;
+		int i, dbinc = 0, hwidx = nm_rxq->fl_hwidx;
+
+		/*
+		 * We always deal with 8 buffers at a time.  We must have
+		 * stopped at an 8B boundary (fl_pidx) last time around and we
+		 * must have a multiple of 8B buffers to give to the freelist.
+		 */
+		MPASS((fl_pidx & 7) == 0);
+		MPASS((n & 7) == 0);
+
+		IDXINCR(kring->nr_hwcur, n, kring->nkr_num_slots);
+		IDXINCR(nm_rxq->fl_pidx, n, nm_rxq->fl_sidx);
+
+		while (n > 0) {
+			for (i = 0; i < 8; i++, fl_pidx++, slot++) {
+				PNMB(na, slot, &ba);
+				nm_rxq->fl_desc[fl_pidx] = htobe64(ba | hwidx);
+				slot->flags &= ~NS_BUF_CHANGED;
+				MPASS(fl_pidx <= nm_rxq->fl_sidx);
+			}
+			n -= 8;
+			if (fl_pidx == nm_rxq->fl_sidx) {
+				fl_pidx = 0;
+				slot = &ring->slot[0];
+			}
+			if (++dbinc == 8 && n >= 32) {
+				wmb();
+				t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
+				    nm_rxq->fl_db_val | V_PIDX(dbinc));
+				dbinc = 0;
+			}
+		}
+		MPASS(nm_rxq->fl_pidx == fl_pidx);
+
+		if (dbinc > 0) {
+			wmb();
+			t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
+			    nm_rxq->fl_db_val | V_PIDX(dbinc));
+		}
+	}
+
+	nm_rxsync_finalize(kring);
+
+	return (0);
+}
+
+/*
+ * Create an ifnet solely for netmap use and register it with the kernel.
+ */
+int
+create_netmap_ifnet(struct port_info *pi)
+{
+	struct adapter *sc = pi->adapter;
+	struct netmap_adapter na;
+	struct ifnet *ifp;
+	device_t dev = pi->dev;
+	uint8_t mac[ETHER_ADDR_LEN];
+	int rc;
+
+	if (pi->nnmtxq <= 0 || pi->nnmrxq <= 0)
+		return (0);
+	MPASS(pi->nm_ifp == NULL);
+
+	/*
+	 * Allocate a virtual interface exclusively for netmap use.  Give it the
+	 * MAC address normally reserved for use by a TOE interface.  (The TOE
+	 * driver on FreeBSD doesn't use it).
+	 */
+	rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1, &mac[0],
+	    &pi->nm_rss_size, FW_VI_FUNC_OFLD, 0);
+	if (rc < 0) {
+		device_printf(dev, "unable to allocate netmap virtual "
+		    "interface for port %d: %d\n", pi->port_id, -rc);
+		return (-rc);
+	}
+	pi->nm_viid = rc;
+	pi->nm_xact_addr_filt = -1;
+
+	ifp = if_alloc(IFT_ETHER);
+	if (ifp == NULL) {
+		device_printf(dev, "Cannot allocate netmap ifnet\n");
+		return (ENOMEM);
+	}
+	pi->nm_ifp = ifp;
+	ifp->if_softc = pi;
+
+	if_initname(ifp, is_t4(pi->adapter) ? "ncxgbe" : "ncxl",
+	    device_get_unit(dev));
+	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
+
+	ifp->if_init = cxgbe_nm_init;
+	ifp->if_ioctl = cxgbe_nm_ioctl;
+	ifp->if_transmit = cxgbe_nm_transmit;
+	ifp->if_qflush = cxgbe_nm_qflush;
+
+	/*
+	 * netmap(4) says "netmap does not use features such as checksum
+	 * offloading, TCP segmentation offloading, encryption, VLAN
+	 * encapsulation/decapsulation, etc."
+	 *
+	 * By default we comply with the statement above.  But we do declare the
+	 * ifnet capable of L3/L4 checksumming so that a user can override
+	 * netmap and have the hardware do the L3/L4 checksums.
+	 */
+	ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_JUMBO_MTU |
+	    IFCAP_HWCSUM_IPV6;
+	ifp->if_capenable = 0;
+	ifp->if_hwassist = 0;
+
+	/* nm_media has already been setup by the caller */
+
+	ether_ifattach(ifp, mac);
+
+	/*
+	 * Register with netmap in the kernel.
+	 */
+	bzero(&na, sizeof(na));
+
+	na.ifp = pi->nm_ifp;
+	na.na_flags = NAF_BDG_MAYSLEEP;
+
+	/* Netmap doesn't know about the space reserved for the status page. */
+	na.num_tx_desc = pi->qsize_txq - spg_len / EQ_ESIZE;
+
+	/*
+	 * The freelist's cidx/pidx drives netmap's rx cidx/pidx.  So
+	 * num_rx_desc is based on the number of buffers that can be held in the
+	 * freelist, and not the number of entries in the iq.  (These two are
+	 * not exactly the same due to the space taken up by the status page).
+	 */
+	na.num_rx_desc = (pi->qsize_rxq / 8) * 8;
+	na.nm_txsync = cxgbe_netmap_txsync;
+	na.nm_rxsync = cxgbe_netmap_rxsync;
+	na.nm_register = cxgbe_netmap_reg;
+	na.num_tx_rings = pi->nnmtxq;
+	na.num_rx_rings = pi->nnmrxq;
+	netmap_attach(&na);	/* This adds IFCAP_NETMAP to if_capabilities */
+
+	return (0);
+}
+
+int
+destroy_netmap_ifnet(struct port_info *pi)
+{
+	struct adapter *sc = pi->adapter;
+
+	if (pi->nm_ifp == NULL)
+		return (0);
+
+	netmap_detach(pi->nm_ifp);
+	ifmedia_removeall(&pi->nm_media);
+	ether_ifdetach(pi->nm_ifp);
+	if_free(pi->nm_ifp);
+	t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->nm_viid);
+
+	return (0);
+}
+
+static void
+handle_nm_fw6_msg(struct adapter *sc, struct ifnet *ifp,
+    const struct cpl_fw6_msg *cpl)
+{
+	const struct cpl_sge_egr_update *egr;
+	uint32_t oq;
+	struct sge_nm_txq *nm_txq;
+
+	if (cpl->type != FW_TYPE_RSSCPL && cpl->type != FW6_TYPE_RSSCPL)
+		panic("%s: FW_TYPE 0x%x on nm_rxq.", __func__, cpl->type);
+
+	/* data[0] is RSS header */
+	egr = (const void *)&cpl->data[1];
+	oq = be32toh(egr->opcode_qid);
+	MPASS(G_CPL_OPCODE(oq) == CPL_SGE_EGR_UPDATE);
+	nm_txq = (void *)sc->sge.eqmap[G_EGR_QID(oq) - sc->sge.eq_start];
+
+	netmap_tx_irq(ifp, nm_txq->nid);
+}
+
+void
+t4_nm_intr(void *arg)
+{
+	struct sge_nm_rxq *nm_rxq = arg;
+	struct port_info *pi = nm_rxq->pi;
+	struct adapter *sc = pi->adapter;
+	struct ifnet *ifp = pi->nm_ifp;
+	struct netmap_adapter *na = NA(ifp);
+	struct netmap_kring *kring = &na->rx_rings[nm_rxq->nid];
+	struct netmap_ring *ring = kring->ring;
+	struct iq_desc *d = &nm_rxq->iq_desc[nm_rxq->iq_cidx];
+	uint32_t lq;
+	u_int n = 0;
+	int processed = 0;
+	uint8_t opcode;
+	uint32_t fl_cidx = atomic_load_acq_32(&nm_rxq->fl_cidx);
+
+	while ((d->rsp.u.type_gen & F_RSPD_GEN) == nm_rxq->iq_gen) {
+
+		rmb();
+
+		lq = be32toh(d->rsp.pldbuflen_qid);
+		opcode = d->rss.opcode;
+
+		switch (G_RSPD_TYPE(d->rsp.u.type_gen)) {
+		case X_RSPD_TYPE_FLBUF:
+			/* No buffer packing so new buf every time */
+			MPASS(lq & F_RSPD_NEWBUF);
+
+			/* fall through */
+
+		case X_RSPD_TYPE_CPL:
+			MPASS(opcode < NUM_CPL_CMDS);
+
+			switch (opcode) {
+			case CPL_FW4_MSG:
+			case CPL_FW6_MSG:
+				handle_nm_fw6_msg(sc, ifp,
+				    (const void *)&d->cpl[0]);
+				break;
+			case CPL_RX_PKT:
+				ring->slot[fl_cidx].len = G_RSPD_LEN(lq) - fl_pktshift;
+				ring->slot[fl_cidx].flags = kring->nkr_slot_flags;
+				if (__predict_false(++fl_cidx == nm_rxq->fl_sidx))
+					fl_cidx = 0;
+				break;
+			default:
+				panic("%s: unexpected opcode 0x%x on nm_rxq %p",
+				    __func__, opcode, nm_rxq);
+			}
+			break;
+
+		case X_RSPD_TYPE_INTR:
+			/* Not equipped to handle forwarded interrupts. */
+			panic("%s: netmap queue received interrupt for iq %u\n",
+			    __func__, lq);
+
+		default:
+			panic("%s: illegal response type %d on nm_rxq %p",
+			    __func__, G_RSPD_TYPE(d->rsp.u.type_gen), nm_rxq);
+		}
+
+		d++;
+		if (__predict_false(++nm_rxq->iq_cidx == nm_rxq->iq_sidx)) {
+			nm_rxq->iq_cidx = 0;
+			d = &nm_rxq->iq_desc[0];
+			nm_rxq->iq_gen ^= F_RSPD_GEN;
+		}
+
+		if (__predict_false(++n == 64)) {	/* XXXNM: tune */
+			t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
+			    V_CIDXINC(n) | V_INGRESSQID(nm_rxq->iq_cntxt_id) |
+			    V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
+			n = 0;
+		}
+	}
+	if (fl_cidx != nm_rxq->fl_cidx) {
+		atomic_store_rel_32(&nm_rxq->fl_cidx, fl_cidx);
+		netmap_rx_irq(ifp, nm_rxq->nid, &processed);
+	}
+	t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(n) |
+	    V_INGRESSQID((u32)nm_rxq->iq_cntxt_id) | V_SEINTARM(F_QINTR_CNT_EN));
+}
+#endif

Property changes on: stable/10/sys/dev/cxgbe/t4_netmap.c
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+FreeBSD=%H
\ No newline at end of property
Added: svn:mime-type
## -0,0 +1 ##
+text/plain
\ No newline at end of property
Index: stable/10/sys/dev/cxgbe/t4_sge.c
===================================================================
--- stable/10/sys/dev/cxgbe/t4_sge.c	(revision 270296)
+++ stable/10/sys/dev/cxgbe/t4_sge.c	(revision 270297)
@@ -1,4337 +1,4559 @@
 /*-
  * Copyright (c) 2011 Chelsio Communications, Inc.
  * All rights reserved.
  * Written by: Navdeep Parhar <np@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
 #include "opt_inet6.h"
 
 #include <sys/types.h>
 #include <sys/mbuf.h>
 #include <sys/socket.h>
 #include <sys/kernel.h>
 #include <sys/kdb.h>
 #include <sys/malloc.h>
 #include <sys/queue.h>
 #include <sys/sbuf.h>
 #include <sys/taskqueue.h>
 #include <sys/time.h>
 #include <sys/sysctl.h>
 #include <sys/smp.h>
 #include <sys/counter.h>
 #include <net/bpf.h>
 #include <net/ethernet.h>
 #include <net/if.h>
 #include <net/if_vlan_var.h>
 #include <netinet/in.h>
 #include <netinet/ip.h>
 #include <netinet/ip6.h>
 #include <netinet/tcp.h>
 #include <machine/md_var.h>
 #include <vm/vm.h>
 #include <vm/pmap.h>
+#ifdef DEV_NETMAP
+#include <machine/bus.h>
+#include <sys/selinfo.h>
+#include <net/if_var.h>
+#include <net/netmap.h>
+#include <dev/netmap/netmap_kern.h>
+#endif
 
 #include "common/common.h"
 #include "common/t4_regs.h"
 #include "common/t4_regs_values.h"
 #include "common/t4_msg.h"
 
 #ifdef T4_PKT_TIMESTAMP
 #define RX_COPY_THRESHOLD (MINCLSIZE - 8)
 #else
 #define RX_COPY_THRESHOLD MINCLSIZE
 #endif
 
 /*
  * Ethernet frames are DMA'd at this byte offset into the freelist buffer.
  * 0-7 are valid values.
  */
-static int fl_pktshift = 2;
+int fl_pktshift = 2;
 TUNABLE_INT("hw.cxgbe.fl_pktshift", &fl_pktshift);
 
 /*
  * Pad ethernet payload up to this boundary.
  * -1: driver should figure out a good value.
  *  0: disable padding.
  *  Any power of 2 from 32 to 4096 (both inclusive) is also a valid value.
  */
-static int fl_pad = -1;
+int fl_pad = -1;
 TUNABLE_INT("hw.cxgbe.fl_pad", &fl_pad);
 
 /*
  * Status page length.
  * -1: driver should figure out a good value.
  *  64 or 128 are the only other valid values.
  */
-static int spg_len = -1;
+int spg_len = -1;
 TUNABLE_INT("hw.cxgbe.spg_len", &spg_len);
 
 /*
  * Congestion drops.
  * -1: no congestion feedback (not recommended).
  *  0: backpressure the channel instead of dropping packets right away.
  *  1: no backpressure, drop packets for the congested queue immediately.
  */
 static int cong_drop = 0;
 TUNABLE_INT("hw.cxgbe.cong_drop", &cong_drop);
 
 /*
  * Deliver multiple frames in the same free list buffer if they fit.
  * -1: let the driver decide whether to enable buffer packing or not.
  *  0: disable buffer packing.
  *  1: enable buffer packing.
  */
 static int buffer_packing = -1;
 TUNABLE_INT("hw.cxgbe.buffer_packing", &buffer_packing);
 
 /*
  * Start next frame in a packed buffer at this boundary.
  * -1: driver should figure out a good value.
  * T4:
  * ---
  * if fl_pad != 0
  * 	value specified here will be overridden by fl_pad.
  * else
  * 	power of 2 from 32 to 4096 (both inclusive) is a valid value here.
  * T5:
  * ---
  * 16, or a power of 2 from 64 to 4096 (both inclusive) is a valid value.
  */
 static int fl_pack = -1;
 static int t4_fl_pack;
 static int t5_fl_pack;
 TUNABLE_INT("hw.cxgbe.fl_pack", &fl_pack);
 
 /*
  * Allow the driver to create mbuf(s) in a cluster allocated for rx.
  * 0: never; always allocate mbufs from the zone_mbuf UMA zone.
  * 1: ok to create mbuf(s) within a cluster if there is room.
  */
 static int allow_mbufs_in_cluster = 1;
 TUNABLE_INT("hw.cxgbe.allow_mbufs_in_cluster", &allow_mbufs_in_cluster);
 
 /*
  * Largest rx cluster size that the driver is allowed to allocate.
  */
 static int largest_rx_cluster = MJUM16BYTES;
 TUNABLE_INT("hw.cxgbe.largest_rx_cluster", &largest_rx_cluster);
 
 /*
  * Size of cluster allocation that's most likely to succeed.  The driver will
  * fall back to this size if it fails to allocate clusters larger than this.
  */
 static int safest_rx_cluster = PAGE_SIZE;
 TUNABLE_INT("hw.cxgbe.safest_rx_cluster", &safest_rx_cluster);
 
 /* Used to track coalesced tx work request */
 struct txpkts {
 	uint64_t *flitp;	/* ptr to flit where next pkt should start */
 	uint8_t npkt;		/* # of packets in this work request */
 	uint8_t nflits;		/* # of flits used by this work request */
 	uint16_t plen;		/* total payload (sum of all packets) */
 };
 
 /* A packet's SGL.  This + m_pkthdr has all info needed for tx */
 struct sgl {
 	int nsegs;		/* # of segments in the SGL, 0 means imm. tx */
 	int nflits;		/* # of flits needed for the SGL */
 	bus_dma_segment_t seg[TX_SGL_SEGS];
 };
 
 static int service_iq(struct sge_iq *, int);
-static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t,
-    int *);
+static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t);
 static int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf *);
-static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int,
-    int);
+static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int);
 static inline void init_fl(struct adapter *, struct sge_fl *, int, int, int,
     char *);
 static inline void init_eq(struct sge_eq *, int, int, uint8_t, uint16_t,
     char *);
 static int alloc_ring(struct adapter *, size_t, bus_dma_tag_t *, bus_dmamap_t *,
     bus_addr_t *, void **);
 static int free_ring(struct adapter *, bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
     void *);
 static int alloc_iq_fl(struct port_info *, struct sge_iq *, struct sge_fl *,
     int, int);
 static int free_iq_fl(struct port_info *, struct sge_iq *, struct sge_fl *);
 static void add_fl_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
     struct sge_fl *);
 static int alloc_fwq(struct adapter *);
 static int free_fwq(struct adapter *);
 static int alloc_mgmtq(struct adapter *);
 static int free_mgmtq(struct adapter *);
 static int alloc_rxq(struct port_info *, struct sge_rxq *, int, int,
     struct sysctl_oid *);
 static int free_rxq(struct port_info *, struct sge_rxq *);
 #ifdef TCP_OFFLOAD
 static int alloc_ofld_rxq(struct port_info *, struct sge_ofld_rxq *, int, int,
     struct sysctl_oid *);
 static int free_ofld_rxq(struct port_info *, struct sge_ofld_rxq *);
 #endif
+#ifdef DEV_NETMAP
+static int alloc_nm_rxq(struct port_info *, struct sge_nm_rxq *, int, int,
+    struct sysctl_oid *);
+static int free_nm_rxq(struct port_info *, struct sge_nm_rxq *);
+static int alloc_nm_txq(struct port_info *, struct sge_nm_txq *, int, int,
+    struct sysctl_oid *);
+static int free_nm_txq(struct port_info *, struct sge_nm_txq *);
+#endif
 static int ctrl_eq_alloc(struct adapter *, struct sge_eq *);
 static int eth_eq_alloc(struct adapter *, struct port_info *, struct sge_eq *);
 #ifdef TCP_OFFLOAD
 static int ofld_eq_alloc(struct adapter *, struct port_info *, struct sge_eq *);
 #endif
 static int alloc_eq(struct adapter *, struct port_info *, struct sge_eq *);
 static int free_eq(struct adapter *, struct sge_eq *);
 static int alloc_wrq(struct adapter *, struct port_info *, struct sge_wrq *,
     struct sysctl_oid *);
 static int free_wrq(struct adapter *, struct sge_wrq *);
 static int alloc_txq(struct port_info *, struct sge_txq *, int,
     struct sysctl_oid *);
 static int free_txq(struct port_info *, struct sge_txq *);
 static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int);
-static inline bool is_new_response(const struct sge_iq *, struct rsp_ctrl **);
-static inline void iq_next(struct sge_iq *);
 static inline void ring_fl_db(struct adapter *, struct sge_fl *);
 static int refill_fl(struct adapter *, struct sge_fl *, int);
 static void refill_sfl(void *);
 static int alloc_fl_sdesc(struct sge_fl *);
 static void free_fl_sdesc(struct adapter *, struct sge_fl *);
 static void find_best_refill_source(struct adapter *, struct sge_fl *, int);
 static void find_safe_refill_source(struct adapter *, struct sge_fl *);
 static void add_fl_to_sfl(struct adapter *, struct sge_fl *);
 
 static int get_pkt_sgl(struct sge_txq *, struct mbuf **, struct sgl *, int);
 static int free_pkt_sgl(struct sge_txq *, struct sgl *);
 static int write_txpkt_wr(struct port_info *, struct sge_txq *, struct mbuf *,
     struct sgl *);
 static int add_to_txpkts(struct port_info *, struct sge_txq *, struct txpkts *,
     struct mbuf *, struct sgl *);
 static void write_txpkts_wr(struct sge_txq *, struct txpkts *);
 static inline void write_ulp_cpl_sgl(struct port_info *, struct sge_txq *,
     struct txpkts *, struct mbuf *, struct sgl *);
 static int write_sgl_to_txd(struct sge_eq *, struct sgl *, caddr_t *);
 static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int);
 static inline void ring_eq_db(struct adapter *, struct sge_eq *);
 static inline int reclaimable(struct sge_eq *);
 static int reclaim_tx_descs(struct sge_txq *, int, int);
 static void write_eqflush_wr(struct sge_eq *);
 static __be64 get_flit(bus_dma_segment_t *, int, int);
 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *,
     struct mbuf *);
 static int handle_fw_msg(struct sge_iq *, const struct rss_header *,
     struct mbuf *);
 
 static int sysctl_uint16(SYSCTL_HANDLER_ARGS);
 static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS);
 
 static counter_u64_t extfree_refs;
 static counter_u64_t extfree_rels;
 
 /*
  * Called on MOD_LOAD.  Validates and calculates the SGE tunables.
  */
 void
 t4_sge_modload(void)
 {
 	int pad;
 
 	/* set pad to a reasonable powerof2 between 16 and 4096 (inclusive) */
 #if defined(__i386__) || defined(__amd64__)
 	pad = max(cpu_clflush_line_size, 16);
 #else
 	pad = max(CACHE_LINE_SIZE, 16);
 #endif
 	pad = min(pad, 4096);
 
 	if (fl_pktshift < 0 || fl_pktshift > 7) {
 		printf("Invalid hw.cxgbe.fl_pktshift value (%d),"
 		    " using 2 instead.\n", fl_pktshift);
 		fl_pktshift = 2;
 	}
 
 	if (fl_pad != 0 &&
 	    (fl_pad < 32 || fl_pad > 4096 || !powerof2(fl_pad))) {
 
 		if (fl_pad != -1) {
 			printf("Invalid hw.cxgbe.fl_pad value (%d),"
 			    " using %d instead.\n", fl_pad, max(pad, 32));
 		}
 		fl_pad = max(pad, 32);
 	}
 
 	/*
 	 * T4 has the same pad and pack boundary.  If a pad boundary is set,
 	 * pack boundary must be set to the same value.  Otherwise take the
 	 * specified value or auto-calculate something reasonable.
 	 */
 	if (fl_pad)
 		t4_fl_pack = fl_pad;
 	else if (fl_pack < 32 || fl_pack > 4096 || !powerof2(fl_pack))
 		t4_fl_pack = max(pad, 32);
 	else
 		t4_fl_pack = fl_pack;
 
 	/* T5's pack boundary is independent of the pad boundary. */
 	if (fl_pack < 16 || fl_pack == 32 || fl_pack > 4096 ||
 	    !powerof2(fl_pack))
 	       t5_fl_pack = max(pad, CACHE_LINE_SIZE);
 	else
 	       t5_fl_pack = fl_pack;
 
 	if (spg_len != 64 && spg_len != 128) {
 		int len;
 
 #if defined(__i386__) || defined(__amd64__)
 		len = cpu_clflush_line_size > 64 ? 128 : 64;
 #else
 		len = 64;
 #endif
 		if (spg_len != -1) {
 			printf("Invalid hw.cxgbe.spg_len value (%d),"
 			    " using %d instead.\n", spg_len, len);
 		}
 		spg_len = len;
 	}
 
 	if (cong_drop < -1 || cong_drop > 1) {
 		printf("Invalid hw.cxgbe.cong_drop value (%d),"
 		    " using 0 instead.\n", cong_drop);
 		cong_drop = 0;
 	}
 
 	extfree_refs = counter_u64_alloc(M_WAITOK);
 	extfree_rels = counter_u64_alloc(M_WAITOK);
 	counter_u64_zero(extfree_refs);
 	counter_u64_zero(extfree_rels);
 }
 
 void
 t4_sge_modunload(void)
 {
 
 	counter_u64_free(extfree_refs);
 	counter_u64_free(extfree_rels);
 }
 
 uint64_t
 t4_sge_extfree_refs(void)
 {
 	uint64_t refs, rels;
 
 	rels = counter_u64_fetch(extfree_rels);
 	refs = counter_u64_fetch(extfree_refs);
 
 	return (refs - rels);
 }
 
 void
 t4_init_sge_cpl_handlers(struct adapter *sc)
 {
 
 	t4_register_cpl_handler(sc, CPL_FW4_MSG, handle_fw_msg);
 	t4_register_cpl_handler(sc, CPL_FW6_MSG, handle_fw_msg);
 	t4_register_cpl_handler(sc, CPL_SGE_EGR_UPDATE, handle_sge_egr_update);
 	t4_register_cpl_handler(sc, CPL_RX_PKT, t4_eth_rx);
 	t4_register_fw_msg_handler(sc, FW6_TYPE_CMD_RPL, t4_handle_fw_rpl);
 }
 
 /*
  * adap->params.vpd.cclk must be set up before this is called.
  */
 void
 t4_tweak_chip_settings(struct adapter *sc)
 {
 	int i;
 	uint32_t v, m;
 	int intr_timer[SGE_NTIMERS] = {1, 5, 10, 50, 100, 200};
 	int timer_max = M_TIMERVALUE0 * 1000 / sc->params.vpd.cclk;
 	int intr_pktcount[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */
 	uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
 	static int sge_flbuf_sizes[] = {
 		MCLBYTES,
 #if MJUMPAGESIZE != MCLBYTES
 		MJUMPAGESIZE,
 		MJUMPAGESIZE - CL_METADATA_SIZE,
 		MJUMPAGESIZE - 2 * MSIZE - CL_METADATA_SIZE,
 #endif
 		MJUM9BYTES,
 		MJUM16BYTES,
 		MCLBYTES - MSIZE - CL_METADATA_SIZE,
 		MJUM9BYTES - CL_METADATA_SIZE,
 		MJUM16BYTES - CL_METADATA_SIZE,
 	};
 
 	KASSERT(sc->flags & MASTER_PF,
 	    ("%s: trying to change chip settings when not master.", __func__));
 
 	m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE | F_EGRSTATUSPAGESIZE;
 	v = V_PKTSHIFT(fl_pktshift) | F_RXPKTCPLMODE |
 	    V_EGRSTATUSPAGESIZE(spg_len == 128);
 	if (is_t4(sc) && (fl_pad || buffer_packing)) {
 		/* t4_fl_pack has the correct value even when fl_pad = 0 */
 		m |= V_INGPADBOUNDARY(M_INGPADBOUNDARY);
 		v |= V_INGPADBOUNDARY(ilog2(t4_fl_pack) - 5);
 	} else if (is_t5(sc) && fl_pad) {
 		m |= V_INGPADBOUNDARY(M_INGPADBOUNDARY);
 		v |= V_INGPADBOUNDARY(ilog2(fl_pad) - 5);
 	}
 	t4_set_reg_field(sc, A_SGE_CONTROL, m, v);
 
 	if (is_t5(sc) && buffer_packing) {
 		m = V_INGPACKBOUNDARY(M_INGPACKBOUNDARY);
 		if (t5_fl_pack == 16)
 			v = V_INGPACKBOUNDARY(0);
 		else
 			v = V_INGPACKBOUNDARY(ilog2(t5_fl_pack) - 5);
 		t4_set_reg_field(sc, A_SGE_CONTROL2, m, v);
 	}
 
 	v = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF3(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF4(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF5(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF6(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF7(PAGE_SHIFT - 10);
 	t4_write_reg(sc, A_SGE_HOST_PAGE_SIZE, v);
 
 	KASSERT(nitems(sge_flbuf_sizes) <= SGE_FLBUF_SIZES,
 	    ("%s: hw buffer size table too big", __func__));
 	for (i = 0; i < min(nitems(sge_flbuf_sizes), SGE_FLBUF_SIZES); i++) {
 		t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0 + (4 * i),
 		    sge_flbuf_sizes[i]);
 	}
 
 	v = V_THRESHOLD_0(intr_pktcount[0]) | V_THRESHOLD_1(intr_pktcount[1]) |
 	    V_THRESHOLD_2(intr_pktcount[2]) | V_THRESHOLD_3(intr_pktcount[3]);
 	t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, v);
 
 	KASSERT(intr_timer[0] <= timer_max,
 	    ("%s: not a single usable timer (%d, %d)", __func__, intr_timer[0],
 	    timer_max));
 	for (i = 1; i < nitems(intr_timer); i++) {
 		KASSERT(intr_timer[i] >= intr_timer[i - 1],
 		    ("%s: timers not listed in increasing order (%d)",
 		    __func__, i));
 
 		while (intr_timer[i] > timer_max) {
 			if (i == nitems(intr_timer) - 1) {
 				intr_timer[i] = timer_max;
 				break;
 			}
 			intr_timer[i] += intr_timer[i - 1];
 			intr_timer[i] /= 2;
 		}
 	}
 
 	v = V_TIMERVALUE0(us_to_core_ticks(sc, intr_timer[0])) |
 	    V_TIMERVALUE1(us_to_core_ticks(sc, intr_timer[1]));
 	t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, v);
 	v = V_TIMERVALUE2(us_to_core_ticks(sc, intr_timer[2])) |
 	    V_TIMERVALUE3(us_to_core_ticks(sc, intr_timer[3]));
 	t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, v);
 	v = V_TIMERVALUE4(us_to_core_ticks(sc, intr_timer[4])) |
 	    V_TIMERVALUE5(us_to_core_ticks(sc, intr_timer[5]));
 	t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, v);
 
 	if (cong_drop == 0) {
 		m = F_TUNNELCNGDROP0 | F_TUNNELCNGDROP1 | F_TUNNELCNGDROP2 |
 		    F_TUNNELCNGDROP3;
 		t4_set_reg_field(sc, A_TP_PARA_REG3, m, 0);
 	}
 
 	/* 4K, 16K, 64K, 256K DDP "page sizes" */
 	v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6);
 	t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, v);
 
 	m = v = F_TDDPTAGTCB;
 	t4_set_reg_field(sc, A_ULP_RX_CTL, m, v);
 
 	m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET |
 	    F_RESETDDPOFFSET;
 	v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET;
 	t4_set_reg_field(sc, A_TP_PARA_REG5, m, v);
 }
 
 /*
  * SGE wants the buffer to be at least 64B and then a multiple of the pad
  * boundary or 16, whichever is greater.
  */
 static inline int
 hwsz_ok(int hwsz)
 {
 	int mask = max(fl_pad, 16) - 1;
 
 	return (hwsz >= 64 && (hwsz & mask) == 0);
 }
 
 /*
  * XXX: driver really should be able to deal with unexpected settings.
  */
 int
 t4_read_chip_settings(struct adapter *sc)
 {
 	struct sge *s = &sc->sge;
 	int i, j, n, rc = 0;
 	uint32_t m, v, r;
 	uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
 	static int sw_buf_sizes[] = {	/* Sorted by size */
 		MCLBYTES,
 #if MJUMPAGESIZE != MCLBYTES
 		MJUMPAGESIZE,
 #endif
 		MJUM9BYTES,
 		MJUM16BYTES
 	};
 	struct sw_zone_info *swz, *safe_swz;
 	struct hw_buf_info *hwb;
 
 	m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE | F_EGRSTATUSPAGESIZE;
 	v = V_PKTSHIFT(fl_pktshift) | F_RXPKTCPLMODE |
 	    V_EGRSTATUSPAGESIZE(spg_len == 128);
 	if (is_t4(sc) && (fl_pad || buffer_packing)) {
 		m |= V_INGPADBOUNDARY(M_INGPADBOUNDARY);
 		v |= V_INGPADBOUNDARY(ilog2(t4_fl_pack) - 5);
 	} else if (is_t5(sc) && fl_pad) {
 		m |= V_INGPADBOUNDARY(M_INGPADBOUNDARY);
 		v |= V_INGPADBOUNDARY(ilog2(fl_pad) - 5);
 	}
 	r = t4_read_reg(sc, A_SGE_CONTROL);
 	if ((r & m) != v) {
 		device_printf(sc->dev, "invalid SGE_CONTROL(0x%x)\n", r);
 		rc = EINVAL;
 	}
 
 	if (is_t5(sc) && buffer_packing) {
 		m = V_INGPACKBOUNDARY(M_INGPACKBOUNDARY);
 		if (t5_fl_pack == 16)
 			v = V_INGPACKBOUNDARY(0);
 		else
 			v = V_INGPACKBOUNDARY(ilog2(t5_fl_pack) - 5);
 		r = t4_read_reg(sc, A_SGE_CONTROL2);
 		if ((r & m) != v) {
 			device_printf(sc->dev,
 			    "invalid SGE_CONTROL2(0x%x)\n", r);
 			rc = EINVAL;
 		}
 	}
 	s->pack_boundary = is_t4(sc) ? t4_fl_pack : t5_fl_pack;
 
 	v = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF3(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF4(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF5(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF6(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF7(PAGE_SHIFT - 10);
 	r = t4_read_reg(sc, A_SGE_HOST_PAGE_SIZE);
 	if (r != v) {
 		device_printf(sc->dev, "invalid SGE_HOST_PAGE_SIZE(0x%x)\n", r);
 		rc = EINVAL;
 	}
 
 	/* Filter out unusable hw buffer sizes entirely (mark with -2). */
 	hwb = &s->hw_buf_info[0];
 	for (i = 0; i < nitems(s->hw_buf_info); i++, hwb++) {
 		r = t4_read_reg(sc, A_SGE_FL_BUFFER_SIZE0 + (4 * i));
 		hwb->size = r;
 		hwb->zidx = hwsz_ok(r) ? -1 : -2;
 		hwb->next = -1;
 	}
 
 	/*
 	 * Create a sorted list in decreasing order of hw buffer sizes (and so
 	 * increasing order of spare area) for each software zone.
 	 */
 	n = 0;	/* no usable buffer size to begin with */
 	swz = &s->sw_zone_info[0];
 	safe_swz = NULL;
 	for (i = 0; i < SW_ZONE_SIZES; i++, swz++) {
 		int8_t head = -1, tail = -1;
 
 		swz->size = sw_buf_sizes[i];
 		swz->zone = m_getzone(swz->size);
 		swz->type = m_gettype(swz->size);
 
 		if (swz->size == safest_rx_cluster)
 			safe_swz = swz;
 
 		hwb = &s->hw_buf_info[0];
 		for (j = 0; j < SGE_FLBUF_SIZES; j++, hwb++) {
 			if (hwb->zidx != -1 || hwb->size > swz->size)
 				continue;
 			hwb->zidx = i;
 			if (head == -1)
 				head = tail = j;
 			else if (hwb->size < s->hw_buf_info[tail].size) {
 				s->hw_buf_info[tail].next = j;
 				tail = j;
 			} else {
 				int8_t *cur;
 				struct hw_buf_info *t;
 
 				for (cur = &head; *cur != -1; cur = &t->next) {
 					t = &s->hw_buf_info[*cur];
 					if (hwb->size == t->size) {
 						hwb->zidx = -2;
 						break;
 					}
 					if (hwb->size > t->size) {
 						hwb->next = *cur;
 						*cur = j;
 						break;
 					}
 				}
 			}
 		}
 		swz->head_hwidx = head;
 		swz->tail_hwidx = tail;
 
 		if (tail != -1) {
 			n++;
 			if (swz->size - s->hw_buf_info[tail].size >=
 			    CL_METADATA_SIZE)
 				sc->flags |= BUF_PACKING_OK;
 		}
 	}
 	if (n == 0) {
 		device_printf(sc->dev, "no usable SGE FL buffer size.\n");
 		rc = EINVAL;
 	}
 
 	s->safe_hwidx1 = -1;
 	s->safe_hwidx2 = -1;
 	if (safe_swz != NULL) {
 		s->safe_hwidx1 = safe_swz->head_hwidx;
 		for (i = safe_swz->head_hwidx; i != -1; i = hwb->next) {
 			int spare;
 
 			hwb = &s->hw_buf_info[i];
 			spare = safe_swz->size - hwb->size;
 			if (spare < CL_METADATA_SIZE)
 				continue;
 			if (s->safe_hwidx2 == -1 ||
 			    spare == CL_METADATA_SIZE + MSIZE)
 				s->safe_hwidx2 = i;
 			if (spare >= CL_METADATA_SIZE + MSIZE)
 				break;
 		}
 	}
 
 	r = t4_read_reg(sc, A_SGE_INGRESS_RX_THRESHOLD);
 	s->counter_val[0] = G_THRESHOLD_0(r);
 	s->counter_val[1] = G_THRESHOLD_1(r);
 	s->counter_val[2] = G_THRESHOLD_2(r);
 	s->counter_val[3] = G_THRESHOLD_3(r);
 
 	r = t4_read_reg(sc, A_SGE_TIMER_VALUE_0_AND_1);
 	s->timer_val[0] = G_TIMERVALUE0(r) / core_ticks_per_usec(sc);
 	s->timer_val[1] = G_TIMERVALUE1(r) / core_ticks_per_usec(sc);
 	r = t4_read_reg(sc, A_SGE_TIMER_VALUE_2_AND_3);
 	s->timer_val[2] = G_TIMERVALUE2(r) / core_ticks_per_usec(sc);
 	s->timer_val[3] = G_TIMERVALUE3(r) / core_ticks_per_usec(sc);
 	r = t4_read_reg(sc, A_SGE_TIMER_VALUE_4_AND_5);
 	s->timer_val[4] = G_TIMERVALUE4(r) / core_ticks_per_usec(sc);
 	s->timer_val[5] = G_TIMERVALUE5(r) / core_ticks_per_usec(sc);
 
 	if (cong_drop == 0) {
 		m = F_TUNNELCNGDROP0 | F_TUNNELCNGDROP1 | F_TUNNELCNGDROP2 |
 		    F_TUNNELCNGDROP3;
 		r = t4_read_reg(sc, A_TP_PARA_REG3);
 		if (r & m) {
 			device_printf(sc->dev,
 			    "invalid TP_PARA_REG3(0x%x)\n", r);
 			rc = EINVAL;
 		}
 	}
 
 	v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6);
 	r = t4_read_reg(sc, A_ULP_RX_TDDP_PSZ);
 	if (r != v) {
 		device_printf(sc->dev, "invalid ULP_RX_TDDP_PSZ(0x%x)\n", r);
 		rc = EINVAL;
 	}
 
 	m = v = F_TDDPTAGTCB;
 	r = t4_read_reg(sc, A_ULP_RX_CTL);
 	if ((r & m) != v) {
 		device_printf(sc->dev, "invalid ULP_RX_CTL(0x%x)\n", r);
 		rc = EINVAL;
 	}
 
 	m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET |
 	    F_RESETDDPOFFSET;
 	v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET;
 	r = t4_read_reg(sc, A_TP_PARA_REG5);
 	if ((r & m) != v) {
 		device_printf(sc->dev, "invalid TP_PARA_REG5(0x%x)\n", r);
 		rc = EINVAL;
 	}
 
 	r = t4_read_reg(sc, A_SGE_CONM_CTRL);
 	s->fl_starve_threshold = G_EGRTHRESHOLD(r) * 2 + 1;
 	if (is_t4(sc))
 		s->fl_starve_threshold2 = s->fl_starve_threshold;
 	else
 		s->fl_starve_threshold2 = G_EGRTHRESHOLDPACKING(r) * 2 + 1;
 
 	/* egress queues: log2 of # of doorbells per BAR2 page */
 	r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF);
 	r >>= S_QUEUESPERPAGEPF0 +
 	    (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf;
 	s->eq_s_qpp = r & M_QUEUESPERPAGEPF0;
 
 	/* ingress queues: log2 of # of doorbells per BAR2 page */
 	r = t4_read_reg(sc, A_SGE_INGRESS_QUEUES_PER_PAGE_PF);
 	r >>= S_QUEUESPERPAGEPF0 +
 	    (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf;
 	s->iq_s_qpp = r & M_QUEUESPERPAGEPF0;
 
 	t4_init_tp_params(sc);
 
 	t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
 	t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd);
 
 	return (rc);
 }
 
 int
 t4_create_dma_tag(struct adapter *sc)
 {
 	int rc;
 
 	rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0,
 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE,
 	    BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL,
 	    NULL, &sc->dmat);
 	if (rc != 0) {
 		device_printf(sc->dev,
 		    "failed to create main DMA tag: %d\n", rc);
 	}
 
 	return (rc);
 }
 
 static inline int
 enable_buffer_packing(struct adapter *sc)
 {
 
 	if (sc->flags & BUF_PACKING_OK &&
 	    ((is_t5(sc) && buffer_packing) ||	/* 1 or -1 both ok for T5 */
 	    (is_t4(sc) && buffer_packing == 1)))
 		return (1);
 	return (0);
 }
 
 void
 t4_sge_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx,
     struct sysctl_oid_list *children)
 {
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "buffer_sizes",
 	    CTLTYPE_STRING | CTLFLAG_RD, &sc->sge, 0, sysctl_bufsizes, "A",
 	    "freelist buffer sizes");
 
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pktshift", CTLFLAG_RD,
 	    NULL, fl_pktshift, "payload DMA offset in rx buffer (bytes)");
 
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pad", CTLFLAG_RD,
 	    NULL, fl_pad, "payload pad boundary (bytes)");
 
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "spg_len", CTLFLAG_RD,
 	    NULL, spg_len, "status page size (bytes)");
 
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_drop", CTLFLAG_RD,
 	    NULL, cong_drop, "congestion drop setting");
 
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "buffer_packing", CTLFLAG_RD,
 	    NULL, enable_buffer_packing(sc),
 	    "pack multiple frames in one fl buffer");
 
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pack", CTLFLAG_RD,
 	    NULL, sc->sge.pack_boundary, "payload pack boundary (bytes)");
 }
 
 int
 t4_destroy_dma_tag(struct adapter *sc)
 {
 	if (sc->dmat)
 		bus_dma_tag_destroy(sc->dmat);
 
 	return (0);
 }
 
 /*
  * Allocate and initialize the firmware event queue and the management queue.
  *
  * Returns errno on failure.  Resources allocated up to that point may still be
  * allocated.  Caller is responsible for cleanup in case this function fails.
  */
 int
 t4_setup_adapter_queues(struct adapter *sc)
 {
 	int rc;
 
 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 
 	sysctl_ctx_init(&sc->ctx);
 	sc->flags |= ADAP_SYSCTL_CTX;
 
 	/*
 	 * Firmware event queue
 	 */
 	rc = alloc_fwq(sc);
 	if (rc != 0)
 		return (rc);
 
 	/*
 	 * Management queue.  This is just a control queue that uses the fwq as
 	 * its associated iq.
 	 */
 	rc = alloc_mgmtq(sc);
 
 	return (rc);
 }
 
 /*
  * Idempotent
  */
 int
 t4_teardown_adapter_queues(struct adapter *sc)
 {
 
 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 
 	/* Do this before freeing the queue */
 	if (sc->flags & ADAP_SYSCTL_CTX) {
 		sysctl_ctx_free(&sc->ctx);
 		sc->flags &= ~ADAP_SYSCTL_CTX;
 	}
 
 	free_mgmtq(sc);
 	free_fwq(sc);
 
 	return (0);
 }
 
 static inline int
+port_intr_count(struct port_info *pi)
+{
+	int rc = 0;
+
+	if (pi->flags & INTR_RXQ)
+		rc += pi->nrxq;
+#ifdef TCP_OFFLOAD
+	if (pi->flags & INTR_OFLD_RXQ)
+		rc += pi->nofldrxq;
+#endif
+#ifdef DEV_NETMAP
+	if (pi->flags & INTR_NM_RXQ)
+		rc += pi->nnmrxq;
+#endif
+	return (rc);
+}
+
+static inline int
 first_vector(struct port_info *pi)
 {
 	struct adapter *sc = pi->adapter;
 	int rc = T4_EXTRA_INTR, i;
 
 	if (sc->intr_count == 1)
 		return (0);
 
 	for_each_port(sc, i) {
-		struct port_info *p = sc->port[i];
-
 		if (i == pi->port_id)
 			break;
 
-#ifdef TCP_OFFLOAD
-		if (sc->flags & INTR_DIRECT)
-			rc += p->nrxq + p->nofldrxq;
-		else
-			rc += max(p->nrxq, p->nofldrxq);
-#else
-		/*
-		 * Not compiled with offload support and intr_count > 1.  Only
-		 * NIC queues exist and they'd better be taking direct
-		 * interrupts.
-		 */
-		KASSERT(sc->flags & INTR_DIRECT,
-		    ("%s: intr_count %d, !INTR_DIRECT", __func__,
-		    sc->intr_count));
-
-		rc += p->nrxq;
-#endif
+		rc += port_intr_count(sc->port[i]);
 	}
 
 	return (rc);
 }
 
 /*
  * Given an arbitrary "index," come up with an iq that can be used by other
  * queues (of this port) for interrupt forwarding, SGE egress updates, etc.
  * The iq returned is guaranteed to be something that takes direct interrupts.
  */
 static struct sge_iq *
 port_intr_iq(struct port_info *pi, int idx)
 {
 	struct adapter *sc = pi->adapter;
 	struct sge *s = &sc->sge;
 	struct sge_iq *iq = NULL;
+	int nintr, i;
 
 	if (sc->intr_count == 1)
 		return (&sc->sge.fwq);
 
-#ifdef TCP_OFFLOAD
-	if (sc->flags & INTR_DIRECT) {
-		idx %= pi->nrxq + pi->nofldrxq;
+	nintr = port_intr_count(pi);
+	KASSERT(nintr != 0,
+	    ("%s: pi %p has no exclusive interrupts, total interrupts = %d",
+	    __func__, pi, sc->intr_count));
+#ifdef DEV_NETMAP
+	/* Exclude netmap queues as they can't take anyone else's interrupts */
+	if (pi->flags & INTR_NM_RXQ)
+		nintr -= pi->nnmrxq;
+	KASSERT(nintr > 0,
+	    ("%s: pi %p has nintr %d after netmap adjustment of %d", __func__,
+	    pi, nintr, pi->nnmrxq));
+#endif
+	i = idx % nintr;
 
-		if (idx >= pi->nrxq) {
-			idx -= pi->nrxq;
-			iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
-		} else
-			iq = &s->rxq[pi->first_rxq + idx].iq;
-
-	} else {
-		idx %= max(pi->nrxq, pi->nofldrxq);
-
-		if (pi->nrxq >= pi->nofldrxq)
-			iq = &s->rxq[pi->first_rxq + idx].iq;
-		else
-			iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
+	if (pi->flags & INTR_RXQ) {
+	       	if (i < pi->nrxq) {
+			iq = &s->rxq[pi->first_rxq + i].iq;
+			goto done;
+		}
+		i -= pi->nrxq;
 	}
-#else
-	/*
-	 * Not compiled with offload support and intr_count > 1.  Only NIC
-	 * queues exist and they'd better be taking direct interrupts.
-	 */
-	KASSERT(sc->flags & INTR_DIRECT,
-	    ("%s: intr_count %d, !INTR_DIRECT", __func__, sc->intr_count));
-
-	idx %= pi->nrxq;
-	iq = &s->rxq[pi->first_rxq + idx].iq;
+#ifdef TCP_OFFLOAD
+	if (pi->flags & INTR_OFLD_RXQ) {
+	       	if (i < pi->nofldrxq) {
+			iq = &s->ofld_rxq[pi->first_ofld_rxq + i].iq;
+			goto done;
+		}
+		i -= pi->nofldrxq;
+	}
 #endif
-
-	KASSERT(iq->flags & IQ_INTR, ("%s: EDOOFUS", __func__));
+	panic("%s: pi %p, intr_flags 0x%lx, idx %d, total intr %d\n", __func__,
+	    pi, pi->flags & INTR_ALL, idx, nintr);
+done:
+	MPASS(iq != NULL);
+	KASSERT(iq->flags & IQ_INTR,
+	    ("%s: iq %p (port %p, intr_flags 0x%lx, idx %d)", __func__, iq, pi,
+	    pi->flags & INTR_ALL, idx));
 	return (iq);
 }
 
 /* Maximum payload that can be delivered with a single iq descriptor */
 static inline int
 mtu_to_max_payload(struct adapter *sc, int mtu, const int toe)
 {
 	int payload;
 
 #ifdef TCP_OFFLOAD
 	if (toe) {
 		payload = sc->tt.rx_coalesce ?
 		    G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)) : mtu;
 	} else {
 #endif
 		/* large enough even when hw VLAN extraction is disabled */
 		payload = fl_pktshift + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN +
 		    mtu;
 #ifdef TCP_OFFLOAD
 	}
 #endif
 	payload = roundup2(payload, fl_pad);
 
 	return (payload);
 }
 
 int
 t4_setup_port_queues(struct port_info *pi)
 {
 	int rc = 0, i, j, intr_idx, iqid;
 	struct sge_rxq *rxq;
 	struct sge_txq *txq;
 	struct sge_wrq *ctrlq;
 #ifdef TCP_OFFLOAD
 	struct sge_ofld_rxq *ofld_rxq;
 	struct sge_wrq *ofld_txq;
-	struct sysctl_oid *oid2 = NULL;
 #endif
+#ifdef DEV_NETMAP
+	struct sge_nm_rxq *nm_rxq;
+	struct sge_nm_txq *nm_txq;
+#endif
 	char name[16];
 	struct adapter *sc = pi->adapter;
 	struct ifnet *ifp = pi->ifp;
 	struct sysctl_oid *oid = device_get_sysctl_tree(pi->dev);
 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
 	int maxp, pack, mtu = ifp->if_mtu;
 
-	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "rxq", CTLFLAG_RD,
-	    NULL, "rx queues");
-
-#ifdef TCP_OFFLOAD
-	if (is_offload(sc)) {
-		oid2 = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_rxq",
-		    CTLFLAG_RD, NULL,
-		    "rx queues for offloaded TCP connections");
-	}
-#endif
-
 	/* Interrupt vector to start from (when using multiple vectors) */
 	intr_idx = first_vector(pi);
 
 	/*
-	 * First pass over all rx queues (NIC and TOE):
+	 * First pass over all NIC and TOE rx queues:
 	 * a) initialize iq and fl
 	 * b) allocate queue iff it will take direct interrupts.
 	 */
 	maxp = mtu_to_max_payload(sc, mtu, 0);
 	pack = enable_buffer_packing(sc);
+	if (pi->flags & INTR_RXQ) {
+		oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "rxq",
+		    CTLFLAG_RD, NULL, "rx queues");
+	}
 	for_each_rxq(pi, i, rxq) {
 
-		init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, pi->qsize_rxq,
-		    RX_IQ_ESIZE);
+		init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, pi->qsize_rxq);
 
 		snprintf(name, sizeof(name), "%s rxq%d-fl",
 		    device_get_nameunit(pi->dev), i);
 		init_fl(sc, &rxq->fl, pi->qsize_rxq / 8, maxp, pack, name);
 
-		if (sc->flags & INTR_DIRECT
-#ifdef TCP_OFFLOAD
-		    || (sc->intr_count > 1 && pi->nrxq >= pi->nofldrxq)
-#endif
-		   ) {
+		if (pi->flags & INTR_RXQ) {
 			rxq->iq.flags |= IQ_INTR;
 			rc = alloc_rxq(pi, rxq, intr_idx, i, oid);
 			if (rc != 0)
 				goto done;
 			intr_idx++;
 		}
 	}
-
 #ifdef TCP_OFFLOAD
 	maxp = mtu_to_max_payload(sc, mtu, 1);
+	if (is_offload(sc) && pi->flags & INTR_OFLD_RXQ) {
+		oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_rxq",
+		    CTLFLAG_RD, NULL,
+		    "rx queues for offloaded TCP connections");
+	}
 	for_each_ofld_rxq(pi, i, ofld_rxq) {
 
 		init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
-		    pi->qsize_rxq, RX_IQ_ESIZE);
+		    pi->qsize_rxq);
 
 		snprintf(name, sizeof(name), "%s ofld_rxq%d-fl",
 		    device_get_nameunit(pi->dev), i);
 		init_fl(sc, &ofld_rxq->fl, pi->qsize_rxq / 8, maxp, pack, name);
 
-		if (sc->flags & INTR_DIRECT ||
-		    (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) {
+		if (pi->flags & INTR_OFLD_RXQ) {
 			ofld_rxq->iq.flags |= IQ_INTR;
-			rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx, i, oid2);
+			rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx, i, oid);
 			if (rc != 0)
 				goto done;
 			intr_idx++;
 		}
 	}
 #endif
+#ifdef DEV_NETMAP
+	/*
+	 * We don't have buffers to back the netmap rx queues right now so we
+	 * create the queues in a way that doesn't set off any congestion signal
+	 * in the chip.
+	 */
+	if (pi->flags & INTR_NM_RXQ) {
+		oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "nm_rxq",
+		    CTLFLAG_RD, NULL, "rx queues for netmap");
+		for_each_nm_rxq(pi, i, nm_rxq) {
+			rc = alloc_nm_rxq(pi, nm_rxq, intr_idx, i, oid);
+			if (rc != 0)
+				goto done;
+			intr_idx++;
+		}
+	}
+#endif
 
 	/*
-	 * Second pass over all rx queues (NIC and TOE).  The queues forwarding
+	 * Second pass over all NIC and TOE rx queues.  The queues forwarding
 	 * their interrupts are allocated now.
 	 */
 	j = 0;
-	for_each_rxq(pi, i, rxq) {
-		if (rxq->iq.flags & IQ_INTR)
-			continue;
+	if (!(pi->flags & INTR_RXQ)) {
+		oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "rxq",
+		    CTLFLAG_RD, NULL, "rx queues");
+		for_each_rxq(pi, i, rxq) {
+			MPASS(!(rxq->iq.flags & IQ_INTR));
 
-		intr_idx = port_intr_iq(pi, j)->abs_id;
+			intr_idx = port_intr_iq(pi, j)->abs_id;
 
-		rc = alloc_rxq(pi, rxq, intr_idx, i, oid);
-		if (rc != 0)
-			goto done;
-		j++;
+			rc = alloc_rxq(pi, rxq, intr_idx, i, oid);
+			if (rc != 0)
+				goto done;
+			j++;
+		}
 	}
-
 #ifdef TCP_OFFLOAD
-	for_each_ofld_rxq(pi, i, ofld_rxq) {
-		if (ofld_rxq->iq.flags & IQ_INTR)
-			continue;
+	if (is_offload(sc) && !(pi->flags & INTR_OFLD_RXQ)) {
+		oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_rxq",
+		    CTLFLAG_RD, NULL,
+		    "rx queues for offloaded TCP connections");
+		for_each_ofld_rxq(pi, i, ofld_rxq) {
+			MPASS(!(ofld_rxq->iq.flags & IQ_INTR));
 
-		intr_idx = port_intr_iq(pi, j)->abs_id;
+			intr_idx = port_intr_iq(pi, j)->abs_id;
 
-		rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx, i, oid2);
-		if (rc != 0)
-			goto done;
-		j++;
+			rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx, i, oid);
+			if (rc != 0)
+				goto done;
+			j++;
+		}
 	}
 #endif
+#ifdef DEV_NETMAP
+	if (!(pi->flags & INTR_NM_RXQ))
+		CXGBE_UNIMPLEMENTED(__func__);
+#endif
 
 	/*
 	 * Now the tx queues.  Only one pass needed.
 	 */
 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "txq", CTLFLAG_RD,
 	    NULL, "tx queues");
 	j = 0;
 	for_each_txq(pi, i, txq) {
-		uint16_t iqid;
-
 		iqid = port_intr_iq(pi, j)->cntxt_id;
-
 		snprintf(name, sizeof(name), "%s txq%d",
 		    device_get_nameunit(pi->dev), i);
 		init_eq(&txq->eq, EQ_ETH, pi->qsize_txq, pi->tx_chan, iqid,
 		    name);
 
 		rc = alloc_txq(pi, txq, i, oid);
 		if (rc != 0)
 			goto done;
 		j++;
 	}
-
 #ifdef TCP_OFFLOAD
 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_txq",
 	    CTLFLAG_RD, NULL, "tx queues for offloaded TCP connections");
 	for_each_ofld_txq(pi, i, ofld_txq) {
-		uint16_t iqid;
+		struct sysctl_oid *oid2;
 
 		iqid = port_intr_iq(pi, j)->cntxt_id;
-
 		snprintf(name, sizeof(name), "%s ofld_txq%d",
 		    device_get_nameunit(pi->dev), i);
 		init_eq(&ofld_txq->eq, EQ_OFLD, pi->qsize_txq, pi->tx_chan,
 		    iqid, name);
 
 		snprintf(name, sizeof(name), "%d", i);
 		oid2 = SYSCTL_ADD_NODE(&pi->ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
 		    name, CTLFLAG_RD, NULL, "offload tx queue");
 
 		rc = alloc_wrq(sc, pi, ofld_txq, oid2);
 		if (rc != 0)
 			goto done;
 		j++;
 	}
 #endif
+#ifdef DEV_NETMAP
+	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "nm_txq",
+	    CTLFLAG_RD, NULL, "tx queues for netmap use");
+	for_each_nm_txq(pi, i, nm_txq) {
+		iqid = pi->first_nm_rxq + (j % pi->nnmrxq);
+		rc = alloc_nm_txq(pi, nm_txq, iqid, i, oid);
+		if (rc != 0)
+			goto done;
+		j++;
+	}
+#endif
 
 	/*
 	 * Finally, the control queue.
 	 */
 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ctrlq", CTLFLAG_RD,
 	    NULL, "ctrl queue");
 	ctrlq = &sc->sge.ctrlq[pi->port_id];
 	iqid = port_intr_iq(pi, 0)->cntxt_id;
 	snprintf(name, sizeof(name), "%s ctrlq", device_get_nameunit(pi->dev));
 	init_eq(&ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, iqid, name);
 	rc = alloc_wrq(sc, pi, ctrlq, oid);
 
 done:
 	if (rc)
 		t4_teardown_port_queues(pi);
 
 	return (rc);
 }
 
 /*
  * Idempotent
  */
 int
 t4_teardown_port_queues(struct port_info *pi)
 {
 	int i;
 	struct adapter *sc = pi->adapter;
 	struct sge_rxq *rxq;
 	struct sge_txq *txq;
 #ifdef TCP_OFFLOAD
 	struct sge_ofld_rxq *ofld_rxq;
 	struct sge_wrq *ofld_txq;
 #endif
+#ifdef DEV_NETMAP
+	struct sge_nm_rxq *nm_rxq;
+	struct sge_nm_txq *nm_txq;
+#endif
 
 	/* Do this before freeing the queues */
 	if (pi->flags & PORT_SYSCTL_CTX) {
 		sysctl_ctx_free(&pi->ctx);
 		pi->flags &= ~PORT_SYSCTL_CTX;
 	}
 
 	/*
 	 * Take down all the tx queues first, as they reference the rx queues
 	 * (for egress updates, etc.).
 	 */
 
 	free_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
 
 	for_each_txq(pi, i, txq) {
 		free_txq(pi, txq);
 	}
-
 #ifdef TCP_OFFLOAD
 	for_each_ofld_txq(pi, i, ofld_txq) {
 		free_wrq(sc, ofld_txq);
 	}
 #endif
+#ifdef DEV_NETMAP
+	for_each_nm_txq(pi, i, nm_txq)
+	    free_nm_txq(pi, nm_txq);
+#endif
 
 	/*
 	 * Then take down the rx queues that forward their interrupts, as they
 	 * reference other rx queues.
 	 */
 
 	for_each_rxq(pi, i, rxq) {
 		if ((rxq->iq.flags & IQ_INTR) == 0)
 			free_rxq(pi, rxq);
 	}
-
 #ifdef TCP_OFFLOAD
 	for_each_ofld_rxq(pi, i, ofld_rxq) {
 		if ((ofld_rxq->iq.flags & IQ_INTR) == 0)
 			free_ofld_rxq(pi, ofld_rxq);
 	}
 #endif
+#ifdef DEV_NETMAP
+	for_each_nm_rxq(pi, i, nm_rxq)
+	    free_nm_rxq(pi, nm_rxq);
+#endif
 
 	/*
 	 * Then take down the rx queues that take direct interrupts.
 	 */
 
 	for_each_rxq(pi, i, rxq) {
 		if (rxq->iq.flags & IQ_INTR)
 			free_rxq(pi, rxq);
 	}
-
 #ifdef TCP_OFFLOAD
 	for_each_ofld_rxq(pi, i, ofld_rxq) {
 		if (ofld_rxq->iq.flags & IQ_INTR)
 			free_ofld_rxq(pi, ofld_rxq);
 	}
 #endif
+#ifdef DEV_NETMAP
+	CXGBE_UNIMPLEMENTED(__func__);
+#endif
 
 	return (0);
 }
 
 /*
  * Deals with errors and the firmware event queue.  All data rx queues forward
  * their interrupt to the firmware event queue.
  */
 void
 t4_intr_all(void *arg)
 {
 	struct adapter *sc = arg;
 	struct sge_iq *fwq = &sc->sge.fwq;
 
 	t4_intr_err(arg);
 	if (atomic_cmpset_int(&fwq->state, IQS_IDLE, IQS_BUSY)) {
 		service_iq(fwq, 0);
 		atomic_cmpset_int(&fwq->state, IQS_BUSY, IQS_IDLE);
 	}
 }
 
 /* Deals with error interrupts */
 void
 t4_intr_err(void *arg)
 {
 	struct adapter *sc = arg;
 
 	t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0);
 	t4_slow_intr_handler(sc);
 }
 
 void
 t4_intr_evt(void *arg)
 {
 	struct sge_iq *iq = arg;
 
 	if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
 		service_iq(iq, 0);
 		atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
 	}
 }
 
 void
 t4_intr(void *arg)
 {
 	struct sge_iq *iq = arg;
 
 	if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
 		service_iq(iq, 0);
 		atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
 	}
 }
 
 /*
  * Deals with anything and everything on the given ingress queue.
  */
 static int
 service_iq(struct sge_iq *iq, int budget)
 {
 	struct sge_iq *q;
 	struct sge_rxq *rxq = iq_to_rxq(iq);	/* Use iff iq is part of rxq */
-	struct sge_fl *fl = &rxq->fl;		/* Use iff IQ_HAS_FL */
+	struct sge_fl *fl;			/* Use iff IQ_HAS_FL */
 	struct adapter *sc = iq->adapter;
-	struct rsp_ctrl *ctrl;
-	const struct rss_header *rss;
-	int ndescs = 0, limit, fl_bufs_used = 0;
-	int rsp_type;
+	struct iq_desc *d = &iq->desc[iq->cidx];
+	int ndescs = 0, limit;
+	int rsp_type, refill;
 	uint32_t lq;
+	uint16_t fl_hw_cidx;
 	struct mbuf *m0;
 	STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
 #if defined(INET) || defined(INET6)
 	const struct timeval lro_timeout = {0, sc->lro_timeout};
 #endif
 
-	limit = budget ? budget : iq->qsize / 8;
-
 	KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
 
+	limit = budget ? budget : iq->qsize / 16;
+
+	if (iq->flags & IQ_HAS_FL) {
+		fl = &rxq->fl;
+		fl_hw_cidx = fl->hw_cidx;	/* stable snapshot */
+	} else {
+		fl = NULL;
+		fl_hw_cidx = 0;			/* to silence gcc warning */
+	}
+
 	/*
 	 * We always come back and check the descriptor ring for new indirect
 	 * interrupts and other responses after running a single handler.
 	 */
 	for (;;) {
-		while (is_new_response(iq, &ctrl)) {
+		while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) {
 
 			rmb();
 
+			refill = 0;
 			m0 = NULL;
-			rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
-			lq = be32toh(ctrl->pldbuflen_qid);
-			rss = (const void *)iq->cdesc;
+			rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen);
+			lq = be32toh(d->rsp.pldbuflen_qid);
 
 			switch (rsp_type) {
 			case X_RSPD_TYPE_FLBUF:
 
 				KASSERT(iq->flags & IQ_HAS_FL,
 				    ("%s: data for an iq (%p) with no freelist",
 				    __func__, iq));
 
-				m0 = get_fl_payload(sc, fl, lq, &fl_bufs_used);
+				m0 = get_fl_payload(sc, fl, lq);
 				if (__predict_false(m0 == NULL))
 					goto process_iql;
+				refill = IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 2;
 #ifdef T4_PKT_TIMESTAMP
 				/*
 				 * 60 bit timestamp for the payload is
 				 * *(uint64_t *)m0->m_pktdat.  Note that it is
 				 * in the leading free-space in the mbuf.  The
 				 * kernel can clobber it during a pullup,
 				 * m_copymdata, etc.  You need to make sure that
 				 * the mbuf reaches you unmolested if you care
 				 * about the timestamp.
 				 */
 				*(uint64_t *)m0->m_pktdat =
 				    be64toh(ctrl->u.last_flit) &
 				    0xfffffffffffffff;
 #endif
 
 				/* fall through */
 
 			case X_RSPD_TYPE_CPL:
-				KASSERT(rss->opcode < NUM_CPL_CMDS,
+				KASSERT(d->rss.opcode < NUM_CPL_CMDS,
 				    ("%s: bad opcode %02x.", __func__,
-				    rss->opcode));
-				sc->cpl_handler[rss->opcode](iq, rss, m0);
+				    d->rss.opcode));
+				sc->cpl_handler[d->rss.opcode](iq, &d->rss, m0);
 				break;
 
 			case X_RSPD_TYPE_INTR:
 
 				/*
 				 * Interrupts should be forwarded only to queues
 				 * that are not forwarding their interrupts.
 				 * This means service_iq can recurse but only 1
 				 * level deep.
 				 */
 				KASSERT(budget == 0,
 				    ("%s: budget %u, rsp_type %u", __func__,
 				    budget, rsp_type));
 
 				/*
 				 * There are 1K interrupt-capable queues (qids 0
 				 * through 1023).  A response type indicating a
 				 * forwarded interrupt with a qid >= 1K is an
 				 * iWARP async notification.
 				 */
 				if (lq >= 1024) {
-                                        sc->an_handler(iq, ctrl);
+                                        sc->an_handler(iq, &d->rsp);
                                         break;
                                 }
 
 				q = sc->sge.iqmap[lq - sc->sge.iq_start];
 				if (atomic_cmpset_int(&q->state, IQS_IDLE,
 				    IQS_BUSY)) {
-					if (service_iq(q, q->qsize / 8) == 0) {
+					if (service_iq(q, q->qsize / 16) == 0) {
 						atomic_cmpset_int(&q->state,
 						    IQS_BUSY, IQS_IDLE);
 					} else {
 						STAILQ_INSERT_TAIL(&iql, q,
 						    link);
 					}
 				}
 				break;
 
 			default:
 				KASSERT(0,
 				    ("%s: illegal response type %d on iq %p",
 				    __func__, rsp_type, iq));
 				log(LOG_ERR,
 				    "%s: illegal response type %d on iq %p",
 				    device_get_nameunit(sc->dev), rsp_type, iq);
 				break;
 			}
 
-			if (fl_bufs_used >= 16) {
-				FL_LOCK(fl);
-				fl->needed += fl_bufs_used;
-				refill_fl(sc, fl, 32);
-				FL_UNLOCK(fl);
-				fl_bufs_used = 0;
+			d++;
+			if (__predict_false(++iq->cidx == iq->sidx)) {
+				iq->cidx = 0;
+				iq->gen ^= F_RSPD_GEN;
+				d = &iq->desc[0];
 			}
-
-			iq_next(iq);
-			if (++ndescs == limit) {
+			if (__predict_false(++ndescs == limit)) {
 				t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
 				    V_CIDXINC(ndescs) |
 				    V_INGRESSQID(iq->cntxt_id) |
 				    V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
 				ndescs = 0;
 
 #if defined(INET) || defined(INET6)
 				if (iq->flags & IQ_LRO_ENABLED &&
 				    sc->lro_timeout != 0) {
 					tcp_lro_flush_inactive(&rxq->lro,
 					    &lro_timeout);
 				}
 #endif
 
 				if (budget) {
-					if (fl_bufs_used) {
+					if (iq->flags & IQ_HAS_FL) {
 						FL_LOCK(fl);
-						fl->needed += fl_bufs_used;
 						refill_fl(sc, fl, 32);
 						FL_UNLOCK(fl);
 					}
 					return (EINPROGRESS);
 				}
 			}
+			if (refill) {
+				FL_LOCK(fl);
+				refill_fl(sc, fl, 32);
+				FL_UNLOCK(fl);
+				fl_hw_cidx = fl->hw_cidx;
+			}
 		}
 
 process_iql:
 		if (STAILQ_EMPTY(&iql))
 			break;
 
 		/*
 		 * Process the head only, and send it to the back of the list if
 		 * it's still not done.
 		 */
 		q = STAILQ_FIRST(&iql);
 		STAILQ_REMOVE_HEAD(&iql, link);
 		if (service_iq(q, q->qsize / 8) == 0)
 			atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE);
 		else
 			STAILQ_INSERT_TAIL(&iql, q, link);
 	}
 
 #if defined(INET) || defined(INET6)
 	if (iq->flags & IQ_LRO_ENABLED) {
 		struct lro_ctrl *lro = &rxq->lro;
 		struct lro_entry *l;
 
 		while (!SLIST_EMPTY(&lro->lro_active)) {
 			l = SLIST_FIRST(&lro->lro_active);
 			SLIST_REMOVE_HEAD(&lro->lro_active, next);
 			tcp_lro_flush(lro, l);
 		}
 	}
 #endif
 
 	t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
 	    V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
 
 	if (iq->flags & IQ_HAS_FL) {
 		int starved;
 
 		FL_LOCK(fl);
-		fl->needed += fl_bufs_used;
 		starved = refill_fl(sc, fl, 64);
 		FL_UNLOCK(fl);
 		if (__predict_false(starved != 0))
 			add_fl_to_sfl(sc, fl);
 	}
 
 	return (0);
 }
 
 static inline int
 cl_has_metadata(struct sge_fl *fl, struct cluster_layout *cll)
 {
 	int rc = fl->flags & FL_BUF_PACKING || cll->region1 > 0;
 
 	if (rc)
 		MPASS(cll->region3 >= CL_METADATA_SIZE);
 
 	return (rc);
 }
 
 static inline struct cluster_metadata *
 cl_metadata(struct adapter *sc, struct sge_fl *fl, struct cluster_layout *cll,
     caddr_t cl)
 {
 
 	if (cl_has_metadata(fl, cll)) {
 		struct sw_zone_info *swz = &sc->sge.sw_zone_info[cll->zidx];
 
 		return ((struct cluster_metadata *)(cl + swz->size) - 1);
 	}
 	return (NULL);
 }
 
 static int
 rxb_free(struct mbuf *m, void *arg1, void *arg2)
 {
 	uma_zone_t zone = arg1;
 	caddr_t cl = arg2;
 
 	uma_zfree(zone, cl);
 	counter_u64_add(extfree_rels, 1);
 
 	return (EXT_FREE_OK);
 }
 
 /*
  * The mbuf returned by this function could be allocated from zone_mbuf or
  * constructed in spare room in the cluster.
  *
  * The mbuf carries the payload in one of these ways
  * a) frame inside the mbuf (mbuf from zone_mbuf)
  * b) m_cljset (for clusters without metadata) zone_mbuf
  * c) m_extaddref (cluster with metadata) inline mbuf
  * d) m_extaddref (cluster with metadata) zone_mbuf
  */
 static struct mbuf *
 get_scatter_segment(struct adapter *sc, struct sge_fl *fl, int total, int flags)
 {
 	struct mbuf *m;
 	struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
 	struct cluster_layout *cll = &sd->cll;
 	struct sw_zone_info *swz = &sc->sge.sw_zone_info[cll->zidx];
 	struct hw_buf_info *hwb = &sc->sge.hw_buf_info[cll->hwidx];
 	struct cluster_metadata *clm = cl_metadata(sc, fl, cll, sd->cl);
 	int len, padded_len;
 	caddr_t payload;
 
 	len = min(total, hwb->size - fl->rx_offset);
-	padded_len = roundup2(len, fl_pad);
+	padded_len = roundup2(len, fl->buf_boundary);
 	payload = sd->cl + cll->region1 + fl->rx_offset;
 
 	if (sc->sc_do_rxcopy && len < RX_COPY_THRESHOLD) {
 
 		/*
 		 * Copy payload into a freshly allocated mbuf.
 		 */
 
 		m = flags & M_PKTHDR ?
 		    m_gethdr(M_NOWAIT, MT_DATA) : m_get(M_NOWAIT, MT_DATA);
 		if (m == NULL)
 			return (NULL);
 		fl->mbuf_allocated++;
 #ifdef T4_PKT_TIMESTAMP
 		/* Leave room for a timestamp */
 		m->m_data += 8;
 #endif
 		/* copy data to mbuf */
 		bcopy(payload, mtod(m, caddr_t), len);
 
 	} else if (sd->nmbuf * MSIZE < cll->region1) {
 
 		/*
 		 * There's spare room in the cluster for an mbuf.  Create one
 		 * and associate it with the payload that's in the cluster.
 		 */
 
 		MPASS(clm != NULL);
 		m = (struct mbuf *)(sd->cl + sd->nmbuf * MSIZE);
 		/* No bzero required */
 		if (m_init(m, NULL, 0, M_NOWAIT, MT_DATA, flags | M_NOFREE))
 			return (NULL);
 		fl->mbuf_inlined++;
 		m_extaddref(m, payload, padded_len, &clm->refcount, rxb_free,
 		    swz->zone, sd->cl);
 		if (sd->nmbuf++ == 0)
 			counter_u64_add(extfree_refs, 1);
 
 	} else {
 
 		/*
 		 * Grab an mbuf from zone_mbuf and associate it with the
 		 * payload in the cluster.
 		 */
 
 		m = flags & M_PKTHDR ?
 		    m_gethdr(M_NOWAIT, MT_DATA) : m_get(M_NOWAIT, MT_DATA);
 		if (m == NULL)
 			return (NULL);
 		fl->mbuf_allocated++;
 		if (clm != NULL) {
 			m_extaddref(m, payload, padded_len, &clm->refcount,
 			    rxb_free, swz->zone, sd->cl);
 			if (sd->nmbuf++ == 0)
 				counter_u64_add(extfree_refs, 1);
 		} else {
 			m_cljset(m, sd->cl, swz->type);
 			sd->cl = NULL;	/* consumed, not a recycle candidate */
 		}
 	}
 	if (flags & M_PKTHDR)
 		m->m_pkthdr.len = total;
 	m->m_len = len;
 
 	if (fl->flags & FL_BUF_PACKING) {
-		fl->rx_offset += roundup2(padded_len, sc->sge.pack_boundary);
+		fl->rx_offset += padded_len;
 		MPASS(fl->rx_offset <= hwb->size);
 		if (fl->rx_offset < hwb->size)
 			return (m);	/* without advancing the cidx */
 	}
 
-	if (__predict_false(++fl->cidx == fl->cap))
-		fl->cidx = 0;
+	if (__predict_false(++fl->cidx % 8 == 0)) {
+		uint16_t cidx = fl->cidx / 8;
+
+		if (__predict_false(cidx == fl->sidx))
+			fl->cidx = cidx = 0;
+		fl->hw_cidx = cidx;
+	}
 	fl->rx_offset = 0;
 
 	return (m);
 }
 
 static struct mbuf *
-get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf,
-    int *fl_bufs_used)
+get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf)
 {
 	struct mbuf *m0, *m, **pnext;
-	u_int nbuf, len;
+	u_int len;
 
-	/*
-	 * No assertion for the fl lock because we don't need it.  This routine
-	 * is called only from the rx interrupt handler and it only updates
-	 * fl->cidx.  (Contrast that with fl->pidx/fl->needed which could be
-	 * updated in the rx interrupt handler or the starvation helper routine.
-	 * That's why code that manipulates fl->pidx/fl->needed needs the fl
-	 * lock but this routine does not).
-	 */
-
-	nbuf = 0;
 	len = G_RSPD_LEN(len_newbuf);
-	if (__predict_false(fl->m0 != NULL)) {
+	if (__predict_false(fl->flags & FL_BUF_RESUME)) {
 		M_ASSERTPKTHDR(fl->m0);
 		MPASS(len == fl->m0->m_pkthdr.len);
 		MPASS(fl->remaining < len);
 
 		m0 = fl->m0;
 		pnext = fl->pnext;
 		len = fl->remaining;
-		fl->m0 = NULL;
+		fl->flags &= ~FL_BUF_RESUME;
 		goto get_segment;
 	}
 
 	if (fl->rx_offset > 0 && len_newbuf & F_RSPD_NEWBUF) {
-		nbuf++;
 		fl->rx_offset = 0;
-		if (__predict_false(++fl->cidx == fl->cap))
-			fl->cidx = 0;
+		if (__predict_false(++fl->cidx % 8 == 0)) {
+			uint16_t cidx = fl->cidx / 8;
+
+			if (__predict_false(cidx == fl->sidx))
+				fl->cidx = cidx = 0;
+			fl->hw_cidx = cidx;
+		}
 	}
 
 	/*
 	 * Payload starts at rx_offset in the current hw buffer.  Its length is
 	 * 'len' and it may span multiple hw buffers.
 	 */
 
 	m0 = get_scatter_segment(sc, fl, len, M_PKTHDR);
 	if (m0 == NULL)
-		goto done;
+		return (NULL);
 	len -= m0->m_len;
 	pnext = &m0->m_next;
 	while (len > 0) {
-		nbuf++;
 get_segment:
 		MPASS(fl->rx_offset == 0);
 		m = get_scatter_segment(sc, fl, len, 0);
-		if (m == NULL) {
+		if (__predict_false(m == NULL)) {
 			fl->m0 = m0;
 			fl->pnext = pnext;
 			fl->remaining = len;
-			m0 = NULL;
-			goto done;
+			fl->flags |= FL_BUF_RESUME;
+			return (NULL);
 		}
 		*pnext = m;
 		pnext = &m->m_next;
 		len -= m->m_len;
 	}
 	*pnext = NULL;
-	if (fl->rx_offset == 0)
-		nbuf++;
-done:
-	(*fl_bufs_used) += nbuf;
+
 	return (m0);
 }
 
 static int
 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0)
 {
 	struct sge_rxq *rxq = iq_to_rxq(iq);
 	struct ifnet *ifp = rxq->ifp;
 	const struct cpl_rx_pkt *cpl = (const void *)(rss + 1);
 #if defined(INET) || defined(INET6)
 	struct lro_ctrl *lro = &rxq->lro;
 #endif
 
 	KASSERT(m0 != NULL, ("%s: no payload with opcode %02x", __func__,
 	    rss->opcode));
 
 	m0->m_pkthdr.len -= fl_pktshift;
 	m0->m_len -= fl_pktshift;
 	m0->m_data += fl_pktshift;
 
 	m0->m_pkthdr.rcvif = ifp;
 	m0->m_flags |= M_FLOWID;
 	m0->m_pkthdr.flowid = be32toh(rss->hash_val);
 
 	if (cpl->csum_calc && !cpl->err_vec) {
 		if (ifp->if_capenable & IFCAP_RXCSUM &&
 		    cpl->l2info & htobe32(F_RXF_IP)) {
 			m0->m_pkthdr.csum_flags = (CSUM_IP_CHECKED |
 			    CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
 			rxq->rxcsum++;
 		} else if (ifp->if_capenable & IFCAP_RXCSUM_IPV6 &&
 		    cpl->l2info & htobe32(F_RXF_IP6)) {
 			m0->m_pkthdr.csum_flags = (CSUM_DATA_VALID_IPV6 |
 			    CSUM_PSEUDO_HDR);
 			rxq->rxcsum++;
 		}
 
 		if (__predict_false(cpl->ip_frag))
 			m0->m_pkthdr.csum_data = be16toh(cpl->csum);
 		else
 			m0->m_pkthdr.csum_data = 0xffff;
 	}
 
 	if (cpl->vlan_ex) {
 		m0->m_pkthdr.ether_vtag = be16toh(cpl->vlan);
 		m0->m_flags |= M_VLANTAG;
 		rxq->vlan_extraction++;
 	}
 
 #if defined(INET) || defined(INET6)
 	if (cpl->l2info & htobe32(F_RXF_LRO) &&
 	    iq->flags & IQ_LRO_ENABLED &&
 	    tcp_lro_rx(lro, m0, 0) == 0) {
 		/* queued for LRO */
 	} else
 #endif
 	ifp->if_input(ifp, m0);
 
 	return (0);
 }
 
 /*
  * Doesn't fail.  Holds on to work requests it can't send right away.
  */
 void
 t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr)
 {
 	struct sge_eq *eq = &wrq->eq;
 	int can_reclaim;
 	caddr_t dst;
 
 	TXQ_LOCK_ASSERT_OWNED(wrq);
 #ifdef TCP_OFFLOAD
 	KASSERT((eq->flags & EQ_TYPEMASK) == EQ_OFLD ||
 	    (eq->flags & EQ_TYPEMASK) == EQ_CTRL,
 	    ("%s: eq type %d", __func__, eq->flags & EQ_TYPEMASK));
 #else
 	KASSERT((eq->flags & EQ_TYPEMASK) == EQ_CTRL,
 	    ("%s: eq type %d", __func__, eq->flags & EQ_TYPEMASK));
 #endif
 
 	if (__predict_true(wr != NULL))
 		STAILQ_INSERT_TAIL(&wrq->wr_list, wr, link);
 
 	can_reclaim = reclaimable(eq);
 	if (__predict_false(eq->flags & EQ_STALLED)) {
 		if (eq->avail + can_reclaim < tx_resume_threshold(eq))
 			return;
 		eq->flags &= ~EQ_STALLED;
 		eq->unstalled++;
 	}
 	eq->cidx += can_reclaim;
 	eq->avail += can_reclaim;
 	if (__predict_false(eq->cidx >= eq->cap))
 		eq->cidx -= eq->cap;
 
 	while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL) {
 		int ndesc;
 
 		if (__predict_false(wr->wr_len < 0 ||
 		    wr->wr_len > SGE_MAX_WR_LEN || (wr->wr_len & 0x7))) {
 
 #ifdef INVARIANTS
 			panic("%s: work request with length %d", __func__,
 			    wr->wr_len);
 #endif
 #ifdef KDB
 			kdb_backtrace();
 #endif
 			log(LOG_ERR, "%s: %s work request with length %d",
 			    device_get_nameunit(sc->dev), __func__, wr->wr_len);
 			STAILQ_REMOVE_HEAD(&wrq->wr_list, link);
 			free_wrqe(wr);
 			continue;
 		}
 
 		ndesc = howmany(wr->wr_len, EQ_ESIZE);
 		if (eq->avail < ndesc) {
 			wrq->no_desc++;
 			break;
 		}
 
 		dst = (void *)&eq->desc[eq->pidx];
 		copy_to_txd(eq, wrtod(wr), &dst, wr->wr_len);
 
 		eq->pidx += ndesc;
 		eq->avail -= ndesc;
 		if (__predict_false(eq->pidx >= eq->cap))
 			eq->pidx -= eq->cap;
 
 		eq->pending += ndesc;
 		if (eq->pending >= 8)
 			ring_eq_db(sc, eq);
 
 		wrq->tx_wrs++;
 		STAILQ_REMOVE_HEAD(&wrq->wr_list, link);
 		free_wrqe(wr);
 
 		if (eq->avail < 8) {
 			can_reclaim = reclaimable(eq);
 			eq->cidx += can_reclaim;
 			eq->avail += can_reclaim;
 			if (__predict_false(eq->cidx >= eq->cap))
 				eq->cidx -= eq->cap;
 		}
 	}
 
 	if (eq->pending)
 		ring_eq_db(sc, eq);
 
 	if (wr != NULL) {
 		eq->flags |= EQ_STALLED;
 		if (callout_pending(&eq->tx_callout) == 0)
 			callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
 	}
 }
 
 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */
 #define TXPKTS_PKT_HDR ((\
     sizeof(struct ulp_txpkt) + \
     sizeof(struct ulptx_idata) + \
     sizeof(struct cpl_tx_pkt_core) \
     ) / 8)
 
 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */
 #define TXPKTS_WR_HDR (\
     sizeof(struct fw_eth_tx_pkts_wr) / 8 + \
     TXPKTS_PKT_HDR)
 
 /* Header of a tx WR, before SGL of first packet (in flits) */
 #define TXPKT_WR_HDR ((\
     sizeof(struct fw_eth_tx_pkt_wr) + \
     sizeof(struct cpl_tx_pkt_core) \
     ) / 8 )
 
 /* Header of a tx LSO WR, before SGL of first packet (in flits) */
 #define TXPKT_LSO_WR_HDR ((\
     sizeof(struct fw_eth_tx_pkt_wr) + \
     sizeof(struct cpl_tx_pkt_lso_core) + \
     sizeof(struct cpl_tx_pkt_core) \
     ) / 8 )
 
 int
 t4_eth_tx(struct ifnet *ifp, struct sge_txq *txq, struct mbuf *m)
 {
 	struct port_info *pi = (void *)ifp->if_softc;
 	struct adapter *sc = pi->adapter;
 	struct sge_eq *eq = &txq->eq;
 	struct buf_ring *br = txq->br;
 	struct mbuf *next;
 	int rc, coalescing, can_reclaim;
 	struct txpkts txpkts;
 	struct sgl sgl;
 
 	TXQ_LOCK_ASSERT_OWNED(txq);
 	KASSERT(m, ("%s: called with nothing to do.", __func__));
 	KASSERT((eq->flags & EQ_TYPEMASK) == EQ_ETH,
 	    ("%s: eq type %d", __func__, eq->flags & EQ_TYPEMASK));
 
 	prefetch(&eq->desc[eq->pidx]);
 	prefetch(&txq->sdesc[eq->pidx]);
 
 	txpkts.npkt = 0;/* indicates there's nothing in txpkts */
 	coalescing = 0;
 
 	can_reclaim = reclaimable(eq);
 	if (__predict_false(eq->flags & EQ_STALLED)) {
 		if (eq->avail + can_reclaim < tx_resume_threshold(eq)) {
 			txq->m = m;
 			return (0);
 		}
 		eq->flags &= ~EQ_STALLED;
 		eq->unstalled++;
 	}
 
 	if (__predict_false(eq->flags & EQ_DOOMED)) {
 		m_freem(m);
 		while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
 			m_freem(m);
 		return (ENETDOWN);
 	}
 
 	if (eq->avail < 8 && can_reclaim)
 		reclaim_tx_descs(txq, can_reclaim, 32);
 
 	for (; m; m = next ? next : drbr_dequeue(ifp, br)) {
 
 		if (eq->avail < 8)
 			break;
 
 		next = m->m_nextpkt;
 		m->m_nextpkt = NULL;
 
 		if (next || buf_ring_peek(br))
 			coalescing = 1;
 
 		rc = get_pkt_sgl(txq, &m, &sgl, coalescing);
 		if (rc != 0) {
 			if (rc == ENOMEM) {
 
 				/* Short of resources, suspend tx */
 
 				m->m_nextpkt = next;
 				break;
 			}
 
 			/*
 			 * Unrecoverable error for this packet, throw it away
 			 * and move on to the next.  get_pkt_sgl may already
 			 * have freed m (it will be NULL in that case and the
 			 * m_freem here is still safe).
 			 */
 
 			m_freem(m);
 			continue;
 		}
 
 		if (coalescing &&
 		    add_to_txpkts(pi, txq, &txpkts, m, &sgl) == 0) {
 
 			/* Successfully absorbed into txpkts */
 
 			write_ulp_cpl_sgl(pi, txq, &txpkts, m, &sgl);
 			goto doorbell;
 		}
 
 		/*
 		 * We weren't coalescing to begin with, or current frame could
 		 * not be coalesced (add_to_txpkts flushes txpkts if a frame
 		 * given to it can't be coalesced).  Either way there should be
 		 * nothing in txpkts.
 		 */
 		KASSERT(txpkts.npkt == 0,
 		    ("%s: txpkts not empty: %d", __func__, txpkts.npkt));
 
 		/* We're sending out individual packets now */
 		coalescing = 0;
 
 		if (eq->avail < 8)
 			reclaim_tx_descs(txq, 0, 8);
 		rc = write_txpkt_wr(pi, txq, m, &sgl);
 		if (rc != 0) {
 
 			/* Short of hardware descriptors, suspend tx */
 
 			/*
 			 * This is an unlikely but expensive failure.  We've
 			 * done all the hard work (DMA mappings etc.) and now we
 			 * can't send out the packet.  What's worse, we have to
 			 * spend even more time freeing up everything in sgl.
 			 */
 			txq->no_desc++;
 			free_pkt_sgl(txq, &sgl);
 
 			m->m_nextpkt = next;
 			break;
 		}
 
 		ETHER_BPF_MTAP(ifp, m);
 		if (sgl.nsegs == 0)
 			m_freem(m);
 doorbell:
 		if (eq->pending >= 8)
 			ring_eq_db(sc, eq);
 
 		can_reclaim = reclaimable(eq);
 		if (can_reclaim >= 32)
 			reclaim_tx_descs(txq, can_reclaim, 64);
 	}
 
 	if (txpkts.npkt > 0)
 		write_txpkts_wr(txq, &txpkts);
 
 	/*
 	 * m not NULL means there was an error but we haven't thrown it away.
 	 * This can happen when we're short of tx descriptors (no_desc) or maybe
 	 * even DMA maps (no_dmamap).  Either way, a credit flush and reclaim
 	 * will get things going again.
 	 */
 	if (m && !(eq->flags & EQ_CRFLUSHED)) {
 		struct tx_sdesc *txsd = &txq->sdesc[eq->pidx];
 
 		/*
 		 * If EQ_CRFLUSHED is not set then we know we have at least one
 		 * available descriptor because any WR that reduces eq->avail to
 		 * 0 also sets EQ_CRFLUSHED.
 		 */
 		KASSERT(eq->avail > 0, ("%s: no space for eqflush.", __func__));
 
 		txsd->desc_used = 1;
 		txsd->credits = 0;
 		write_eqflush_wr(eq);
 	}
 	txq->m = m;
 
 	if (eq->pending)
 		ring_eq_db(sc, eq);
 
 	reclaim_tx_descs(txq, 0, 128);
 
 	if (eq->flags & EQ_STALLED && callout_pending(&eq->tx_callout) == 0)
 		callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
 
 	return (0);
 }
 
 void
 t4_update_fl_bufsize(struct ifnet *ifp)
 {
 	struct port_info *pi = ifp->if_softc;
 	struct adapter *sc = pi->adapter;
 	struct sge_rxq *rxq;
 #ifdef TCP_OFFLOAD
 	struct sge_ofld_rxq *ofld_rxq;
 #endif
 	struct sge_fl *fl;
 	int i, maxp, mtu = ifp->if_mtu;
 
 	maxp = mtu_to_max_payload(sc, mtu, 0);
 	for_each_rxq(pi, i, rxq) {
 		fl = &rxq->fl;
 
 		FL_LOCK(fl);
 		find_best_refill_source(sc, fl, maxp);
 		FL_UNLOCK(fl);
 	}
 #ifdef TCP_OFFLOAD
 	maxp = mtu_to_max_payload(sc, mtu, 1);
 	for_each_ofld_rxq(pi, i, ofld_rxq) {
 		fl = &ofld_rxq->fl;
 
 		FL_LOCK(fl);
 		find_best_refill_source(sc, fl, maxp);
 		FL_UNLOCK(fl);
 	}
 #endif
 }
 
 int
 can_resume_tx(struct sge_eq *eq)
 {
 
 	return (eq->avail + reclaimable(eq) >= tx_resume_threshold(eq));
 }
 
 static inline void
 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx,
-    int qsize, int esize)
+    int qsize)
 {
+
 	KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS,
 	    ("%s: bad tmr_idx %d", __func__, tmr_idx));
 	KASSERT(pktc_idx < SGE_NCOUNTERS,	/* -ve is ok, means don't use */
 	    ("%s: bad pktc_idx %d", __func__, pktc_idx));
 
 	iq->flags = 0;
 	iq->adapter = sc;
 	iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
 	iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
 	if (pktc_idx >= 0) {
 		iq->intr_params |= F_QINTR_CNT_EN;
 		iq->intr_pktc_idx = pktc_idx;
 	}
 	iq->qsize = roundup2(qsize, 16);	/* See FW_IQ_CMD/iqsize */
-	iq->esize = max(esize, 16);		/* See FW_IQ_CMD/iqesize */
+	iq->sidx = iq->qsize - spg_len / IQ_ESIZE;
 }
 
 static inline void
 init_fl(struct adapter *sc, struct sge_fl *fl, int qsize, int maxp, int pack,
     char *name)
 {
 
 	fl->qsize = qsize;
+	fl->sidx = qsize - spg_len / EQ_ESIZE;
 	strlcpy(fl->lockname, name, sizeof(fl->lockname));
 	if (pack)
 		fl->flags |= FL_BUF_PACKING;
 	find_best_refill_source(sc, fl, maxp);
 	find_safe_refill_source(sc, fl);
 }
 
 static inline void
 init_eq(struct sge_eq *eq, int eqtype, int qsize, uint8_t tx_chan,
     uint16_t iqid, char *name)
 {
 	KASSERT(tx_chan < NCHAN, ("%s: bad tx channel %d", __func__, tx_chan));
 	KASSERT(eqtype <= EQ_TYPEMASK, ("%s: bad qtype %d", __func__, eqtype));
 
 	eq->flags = eqtype & EQ_TYPEMASK;
 	eq->tx_chan = tx_chan;
 	eq->iqid = iqid;
 	eq->qsize = qsize;
 	strlcpy(eq->lockname, name, sizeof(eq->lockname));
 
 	TASK_INIT(&eq->tx_task, 0, t4_tx_task, eq);
 	callout_init(&eq->tx_callout, CALLOUT_MPSAFE);
 }
 
 static int
 alloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag,
     bus_dmamap_t *map, bus_addr_t *pa, void **va)
 {
 	int rc;
 
 	rc = bus_dma_tag_create(sc->dmat, 512, 0, BUS_SPACE_MAXADDR,
 	    BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, 0, NULL, NULL, tag);
 	if (rc != 0) {
 		device_printf(sc->dev, "cannot allocate DMA tag: %d\n", rc);
 		goto done;
 	}
 
 	rc = bus_dmamem_alloc(*tag, va,
 	    BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, map);
 	if (rc != 0) {
 		device_printf(sc->dev, "cannot allocate DMA memory: %d\n", rc);
 		goto done;
 	}
 
 	rc = bus_dmamap_load(*tag, *map, *va, len, oneseg_dma_callback, pa, 0);
 	if (rc != 0) {
 		device_printf(sc->dev, "cannot load DMA map: %d\n", rc);
 		goto done;
 	}
 done:
 	if (rc)
 		free_ring(sc, *tag, *map, *pa, *va);
 
 	return (rc);
 }
 
 static int
 free_ring(struct adapter *sc, bus_dma_tag_t tag, bus_dmamap_t map,
     bus_addr_t pa, void *va)
 {
 	if (pa)
 		bus_dmamap_unload(tag, map);
 	if (va)
 		bus_dmamem_free(tag, va, map);
 	if (tag)
 		bus_dma_tag_destroy(tag);
 
 	return (0);
 }
 
 /*
  * Allocates the ring for an ingress queue and an optional freelist.  If the
  * freelist is specified it will be allocated and then associated with the
  * ingress queue.
  *
  * Returns errno on failure.  Resources allocated up to that point may still be
  * allocated.  Caller is responsible for cleanup in case this function fails.
  *
  * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then
  * the intr_idx specifies the vector, starting from 0.  Otherwise it specifies
  * the abs_id of the ingress queue to which its interrupts should be forwarded.
  */
 static int
 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
     int intr_idx, int cong)
 {
 	int rc, i, cntxt_id;
 	size_t len;
 	struct fw_iq_cmd c;
 	struct adapter *sc = iq->adapter;
 	__be32 v = 0;
 
-	len = iq->qsize * iq->esize;
+	len = iq->qsize * IQ_ESIZE;
 	rc = alloc_ring(sc, len, &iq->desc_tag, &iq->desc_map, &iq->ba,
 	    (void **)&iq->desc);
 	if (rc != 0)
 		return (rc);
 
 	bzero(&c, sizeof(c));
 	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
 	    V_FW_IQ_CMD_VFN(0));
 
 	c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
 	    FW_LEN16(c));
 
 	/* Special handling for firmware event queue */
 	if (iq == &sc->sge.fwq)
 		v |= F_FW_IQ_CMD_IQASYNCH;
 
 	if (iq->flags & IQ_INTR) {
 		KASSERT(intr_idx < sc->intr_count,
 		    ("%s: invalid direct intr_idx %d", __func__, intr_idx));
 	} else
 		v |= F_FW_IQ_CMD_IQANDST;
 	v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx);
 
 	c.type_to_iqandstindex = htobe32(v |
 	    V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
 	    V_FW_IQ_CMD_VIID(pi->viid) |
 	    V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
 	c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
 	    F_FW_IQ_CMD_IQGTSMODE |
 	    V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) |
-	    V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4));
+	    V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4));
 	c.iqsize = htobe16(iq->qsize);
 	c.iqaddr = htobe64(iq->ba);
 	if (cong >= 0)
 		c.iqns_to_fl0congen = htobe32(F_FW_IQ_CMD_IQFLINTCONGEN);
 
 	if (fl) {
 		mtx_init(&fl->fl_lock, fl->lockname, NULL, MTX_DEF);
 
-		len = fl->qsize * RX_FL_ESIZE;
+		len = fl->qsize * EQ_ESIZE;
 		rc = alloc_ring(sc, len, &fl->desc_tag, &fl->desc_map,
 		    &fl->ba, (void **)&fl->desc);
 		if (rc)
 			return (rc);
 
 		/* Allocate space for one software descriptor per buffer. */
-		fl->cap = (fl->qsize - spg_len / RX_FL_ESIZE) * 8;
 		rc = alloc_fl_sdesc(fl);
 		if (rc != 0) {
 			device_printf(sc->dev,
 			    "failed to setup fl software descriptors: %d\n",
 			    rc);
 			return (rc);
 		}
-		fl->needed = fl->cap;
-		fl->lowat = fl->flags & FL_BUF_PACKING ?
-		    roundup2(sc->sge.fl_starve_threshold2, 8) :
-		    roundup2(sc->sge.fl_starve_threshold, 8);
 
+		if (fl->flags & FL_BUF_PACKING) {
+			fl->lowat = roundup2(sc->sge.fl_starve_threshold2, 8);
+			fl->buf_boundary = max(fl_pad, sc->sge.pack_boundary);
+		} else {
+			fl->lowat = roundup2(sc->sge.fl_starve_threshold, 8);
+			fl->buf_boundary = fl_pad;
+		}
+
 		c.iqns_to_fl0congen |=
 		    htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
 			F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
 			(fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0) |
 			(fl->flags & FL_BUF_PACKING ? F_FW_IQ_CMD_FL0PACKEN :
 			    0));
 		if (cong >= 0) {
 			c.iqns_to_fl0congen |=
 				htobe32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) |
 				    F_FW_IQ_CMD_FL0CONGCIF |
 				    F_FW_IQ_CMD_FL0CONGEN);
 		}
 		c.fl0dcaen_to_fl0cidxfthresh =
 		    htobe16(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_64B) |
 			V_FW_IQ_CMD_FL0FBMAX(X_FETCHBURSTMAX_512B));
 		c.fl0size = htobe16(fl->qsize);
 		c.fl0addr = htobe64(fl->ba);
 	}
 
 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
 	if (rc != 0) {
 		device_printf(sc->dev,
 		    "failed to create ingress queue: %d\n", rc);
 		return (rc);
 	}
 
-	iq->cdesc = iq->desc;
 	iq->cidx = 0;
-	iq->gen = 1;
+	iq->gen = F_RSPD_GEN;
 	iq->intr_next = iq->intr_params;
 	iq->cntxt_id = be16toh(c.iqid);
 	iq->abs_id = be16toh(c.physiqid);
 	iq->flags |= IQ_ALLOCATED;
 
 	cntxt_id = iq->cntxt_id - sc->sge.iq_start;
 	if (cntxt_id >= sc->sge.niq) {
 		panic ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
 		    cntxt_id, sc->sge.niq - 1);
 	}
 	sc->sge.iqmap[cntxt_id] = iq;
 
 	if (fl) {
+		u_int qid;
+
+		iq->flags |= IQ_HAS_FL;
 		fl->cntxt_id = be16toh(c.fl0id);
 		fl->pidx = fl->cidx = 0;
 
 		cntxt_id = fl->cntxt_id - sc->sge.eq_start;
 		if (cntxt_id >= sc->sge.neq) {
 			panic("%s: fl->cntxt_id (%d) more than the max (%d)",
 			    __func__, cntxt_id, sc->sge.neq - 1);
 		}
 		sc->sge.eqmap[cntxt_id] = (void *)fl;
 
+		qid = fl->cntxt_id;
+		if (isset(&sc->doorbells, DOORBELL_UDB)) {
+			uint32_t s_qpp = sc->sge.eq_s_qpp;
+			uint32_t mask = (1 << s_qpp) - 1;
+			volatile uint8_t *udb;
+
+			udb = sc->udbs_base + UDBS_DB_OFFSET;
+			udb += (qid >> s_qpp) << PAGE_SHIFT;
+			qid &= mask;
+			if (qid < PAGE_SIZE / UDBS_SEG_SIZE) {
+				udb += qid << UDBS_SEG_SHIFT;
+				qid = 0;
+			}
+			fl->udb = (volatile void *)udb;
+		}
+		fl->dbval = F_DBPRIO | V_QID(qid);
+		if (is_t5(sc))
+			fl->dbval |= F_DBTYPE;
+
 		FL_LOCK(fl);
 		/* Enough to make sure the SGE doesn't think it's starved */
 		refill_fl(sc, fl, fl->lowat);
 		FL_UNLOCK(fl);
-
-		iq->flags |= IQ_HAS_FL;
 	}
 
 	if (is_t5(sc) && cong >= 0) {
 		uint32_t param, val;
 
 		param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
 		    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
 		    V_FW_PARAMS_PARAM_YZ(iq->cntxt_id);
 		if (cong == 0)
 			val = 1 << 19;
 		else {
 			val = 2 << 19;
 			for (i = 0; i < 4; i++) {
 				if (cong & (1 << i))
 					val |= 1 << (i << 2);
 			}
 		}
 
 		rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
 		if (rc != 0) {
 			/* report error but carry on */
 			device_printf(sc->dev,
 			    "failed to set congestion manager context for "
 			    "ingress queue %d: %d\n", iq->cntxt_id, rc);
 		}
 	}
 
 	/* Enable IQ interrupts */
 	atomic_store_rel_int(&iq->state, IQS_IDLE);
 	t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) |
 	    V_INGRESSQID(iq->cntxt_id));
 
 	return (0);
 }
 
 static int
 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl)
 {
 	int rc;
 	struct adapter *sc = iq->adapter;
 	device_t dev;
 
 	if (sc == NULL)
 		return (0);	/* nothing to do */
 
 	dev = pi ? pi->dev : sc->dev;
 
 	if (iq->flags & IQ_ALLOCATED) {
 		rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0,
 		    FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id,
 		    fl ? fl->cntxt_id : 0xffff, 0xffff);
 		if (rc != 0) {
 			device_printf(dev,
 			    "failed to free queue %p: %d\n", iq, rc);
 			return (rc);
 		}
 		iq->flags &= ~IQ_ALLOCATED;
 	}
 
 	free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba, iq->desc);
 
 	bzero(iq, sizeof(*iq));
 
 	if (fl) {
 		free_ring(sc, fl->desc_tag, fl->desc_map, fl->ba,
 		    fl->desc);
 
 		if (fl->sdesc)
 			free_fl_sdesc(sc, fl);
 
 		if (mtx_initialized(&fl->fl_lock))
 			mtx_destroy(&fl->fl_lock);
 
 		bzero(fl, sizeof(*fl));
 	}
 
 	return (0);
 }
 
 static void
 add_fl_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
     struct sge_fl *fl)
 {
 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
 
 	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", CTLFLAG_RD, NULL,
 	    "freelist");
 	children = SYSCTL_CHILDREN(oid);
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id",
 	    CTLTYPE_INT | CTLFLAG_RD, &fl->cntxt_id, 0, sysctl_uint16, "I",
 	    "SGE context id of the freelist");
 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &fl->cidx,
 	    0, "consumer index");
 	if (fl->flags & FL_BUF_PACKING) {
 		SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rx_offset",
 		    CTLFLAG_RD, &fl->rx_offset, 0, "packing rx offset");
 	}
 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &fl->pidx,
 	    0, "producer index");
 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "mbuf_allocated",
 	    CTLFLAG_RD, &fl->mbuf_allocated, "# of mbuf allocated");
 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "mbuf_inlined",
 	    CTLFLAG_RD, &fl->mbuf_inlined, "# of mbuf inlined in clusters");
 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_allocated",
 	    CTLFLAG_RD, &fl->cl_allocated, "# of clusters allocated");
 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_recycled",
 	    CTLFLAG_RD, &fl->cl_recycled, "# of clusters recycled");
 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_fast_recycled",
 	    CTLFLAG_RD, &fl->cl_fast_recycled, "# of clusters recycled (fast)");
 }
 
 static int
 alloc_fwq(struct adapter *sc)
 {
 	int rc, intr_idx;
 	struct sge_iq *fwq = &sc->sge.fwq;
 	struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev);
 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
 
-	init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE);
+	init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE);
 	fwq->flags |= IQ_INTR;	/* always */
 	intr_idx = sc->intr_count > 1 ? 1 : 0;
 	rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1);
 	if (rc != 0) {
 		device_printf(sc->dev,
 		    "failed to create firmware event queue: %d\n", rc);
 		return (rc);
 	}
 
 	oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "fwq", CTLFLAG_RD,
 	    NULL, "firmware event queue");
 	children = SYSCTL_CHILDREN(oid);
 
 	SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "abs_id",
 	    CTLTYPE_INT | CTLFLAG_RD, &fwq->abs_id, 0, sysctl_uint16, "I",
 	    "absolute id of the queue");
 	SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "cntxt_id",
 	    CTLTYPE_INT | CTLFLAG_RD, &fwq->cntxt_id, 0, sysctl_uint16, "I",
 	    "SGE context id of the queue");
 	SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "cidx",
 	    CTLTYPE_INT | CTLFLAG_RD, &fwq->cidx, 0, sysctl_uint16, "I",
 	    "consumer index");
 
 	return (0);
 }
 
 static int
 free_fwq(struct adapter *sc)
 {
 	return free_iq_fl(NULL, &sc->sge.fwq, NULL);
 }
 
 static int
 alloc_mgmtq(struct adapter *sc)
 {
 	int rc;
 	struct sge_wrq *mgmtq = &sc->sge.mgmtq;
 	char name[16];
 	struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev);
 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
 
 	oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "mgmtq", CTLFLAG_RD,
 	    NULL, "management queue");
 
 	snprintf(name, sizeof(name), "%s mgmtq", device_get_nameunit(sc->dev));
 	init_eq(&mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan,
 	    sc->sge.fwq.cntxt_id, name);
 	rc = alloc_wrq(sc, NULL, mgmtq, oid);
 	if (rc != 0) {
 		device_printf(sc->dev,
 		    "failed to create management queue: %d\n", rc);
 		return (rc);
 	}
 
 	return (0);
 }
 
 static int
 free_mgmtq(struct adapter *sc)
 {
 
 	return free_wrq(sc, &sc->sge.mgmtq);
 }
 
 static inline int
 tnl_cong(struct port_info *pi)
 {
 
 	if (cong_drop == -1)
 		return (-1);
 	else if (cong_drop == 1)
 		return (0);
 	else
 		return (pi->rx_chan_map);
 }
 
 static int
 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int idx,
     struct sysctl_oid *oid)
 {
 	int rc;
 	struct sysctl_oid_list *children;
 	char name[16];
 
 	rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx, tnl_cong(pi));
 	if (rc != 0)
 		return (rc);
 
+	/*
+	 * The freelist is just barely above the starvation threshold right now,
+	 * fill it up a bit more.
+	 */
 	FL_LOCK(&rxq->fl);
-	refill_fl(pi->adapter, &rxq->fl, rxq->fl.needed / 8);
+	refill_fl(pi->adapter, &rxq->fl, 128);
 	FL_UNLOCK(&rxq->fl);
 
 #if defined(INET) || defined(INET6)
 	rc = tcp_lro_init(&rxq->lro);
 	if (rc != 0)
 		return (rc);
 	rxq->lro.ifp = pi->ifp; /* also indicates LRO init'ed */
 
 	if (pi->ifp->if_capenable & IFCAP_LRO)
 		rxq->iq.flags |= IQ_LRO_ENABLED;
 #endif
 	rxq->ifp = pi->ifp;
 
 	children = SYSCTL_CHILDREN(oid);
 
 	snprintf(name, sizeof(name), "%d", idx);
 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
 	    NULL, "rx queue");
 	children = SYSCTL_CHILDREN(oid);
 
 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "abs_id",
 	    CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.abs_id, 0, sysctl_uint16, "I",
 	    "absolute id of the queue");
 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cntxt_id",
 	    CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.cntxt_id, 0, sysctl_uint16, "I",
 	    "SGE context id of the queue");
 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cidx",
 	    CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.cidx, 0, sysctl_uint16, "I",
 	    "consumer index");
 #if defined(INET) || defined(INET6)
 	SYSCTL_ADD_INT(&pi->ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD,
 	    &rxq->lro.lro_queued, 0, NULL);
 	SYSCTL_ADD_INT(&pi->ctx, children, OID_AUTO, "lro_flushed", CTLFLAG_RD,
 	    &rxq->lro.lro_flushed, 0, NULL);
 #endif
 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "rxcsum", CTLFLAG_RD,
 	    &rxq->rxcsum, "# of times hardware assisted with checksum");
 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "vlan_extraction",
 	    CTLFLAG_RD, &rxq->vlan_extraction,
 	    "# of times hardware extracted 802.1Q tag");
 
 	add_fl_sysctls(&pi->ctx, oid, &rxq->fl);
 
 	return (rc);
 }
 
 static int
 free_rxq(struct port_info *pi, struct sge_rxq *rxq)
 {
 	int rc;
 
 #if defined(INET) || defined(INET6)
 	if (rxq->lro.ifp) {
 		tcp_lro_free(&rxq->lro);
 		rxq->lro.ifp = NULL;
 	}
 #endif
 
 	rc = free_iq_fl(pi, &rxq->iq, &rxq->fl);
 	if (rc == 0)
 		bzero(rxq, sizeof(*rxq));
 
 	return (rc);
 }
 
 #ifdef TCP_OFFLOAD
 static int
 alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq,
     int intr_idx, int idx, struct sysctl_oid *oid)
 {
 	int rc;
 	struct sysctl_oid_list *children;
 	char name[16];
 
 	rc = alloc_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx,
 	    pi->rx_chan_map);
 	if (rc != 0)
 		return (rc);
 
 	children = SYSCTL_CHILDREN(oid);
 
 	snprintf(name, sizeof(name), "%d", idx);
 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
 	    NULL, "rx queue");
 	children = SYSCTL_CHILDREN(oid);
 
 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "abs_id",
 	    CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.abs_id, 0, sysctl_uint16,
 	    "I", "absolute id of the queue");
 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cntxt_id",
 	    CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.cntxt_id, 0, sysctl_uint16,
 	    "I", "SGE context id of the queue");
 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cidx",
 	    CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.cidx, 0, sysctl_uint16, "I",
 	    "consumer index");
 
 	add_fl_sysctls(&pi->ctx, oid, &ofld_rxq->fl);
 
 	return (rc);
 }
 
 static int
 free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq)
 {
 	int rc;
 
 	rc = free_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl);
 	if (rc == 0)
 		bzero(ofld_rxq, sizeof(*ofld_rxq));
 
 	return (rc);
 }
 #endif
 
+#ifdef DEV_NETMAP
 static int
+alloc_nm_rxq(struct port_info *pi, struct sge_nm_rxq *nm_rxq, int intr_idx,
+    int idx, struct sysctl_oid *oid)
+{
+	int rc;
+	struct sysctl_oid_list *children;
+	struct sysctl_ctx_list *ctx;
+	char name[16];
+	size_t len;
+	struct adapter *sc = pi->adapter;
+	struct netmap_adapter *na = NA(pi->nm_ifp);
+
+	MPASS(na != NULL);
+
+	len = pi->qsize_rxq * IQ_ESIZE;
+	rc = alloc_ring(sc, len, &nm_rxq->iq_desc_tag, &nm_rxq->iq_desc_map,
+	    &nm_rxq->iq_ba, (void **)&nm_rxq->iq_desc);
+	if (rc != 0)
+		return (rc);
+
+	len = na->num_rx_desc * EQ_ESIZE + spg_len;
+	rc = alloc_ring(sc, len, &nm_rxq->fl_desc_tag, &nm_rxq->fl_desc_map,
+	    &nm_rxq->fl_ba, (void **)&nm_rxq->fl_desc);
+	if (rc != 0)
+		return (rc);
+
+	nm_rxq->pi = pi;
+	nm_rxq->nid = idx;
+	nm_rxq->iq_cidx = 0;
+	nm_rxq->iq_sidx = pi->qsize_rxq - spg_len / IQ_ESIZE;
+	nm_rxq->iq_gen = F_RSPD_GEN;
+	nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0;
+	nm_rxq->fl_sidx = na->num_rx_desc;
+	nm_rxq->intr_idx = intr_idx;
+
+	ctx = &pi->ctx;
+	children = SYSCTL_CHILDREN(oid);
+
+	snprintf(name, sizeof(name), "%d", idx);
+	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, name, CTLFLAG_RD, NULL,
+	    "rx queue");
+	children = SYSCTL_CHILDREN(oid);
+
+	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "abs_id",
+	    CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_abs_id, 0, sysctl_uint16,
+	    "I", "absolute id of the queue");
+	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id",
+	    CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_cntxt_id, 0, sysctl_uint16,
+	    "I", "SGE context id of the queue");
+	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx",
+	    CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_cidx, 0, sysctl_uint16, "I",
+	    "consumer index");
+
+	children = SYSCTL_CHILDREN(oid);
+	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", CTLFLAG_RD, NULL,
+	    "freelist");
+	children = SYSCTL_CHILDREN(oid);
+
+	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id",
+	    CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->fl_cntxt_id, 0, sysctl_uint16,
+	    "I", "SGE context id of the freelist");
+	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD,
+	    &nm_rxq->fl_cidx, 0, "consumer index");
+	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD,
+	    &nm_rxq->fl_pidx, 0, "producer index");
+
+	return (rc);
+}
+
+
+static int
+free_nm_rxq(struct port_info *pi, struct sge_nm_rxq *nm_rxq)
+{
+	struct adapter *sc = pi->adapter;
+
+	free_ring(sc, nm_rxq->iq_desc_tag, nm_rxq->iq_desc_map, nm_rxq->iq_ba,
+	    nm_rxq->iq_desc);
+	free_ring(sc, nm_rxq->fl_desc_tag, nm_rxq->fl_desc_map, nm_rxq->fl_ba,
+	    nm_rxq->fl_desc);
+
+	return (0);
+}
+
+static int
+alloc_nm_txq(struct port_info *pi, struct sge_nm_txq *nm_txq, int iqidx, int idx,
+    struct sysctl_oid *oid)
+{
+	int rc;
+	size_t len;
+	struct adapter *sc = pi->adapter;
+	struct netmap_adapter *na = NA(pi->nm_ifp);
+	char name[16];
+	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
+
+	len = na->num_tx_desc * EQ_ESIZE + spg_len;
+	rc = alloc_ring(sc, len, &nm_txq->desc_tag, &nm_txq->desc_map,
+	    &nm_txq->ba, (void **)&nm_txq->desc);
+	if (rc)
+		return (rc);
+
+	nm_txq->pidx = nm_txq->cidx = 0;
+	nm_txq->sidx = na->num_tx_desc;
+	nm_txq->nid = idx;
+	nm_txq->iqidx = iqidx;
+	nm_txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |
+	    V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf));
+
+	snprintf(name, sizeof(name), "%d", idx);
+	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
+	    NULL, "netmap tx queue");
+	children = SYSCTL_CHILDREN(oid);
+
+	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
+	    &nm_txq->cntxt_id, 0, "SGE context id of the queue");
+	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cidx",
+	    CTLTYPE_INT | CTLFLAG_RD, &nm_txq->cidx, 0, sysctl_uint16, "I",
+	    "consumer index");
+	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "pidx",
+	    CTLTYPE_INT | CTLFLAG_RD, &nm_txq->pidx, 0, sysctl_uint16, "I",
+	    "producer index");
+
+	return (rc);
+}
+
+static int
+free_nm_txq(struct port_info *pi, struct sge_nm_txq *nm_txq)
+{
+	struct adapter *sc = pi->adapter;
+
+	free_ring(sc, nm_txq->desc_tag, nm_txq->desc_map, nm_txq->ba,
+	    nm_txq->desc);
+
+	return (0);
+}
+#endif
+
+static int
 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
 {
 	int rc, cntxt_id;
 	struct fw_eq_ctrl_cmd c;
 
 	bzero(&c, sizeof(c));
 
 	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) |
 	    V_FW_EQ_CTRL_CMD_VFN(0));
 	c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC |
 	    F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c));
 	c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* XXX */
 	c.physeqid_pkd = htobe32(0);
 	c.fetchszm_to_iqid =
 	    htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
 		V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) |
 		F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
 	c.dcaen_to_eqsize =
 	    htobe32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
 		V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
 		V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
 		V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize));
 	c.eqaddr = htobe64(eq->ba);
 
 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
 	if (rc != 0) {
 		device_printf(sc->dev,
 		    "failed to create control queue %d: %d\n", eq->tx_chan, rc);
 		return (rc);
 	}
 	eq->flags |= EQ_ALLOCATED;
 
 	eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(be32toh(c.cmpliqid_eqid));
 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
 	if (cntxt_id >= sc->sge.neq)
 	    panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
 		cntxt_id, sc->sge.neq - 1);
 	sc->sge.eqmap[cntxt_id] = eq;
 
 	return (rc);
 }
 
 static int
 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
 {
 	int rc, cntxt_id;
 	struct fw_eq_eth_cmd c;
 
 	bzero(&c, sizeof(c));
 
 	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
 	    V_FW_EQ_ETH_CMD_VFN(0));
 	c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
 	    F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
 	c.autoequiqe_to_viid = htobe32(V_FW_EQ_ETH_CMD_VIID(pi->viid));
 	c.fetchszm_to_iqid =
 	    htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
 		V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
 		V_FW_EQ_ETH_CMD_IQID(eq->iqid));
 	c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
 		      V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
 		      V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
 		      V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize));
 	c.eqaddr = htobe64(eq->ba);
 
 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
 	if (rc != 0) {
 		device_printf(pi->dev,
 		    "failed to create Ethernet egress queue: %d\n", rc);
 		return (rc);
 	}
 	eq->flags |= EQ_ALLOCATED;
 
 	eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd));
 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
 	if (cntxt_id >= sc->sge.neq)
 	    panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
 		cntxt_id, sc->sge.neq - 1);
 	sc->sge.eqmap[cntxt_id] = eq;
 
 	return (rc);
 }
 
 #ifdef TCP_OFFLOAD
 static int
 ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
 {
 	int rc, cntxt_id;
 	struct fw_eq_ofld_cmd c;
 
 	bzero(&c, sizeof(c));
 
 	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST |
 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) |
 	    V_FW_EQ_OFLD_CMD_VFN(0));
 	c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC |
 	    F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c));
 	c.fetchszm_to_iqid =
 		htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
 		    V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) |
 		    F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid));
 	c.dcaen_to_eqsize =
 	    htobe32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
 		V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
 		V_FW_EQ_OFLD_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
 		V_FW_EQ_OFLD_CMD_EQSIZE(eq->qsize));
 	c.eqaddr = htobe64(eq->ba);
 
 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
 	if (rc != 0) {
 		device_printf(pi->dev,
 		    "failed to create egress queue for TCP offload: %d\n", rc);
 		return (rc);
 	}
 	eq->flags |= EQ_ALLOCATED;
 
 	eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(be32toh(c.eqid_pkd));
 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
 	if (cntxt_id >= sc->sge.neq)
 	    panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
 		cntxt_id, sc->sge.neq - 1);
 	sc->sge.eqmap[cntxt_id] = eq;
 
 	return (rc);
 }
 #endif
 
 static int
 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
 {
 	int rc;
 	size_t len;
 
 	mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF);
 
 	len = eq->qsize * EQ_ESIZE;
 	rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map,
 	    &eq->ba, (void **)&eq->desc);
 	if (rc)
 		return (rc);
 
 	eq->cap = eq->qsize - spg_len / EQ_ESIZE;
 	eq->spg = (void *)&eq->desc[eq->cap];
 	eq->avail = eq->cap - 1;	/* one less to avoid cidx = pidx */
 	eq->pidx = eq->cidx = 0;
 	eq->doorbells = sc->doorbells;
 
 	switch (eq->flags & EQ_TYPEMASK) {
 	case EQ_CTRL:
 		rc = ctrl_eq_alloc(sc, eq);
 		break;
 
 	case EQ_ETH:
 		rc = eth_eq_alloc(sc, pi, eq);
 		break;
 
 #ifdef TCP_OFFLOAD
 	case EQ_OFLD:
 		rc = ofld_eq_alloc(sc, pi, eq);
 		break;
 #endif
 
 	default:
 		panic("%s: invalid eq type %d.", __func__,
 		    eq->flags & EQ_TYPEMASK);
 	}
 	if (rc != 0) {
 		device_printf(sc->dev,
 		    "failed to allocate egress queue(%d): %d\n",
 		    eq->flags & EQ_TYPEMASK, rc);
 	}
 
 	eq->tx_callout.c_cpu = eq->cntxt_id % mp_ncpus;
 
 	if (isset(&eq->doorbells, DOORBELL_UDB) ||
 	    isset(&eq->doorbells, DOORBELL_UDBWC) ||
 	    isset(&eq->doorbells, DOORBELL_WCWR)) {
 		uint32_t s_qpp = sc->sge.eq_s_qpp;
 		uint32_t mask = (1 << s_qpp) - 1;
 		volatile uint8_t *udb;
 
 		udb = sc->udbs_base + UDBS_DB_OFFSET;
 		udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT;	/* pg offset */
 		eq->udb_qid = eq->cntxt_id & mask;		/* id in page */
-		if (eq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE)
+		if (eq->udb_qid >= PAGE_SIZE / UDBS_SEG_SIZE)
 	    		clrbit(&eq->doorbells, DOORBELL_WCWR);
 		else {
 			udb += eq->udb_qid << UDBS_SEG_SHIFT;	/* seg offset */
 			eq->udb_qid = 0;
 		}
 		eq->udb = (volatile void *)udb;
 	}
 
 	return (rc);
 }
 
 static int
 free_eq(struct adapter *sc, struct sge_eq *eq)
 {
 	int rc;
 
 	if (eq->flags & EQ_ALLOCATED) {
 		switch (eq->flags & EQ_TYPEMASK) {
 		case EQ_CTRL:
 			rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0,
 			    eq->cntxt_id);
 			break;
 
 		case EQ_ETH:
 			rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0,
 			    eq->cntxt_id);
 			break;
 
 #ifdef TCP_OFFLOAD
 		case EQ_OFLD:
 			rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0,
 			    eq->cntxt_id);
 			break;
 #endif
 
 		default:
 			panic("%s: invalid eq type %d.", __func__,
 			    eq->flags & EQ_TYPEMASK);
 		}
 		if (rc != 0) {
 			device_printf(sc->dev,
 			    "failed to free egress queue (%d): %d\n",
 			    eq->flags & EQ_TYPEMASK, rc);
 			return (rc);
 		}
 		eq->flags &= ~EQ_ALLOCATED;
 	}
 
 	free_ring(sc, eq->desc_tag, eq->desc_map, eq->ba, eq->desc);
 
 	if (mtx_initialized(&eq->eq_lock))
 		mtx_destroy(&eq->eq_lock);
 
 	bzero(eq, sizeof(*eq));
 	return (0);
 }
 
 static int
 alloc_wrq(struct adapter *sc, struct port_info *pi, struct sge_wrq *wrq,
     struct sysctl_oid *oid)
 {
 	int rc;
 	struct sysctl_ctx_list *ctx = pi ? &pi->ctx : &sc->ctx;
 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
 
 	rc = alloc_eq(sc, pi, &wrq->eq);
 	if (rc)
 		return (rc);
 
 	wrq->adapter = sc;
 	STAILQ_INIT(&wrq->wr_list);
 
 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
 	    &wrq->eq.cntxt_id, 0, "SGE context id of the queue");
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx",
 	    CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.cidx, 0, sysctl_uint16, "I",
 	    "consumer index");
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pidx",
 	    CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.pidx, 0, sysctl_uint16, "I",
 	    "producer index");
 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs", CTLFLAG_RD,
 	    &wrq->tx_wrs, "# of work requests");
 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "no_desc", CTLFLAG_RD,
 	    &wrq->no_desc, 0,
 	    "# of times queue ran out of hardware descriptors");
 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "unstalled", CTLFLAG_RD,
 	    &wrq->eq.unstalled, 0, "# of times queue recovered after stall");
 
 	return (rc);
 }
 
 static int
 free_wrq(struct adapter *sc, struct sge_wrq *wrq)
 {
 	int rc;
 
 	rc = free_eq(sc, &wrq->eq);
 	if (rc)
 		return (rc);
 
 	bzero(wrq, sizeof(*wrq));
 	return (0);
 }
 
 static int
 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx,
     struct sysctl_oid *oid)
 {
 	int rc;
 	struct adapter *sc = pi->adapter;
 	struct sge_eq *eq = &txq->eq;
 	char name[16];
 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
 
 	rc = alloc_eq(sc, pi, eq);
 	if (rc)
 		return (rc);
 
 	txq->ifp = pi->ifp;
 
 	txq->sdesc = malloc(eq->cap * sizeof(struct tx_sdesc), M_CXGBE,
 	    M_ZERO | M_WAITOK);
 	txq->br = buf_ring_alloc(eq->qsize, M_CXGBE, M_WAITOK, &eq->eq_lock);
 
 	rc = bus_dma_tag_create(sc->dmat, 1, 0, BUS_SPACE_MAXADDR,
 	    BUS_SPACE_MAXADDR, NULL, NULL, 64 * 1024, TX_SGL_SEGS,
 	    BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL, NULL, &txq->tx_tag);
 	if (rc != 0) {
 		device_printf(sc->dev,
 		    "failed to create tx DMA tag: %d\n", rc);
 		return (rc);
 	}
 
 	/*
 	 * We can stuff ~10 frames in an 8-descriptor txpkts WR (8 is the SGE
 	 * limit for any WR).  txq->no_dmamap events shouldn't occur if maps is
 	 * sized for the worst case.
 	 */
 	rc = t4_alloc_tx_maps(&txq->txmaps, txq->tx_tag, eq->qsize * 10 / 8,
 	    M_WAITOK);
 	if (rc != 0) {
 		device_printf(sc->dev, "failed to setup tx DMA maps: %d\n", rc);
 		return (rc);
 	}
 
 	snprintf(name, sizeof(name), "%d", idx);
 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
 	    NULL, "tx queue");
 	children = SYSCTL_CHILDREN(oid);
 
 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
 	    &eq->cntxt_id, 0, "SGE context id of the queue");
 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cidx",
 	    CTLTYPE_INT | CTLFLAG_RD, &eq->cidx, 0, sysctl_uint16, "I",
 	    "consumer index");
 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "pidx",
 	    CTLTYPE_INT | CTLFLAG_RD, &eq->pidx, 0, sysctl_uint16, "I",
 	    "producer index");
 
 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txcsum", CTLFLAG_RD,
 	    &txq->txcsum, "# of times hardware assisted with checksum");
 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "vlan_insertion",
 	    CTLFLAG_RD, &txq->vlan_insertion,
 	    "# of times hardware inserted 802.1Q tag");
 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "tso_wrs", CTLFLAG_RD,
 	    &txq->tso_wrs, "# of TSO work requests");
 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "imm_wrs", CTLFLAG_RD,
 	    &txq->imm_wrs, "# of work requests with immediate data");
 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "sgl_wrs", CTLFLAG_RD,
 	    &txq->sgl_wrs, "# of work requests with direct SGL");
 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkt_wrs", CTLFLAG_RD,
 	    &txq->txpkt_wrs, "# of txpkt work requests (one pkt/WR)");
 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts_wrs", CTLFLAG_RD,
 	    &txq->txpkts_wrs, "# of txpkts work requests (multiple pkts/WR)");
 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts_pkts", CTLFLAG_RD,
 	    &txq->txpkts_pkts, "# of frames tx'd using txpkts work requests");
 
 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "br_drops", CTLFLAG_RD,
 	    &txq->br->br_drops, "# of drops in the buf_ring for this queue");
 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "no_dmamap", CTLFLAG_RD,
 	    &txq->no_dmamap, 0, "# of times txq ran out of DMA maps");
 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "no_desc", CTLFLAG_RD,
 	    &txq->no_desc, 0, "# of times txq ran out of hardware descriptors");
 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "egr_update", CTLFLAG_RD,
 	    &eq->egr_update, 0, "egress update notifications from the SGE");
 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "unstalled", CTLFLAG_RD,
 	    &eq->unstalled, 0, "# of times txq recovered after stall");
 
 	return (rc);
 }
 
 static int
 free_txq(struct port_info *pi, struct sge_txq *txq)
 {
 	int rc;
 	struct adapter *sc = pi->adapter;
 	struct sge_eq *eq = &txq->eq;
 
 	rc = free_eq(sc, eq);
 	if (rc)
 		return (rc);
 
 	free(txq->sdesc, M_CXGBE);
 
 	if (txq->txmaps.maps)
 		t4_free_tx_maps(&txq->txmaps, txq->tx_tag);
 
 	buf_ring_free(txq->br, M_CXGBE);
 
 	if (txq->tx_tag)
 		bus_dma_tag_destroy(txq->tx_tag);
 
 	bzero(txq, sizeof(*txq));
 	return (0);
 }
 
 static void
 oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	bus_addr_t *ba = arg;
 
 	KASSERT(nseg == 1,
 	    ("%s meant for single segment mappings only.", __func__));
 
 	*ba = error ? 0 : segs->ds_addr;
 }
 
-static inline bool
-is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl)
-{
-	*ctrl = (void *)((uintptr_t)iq->cdesc +
-	    (iq->esize - sizeof(struct rsp_ctrl)));
-
-	return (((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen);
-}
-
 static inline void
-iq_next(struct sge_iq *iq)
-{
-	iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize);
-	if (__predict_false(++iq->cidx == iq->qsize - 1)) {
-		iq->cidx = 0;
-		iq->gen ^= 1;
-		iq->cdesc = iq->desc;
-	}
-}
-
-#define FL_HW_IDX(x) ((x) >> 3)
-static inline void
 ring_fl_db(struct adapter *sc, struct sge_fl *fl)
 {
-	int ndesc = fl->pending / 8;
-	uint32_t v;
+	uint32_t n, v;
 
-	if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx))
-		ndesc--;	/* hold back one credit */
+	n = IDXDIFF(fl->pidx / 8, fl->dbidx, fl->sidx);
+	MPASS(n > 0);
 
-	if (ndesc <= 0)
-		return;		/* nothing to do */
-
-	v = F_DBPRIO | V_QID(fl->cntxt_id) | V_PIDX(ndesc);
-	if (is_t5(sc))
-		v |= F_DBTYPE;
-
 	wmb();
-
-	t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v);
-	fl->pending -= ndesc * 8;
+	v = fl->dbval | V_PIDX(n);
+	if (fl->udb)
+		*fl->udb = htole32(v);
+	else
+		t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v);
+	IDXINCR(fl->dbidx, n, fl->sidx);
 }
 
 /*
- * Fill up the freelist by upto nbufs and maybe ring its doorbell.
+ * Fills up the freelist by allocating upto 'n' buffers.  Buffers that are
+ * recycled do not count towards this allocation budget.
  *
- * Returns non-zero to indicate that it should be added to the list of starving
- * freelists.
+ * Returns non-zero to indicate that this freelist should be added to the list
+ * of starving freelists.
  */
 static int
-refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
+refill_fl(struct adapter *sc, struct sge_fl *fl, int n)
 {
-	__be64 *d = &fl->desc[fl->pidx];
-	struct fl_sdesc *sd = &fl->sdesc[fl->pidx];
+	__be64 *d;
+	struct fl_sdesc *sd;
 	uintptr_t pa;
 	caddr_t cl;
-	struct cluster_layout *cll = &fl->cll_def;	/* default layout */
-	struct sw_zone_info *swz = &sc->sge.sw_zone_info[cll->zidx];
+	struct cluster_layout *cll;
+	struct sw_zone_info *swz;
 	struct cluster_metadata *clm;
+	uint16_t max_pidx;
+	uint16_t hw_cidx = fl->hw_cidx;		/* stable snapshot */
 
 	FL_LOCK_ASSERT_OWNED(fl);
 
-	if (nbufs > fl->needed)
-		nbufs = fl->needed;
-	nbufs -= (fl->pidx + nbufs) % 8;
+	/*
+	 * We always stop at the begining of the hardware descriptor that's just
+	 * before the one with the hw cidx.  This is to avoid hw pidx = hw cidx,
+	 * which would mean an empty freelist to the chip.
+	 */
+	max_pidx = __predict_false(hw_cidx == 0) ? fl->sidx - 1 : hw_cidx - 1;
+	if (fl->pidx == max_pidx * 8)
+		return (0);
 
-	while (nbufs--) {
+	d = &fl->desc[fl->pidx];
+	sd = &fl->sdesc[fl->pidx];
+	cll = &fl->cll_def;	/* default layout */
+	swz = &sc->sge.sw_zone_info[cll->zidx];
 
+	while (n > 0) {
+
 		if (sd->cl != NULL) {
 
 			if (sd->nmbuf == 0) {
 				/*
 				 * Fast recycle without involving any atomics on
 				 * the cluster's metadata (if the cluster has
 				 * metadata).  This happens when all frames
 				 * received in the cluster were small enough to
 				 * fit within a single mbuf each.
 				 */
 				fl->cl_fast_recycled++;
 #ifdef INVARIANTS
 				clm = cl_metadata(sc, fl, &sd->cll, sd->cl);
 				if (clm != NULL)
 					MPASS(clm->refcount == 1);
 #endif
 				goto recycled_fast;
 			}
 
 			/*
 			 * Cluster is guaranteed to have metadata.  Clusters
 			 * without metadata always take the fast recycle path
 			 * when they're recycled.
 			 */
 			clm = cl_metadata(sc, fl, &sd->cll, sd->cl);
 			MPASS(clm != NULL);
 
 			if (atomic_fetchadd_int(&clm->refcount, -1) == 1) {
 				fl->cl_recycled++;
 				counter_u64_add(extfree_rels, 1);
 				goto recycled;
 			}
 			sd->cl = NULL;	/* gave up my reference */
 		}
 		MPASS(sd->cl == NULL);
 alloc:
 		cl = uma_zalloc(swz->zone, M_NOWAIT);
 		if (__predict_false(cl == NULL)) {
 			if (cll == &fl->cll_alt || fl->cll_alt.zidx == -1 ||
 			    fl->cll_def.zidx == fl->cll_alt.zidx)
 				break;
 
 			/* fall back to the safe zone */
 			cll = &fl->cll_alt;
 			swz = &sc->sge.sw_zone_info[cll->zidx];
 			goto alloc;
 		}
 		fl->cl_allocated++;
+		n--;
 
 		pa = pmap_kextract((vm_offset_t)cl);
 		pa += cll->region1;
 		sd->cl = cl;
 		sd->cll = *cll;
 		*d = htobe64(pa | cll->hwidx);
 		clm = cl_metadata(sc, fl, cll, cl);
 		if (clm != NULL) {
 recycled:
 #ifdef INVARIANTS
 			clm->sd = sd;
 #endif
 			clm->refcount = 1;
 		}
 		sd->nmbuf = 0;
 recycled_fast:
-		fl->pending++;
-		fl->needed--;
 		d++;
 		sd++;
-		if (__predict_false(++fl->pidx == fl->cap)) {
-			fl->pidx = 0;
-			sd = fl->sdesc;
-			d = fl->desc;
+		if (__predict_false(++fl->pidx % 8 == 0)) {
+			uint16_t pidx = fl->pidx / 8;
+
+			if (__predict_false(pidx == fl->sidx)) {
+				fl->pidx = 0;
+				pidx = 0;
+				sd = fl->sdesc;
+				d = fl->desc;
+			}
+			if (pidx == max_pidx)
+				break;
+
+			if (IDXDIFF(pidx, fl->dbidx, fl->sidx) >= 4)
+				ring_fl_db(sc, fl);
 		}
 	}
 
-	if (fl->pending >= 8)
+	if (fl->pidx / 8 != fl->dbidx)
 		ring_fl_db(sc, fl);
 
 	return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
 }
 
 /*
  * Attempt to refill all starving freelists.
  */
 static void
 refill_sfl(void *arg)
 {
 	struct adapter *sc = arg;
 	struct sge_fl *fl, *fl_temp;
 
 	mtx_lock(&sc->sfl_lock);
 	TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) {
 		FL_LOCK(fl);
 		refill_fl(sc, fl, 64);
 		if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) {
 			TAILQ_REMOVE(&sc->sfl, fl, link);
 			fl->flags &= ~FL_STARVING;
 		}
 		FL_UNLOCK(fl);
 	}
 
 	if (!TAILQ_EMPTY(&sc->sfl))
 		callout_schedule(&sc->sfl_callout, hz / 5);
 	mtx_unlock(&sc->sfl_lock);
 }
 
 static int
 alloc_fl_sdesc(struct sge_fl *fl)
 {
 
-	fl->sdesc = malloc(fl->cap * sizeof(struct fl_sdesc), M_CXGBE,
+	fl->sdesc = malloc(fl->sidx * 8 * sizeof(struct fl_sdesc), M_CXGBE,
 	    M_ZERO | M_WAITOK);
 
 	return (0);
 }
 
 static void
 free_fl_sdesc(struct adapter *sc, struct sge_fl *fl)
 {
 	struct fl_sdesc *sd;
 	struct cluster_metadata *clm;
 	struct cluster_layout *cll;
 	int i;
 
 	sd = fl->sdesc;
-	for (i = 0; i < fl->cap; i++, sd++) {
+	for (i = 0; i < fl->sidx * 8; i++, sd++) {
 		if (sd->cl == NULL)
 			continue;
 
 		cll = &sd->cll;
 		clm = cl_metadata(sc, fl, cll, sd->cl);
 		if (sd->nmbuf == 0)
 			uma_zfree(sc->sge.sw_zone_info[cll->zidx].zone, sd->cl);
 		else if (clm && atomic_fetchadd_int(&clm->refcount, -1) == 1) {
 			uma_zfree(sc->sge.sw_zone_info[cll->zidx].zone, sd->cl);
 			counter_u64_add(extfree_rels, 1);
 		}
 		sd->cl = NULL;
 	}
 
 	free(fl->sdesc, M_CXGBE);
 	fl->sdesc = NULL;
 }
 
 int
 t4_alloc_tx_maps(struct tx_maps *txmaps, bus_dma_tag_t tx_tag, int count,
     int flags)
 {
 	struct tx_map *txm;
 	int i, rc;
 
 	txmaps->map_total = txmaps->map_avail = count;
 	txmaps->map_cidx = txmaps->map_pidx = 0;
 
 	txmaps->maps = malloc(count * sizeof(struct tx_map), M_CXGBE,
 	    M_ZERO | flags);
 
 	txm = txmaps->maps;
 	for (i = 0; i < count; i++, txm++) {
 		rc = bus_dmamap_create(tx_tag, 0, &txm->map);
 		if (rc != 0)
 			goto failed;
 	}
 
 	return (0);
 failed:
 	while (--i >= 0) {
 		txm--;
 		bus_dmamap_destroy(tx_tag, txm->map);
 	}
 	KASSERT(txm == txmaps->maps, ("%s: EDOOFUS", __func__));
 
 	free(txmaps->maps, M_CXGBE);
 	txmaps->maps = NULL;
 
 	return (rc);
 }
 
 void
 t4_free_tx_maps(struct tx_maps *txmaps, bus_dma_tag_t tx_tag)
 {
 	struct tx_map *txm;
 	int i;
 
 	txm = txmaps->maps;
 	for (i = 0; i < txmaps->map_total; i++, txm++) {
 
 		if (txm->m) {
 			bus_dmamap_unload(tx_tag, txm->map);
 			m_freem(txm->m);
 			txm->m = NULL;
 		}
 
 		bus_dmamap_destroy(tx_tag, txm->map);
 	}
 
 	free(txmaps->maps, M_CXGBE);
 	txmaps->maps = NULL;
 }
 
 /*
  * We'll do immediate data tx for non-TSO, but only when not coalescing.  We're
  * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes
  * of immediate data.
  */
 #define IMM_LEN ( \
       2 * EQ_ESIZE \
     - sizeof(struct fw_eth_tx_pkt_wr) \
     - sizeof(struct cpl_tx_pkt_core))
 
 /*
  * Returns non-zero on failure, no need to cleanup anything in that case.
  *
  * Note 1: We always try to defrag the mbuf if required and return EFBIG only
  * if the resulting chain still won't fit in a tx descriptor.
  *
  * Note 2: We'll pullup the mbuf chain if TSO is requested and the first mbuf
  * does not have the TCP header in it.
  */
 static int
 get_pkt_sgl(struct sge_txq *txq, struct mbuf **fp, struct sgl *sgl,
     int sgl_only)
 {
 	struct mbuf *m = *fp;
 	struct tx_maps *txmaps;
 	struct tx_map *txm;
 	int rc, defragged = 0, n;
 
 	TXQ_LOCK_ASSERT_OWNED(txq);
 
 	if (m->m_pkthdr.tso_segsz)
 		sgl_only = 1;	/* Do not allow immediate data with LSO */
 
 start:	sgl->nsegs = 0;
 
 	if (m->m_pkthdr.len <= IMM_LEN && !sgl_only)
 		return (0);	/* nsegs = 0 tells caller to use imm. tx */
 
 	txmaps = &txq->txmaps;
 	if (txmaps->map_avail == 0) {
 		txq->no_dmamap++;
 		return (ENOMEM);
 	}
 	txm = &txmaps->maps[txmaps->map_pidx];
 
 	if (m->m_pkthdr.tso_segsz && m->m_len < 50) {
 		*fp = m_pullup(m, 50);
 		m = *fp;
 		if (m == NULL)
 			return (ENOBUFS);
 	}
 
 	rc = bus_dmamap_load_mbuf_sg(txq->tx_tag, txm->map, m, sgl->seg,
 	    &sgl->nsegs, BUS_DMA_NOWAIT);
 	if (rc == EFBIG && defragged == 0) {
 		m = m_defrag(m, M_NOWAIT);
 		if (m == NULL)
 			return (EFBIG);
 
 		defragged = 1;
 		*fp = m;
 		goto start;
 	}
 	if (rc != 0)
 		return (rc);
 
 	txm->m = m;
 	txmaps->map_avail--;
 	if (++txmaps->map_pidx == txmaps->map_total)
 		txmaps->map_pidx = 0;
 
 	KASSERT(sgl->nsegs > 0 && sgl->nsegs <= TX_SGL_SEGS,
 	    ("%s: bad DMA mapping (%d segments)", __func__, sgl->nsegs));
 
 	/*
 	 * Store the # of flits required to hold this frame's SGL in nflits.  An
 	 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by
 	 * multiple (len0 + len1, addr0, addr1) tuples.  If addr1 is not used
 	 * then len1 must be set to 0.
 	 */
 	n = sgl->nsegs - 1;
 	sgl->nflits = (3 * n) / 2 + (n & 1) + 2;
 
 	return (0);
 }
 
 
 /*
  * Releases all the txq resources used up in the specified sgl.
  */
 static int
 free_pkt_sgl(struct sge_txq *txq, struct sgl *sgl)
 {
 	struct tx_maps *txmaps;
 	struct tx_map *txm;
 
 	TXQ_LOCK_ASSERT_OWNED(txq);
 
 	if (sgl->nsegs == 0)
 		return (0);	/* didn't use any map */
 
 	txmaps = &txq->txmaps;
 
 	/* 1 pkt uses exactly 1 map, back it out */
 
 	txmaps->map_avail++;
 	if (txmaps->map_pidx > 0)
 		txmaps->map_pidx--;
 	else
 		txmaps->map_pidx = txmaps->map_total - 1;
 
 	txm = &txmaps->maps[txmaps->map_pidx];
 	bus_dmamap_unload(txq->tx_tag, txm->map);
 	txm->m = NULL;
 
 	return (0);
 }
 
 static int
 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, struct mbuf *m,
     struct sgl *sgl)
 {
 	struct sge_eq *eq = &txq->eq;
 	struct fw_eth_tx_pkt_wr *wr;
 	struct cpl_tx_pkt_core *cpl;
 	uint32_t ctrl;	/* used in many unrelated places */
 	uint64_t ctrl1;
 	int nflits, ndesc, pktlen;
 	struct tx_sdesc *txsd;
 	caddr_t dst;
 
 	TXQ_LOCK_ASSERT_OWNED(txq);
 
 	pktlen = m->m_pkthdr.len;
 
 	/*
 	 * Do we have enough flits to send this frame out?
 	 */
 	ctrl = sizeof(struct cpl_tx_pkt_core);
 	if (m->m_pkthdr.tso_segsz) {
 		nflits = TXPKT_LSO_WR_HDR;
 		ctrl += sizeof(struct cpl_tx_pkt_lso_core);
 	} else
 		nflits = TXPKT_WR_HDR;
 	if (sgl->nsegs > 0)
 		nflits += sgl->nflits;
 	else {
 		nflits += howmany(pktlen, 8);
 		ctrl += pktlen;
 	}
 	ndesc = howmany(nflits, 8);
 	if (ndesc > eq->avail)
 		return (ENOMEM);
 
 	/* Firmware work request header */
 	wr = (void *)&eq->desc[eq->pidx];
 	wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
 	    V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
 	ctrl = V_FW_WR_LEN16(howmany(nflits, 2));
 	if (eq->avail == ndesc) {
 		if (!(eq->flags & EQ_CRFLUSHED)) {
 			ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
 			eq->flags |= EQ_CRFLUSHED;
 		}
 		eq->flags |= EQ_STALLED;
 	}
 
 	wr->equiq_to_len16 = htobe32(ctrl);
 	wr->r3 = 0;
 
 	if (m->m_pkthdr.tso_segsz) {
 		struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
 		struct ether_header *eh;
 		void *l3hdr;
 #if defined(INET) || defined(INET6)
 		struct tcphdr *tcp;
 #endif
 		uint16_t eh_type;
 
 		ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE |
 		    F_LSO_LAST_SLICE;
 
 		eh = mtod(m, struct ether_header *);
 		eh_type = ntohs(eh->ether_type);
 		if (eh_type == ETHERTYPE_VLAN) {
 			struct ether_vlan_header *evh = (void *)eh;
 
 			ctrl |= V_LSO_ETHHDR_LEN(1);
 			l3hdr = evh + 1;
 			eh_type = ntohs(evh->evl_proto);
 		} else
 			l3hdr = eh + 1;
 
 		switch (eh_type) {
 #ifdef INET6
 		case ETHERTYPE_IPV6:
 		{
 			struct ip6_hdr *ip6 = l3hdr;
 
 			/*
 			 * XXX-BZ For now we do not pretend to support
 			 * IPv6 extension headers.
 			 */
 			KASSERT(ip6->ip6_nxt == IPPROTO_TCP, ("%s: CSUM_TSO "
 			    "with ip6_nxt != TCP: %u", __func__, ip6->ip6_nxt));
 			tcp = (struct tcphdr *)(ip6 + 1);
 			ctrl |= F_LSO_IPV6;
 			ctrl |= V_LSO_IPHDR_LEN(sizeof(*ip6) >> 2) |
 			    V_LSO_TCPHDR_LEN(tcp->th_off);
 			break;
 		}
 #endif
 #ifdef INET
 		case ETHERTYPE_IP:
 		{
 			struct ip *ip = l3hdr;
 
 			tcp = (void *)((uintptr_t)ip + ip->ip_hl * 4);
 			ctrl |= V_LSO_IPHDR_LEN(ip->ip_hl) |
 			    V_LSO_TCPHDR_LEN(tcp->th_off);
 			break;
 		}
 #endif
 		default:
 			panic("%s: CSUM_TSO but no supported IP version "
 			    "(0x%04x)", __func__, eh_type);
 		}
 
 		lso->lso_ctrl = htobe32(ctrl);
 		lso->ipid_ofst = htobe16(0);
 		lso->mss = htobe16(m->m_pkthdr.tso_segsz);
 		lso->seqno_offset = htobe32(0);
 		lso->len = htobe32(pktlen);
 
 		cpl = (void *)(lso + 1);
 
 		txq->tso_wrs++;
 	} else
 		cpl = (void *)(wr + 1);
 
 	/* Checksum offload */
 	ctrl1 = 0;
 	if (!(m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)))
 		ctrl1 |= F_TXPKT_IPCSUM_DIS;
 	if (!(m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 |
 	    CSUM_TCP_IPV6 | CSUM_TSO)))
 		ctrl1 |= F_TXPKT_L4CSUM_DIS;
 	if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP |
 	    CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO))
 		txq->txcsum++;	/* some hardware assistance provided */
 
 	/* VLAN tag insertion */
 	if (m->m_flags & M_VLANTAG) {
 		ctrl1 |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m->m_pkthdr.ether_vtag);
 		txq->vlan_insertion++;
 	}
 
 	/* CPL header */
 	cpl->ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |
 	    V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
 	cpl->pack = 0;
 	cpl->len = htobe16(pktlen);
 	cpl->ctrl1 = htobe64(ctrl1);
 
 	/* Software descriptor */
 	txsd = &txq->sdesc[eq->pidx];
 	txsd->desc_used = ndesc;
 
 	eq->pending += ndesc;
 	eq->avail -= ndesc;
 	eq->pidx += ndesc;
 	if (eq->pidx >= eq->cap)
 		eq->pidx -= eq->cap;
 
 	/* SGL */
 	dst = (void *)(cpl + 1);
 	if (sgl->nsegs > 0) {
 		txsd->credits = 1;
 		txq->sgl_wrs++;
 		write_sgl_to_txd(eq, sgl, &dst);
 	} else {
 		txsd->credits = 0;
 		txq->imm_wrs++;
 		for (; m; m = m->m_next) {
 			copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len);
 #ifdef INVARIANTS
 			pktlen -= m->m_len;
 #endif
 		}
 #ifdef INVARIANTS
 		KASSERT(pktlen == 0, ("%s: %d bytes left.", __func__, pktlen));
 #endif
 
 	}
 
 	txq->txpkt_wrs++;
 	return (0);
 }
 
 /*
  * Returns 0 to indicate that m has been accepted into a coalesced tx work
  * request.  It has either been folded into txpkts or txpkts was flushed and m
  * has started a new coalesced work request (as the first frame in a fresh
  * txpkts).
  *
  * Returns non-zero to indicate a failure - caller is responsible for
  * transmitting m, if there was anything in txpkts it has been flushed.
  */
 static int
 add_to_txpkts(struct port_info *pi, struct sge_txq *txq, struct txpkts *txpkts,
     struct mbuf *m, struct sgl *sgl)
 {
 	struct sge_eq *eq = &txq->eq;
 	int can_coalesce;
 	struct tx_sdesc *txsd;
 	int flits;
 
 	TXQ_LOCK_ASSERT_OWNED(txq);
 
 	KASSERT(sgl->nsegs, ("%s: can't coalesce imm data", __func__));
 
 	if (txpkts->npkt > 0) {
 		flits = TXPKTS_PKT_HDR + sgl->nflits;
 		can_coalesce = m->m_pkthdr.tso_segsz == 0 &&
 		    txpkts->nflits + flits <= TX_WR_FLITS &&
 		    txpkts->nflits + flits <= eq->avail * 8 &&
 		    txpkts->plen + m->m_pkthdr.len < 65536;
 
 		if (can_coalesce) {
 			txpkts->npkt++;
 			txpkts->nflits += flits;
 			txpkts->plen += m->m_pkthdr.len;
 
 			txsd = &txq->sdesc[eq->pidx];
 			txsd->credits++;
 
 			return (0);
 		}
 
 		/*
 		 * Couldn't coalesce m into txpkts.  The first order of business
 		 * is to send txpkts on its way.  Then we'll revisit m.
 		 */
 		write_txpkts_wr(txq, txpkts);
 	}
 
 	/*
 	 * Check if we can start a new coalesced tx work request with m as
 	 * the first packet in it.
 	 */
 
 	KASSERT(txpkts->npkt == 0, ("%s: txpkts not empty", __func__));
 
 	flits = TXPKTS_WR_HDR + sgl->nflits;
 	can_coalesce = m->m_pkthdr.tso_segsz == 0 &&
 	    flits <= eq->avail * 8 && flits <= TX_WR_FLITS;
 
 	if (can_coalesce == 0)
 		return (EINVAL);
 
 	/*
 	 * Start a fresh coalesced tx WR with m as the first frame in it.
 	 */
 	txpkts->npkt = 1;
 	txpkts->nflits = flits;
 	txpkts->flitp = &eq->desc[eq->pidx].flit[2];
 	txpkts->plen = m->m_pkthdr.len;
 
 	txsd = &txq->sdesc[eq->pidx];
 	txsd->credits = 1;
 
 	return (0);
 }
 
 /*
  * Note that write_txpkts_wr can never run out of hardware descriptors (but
  * write_txpkt_wr can).  add_to_txpkts ensures that a frame is accepted for
  * coalescing only if sufficient hardware descriptors are available.
  */
 static void
 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts)
 {
 	struct sge_eq *eq = &txq->eq;
 	struct fw_eth_tx_pkts_wr *wr;
 	struct tx_sdesc *txsd;
 	uint32_t ctrl;
 	int ndesc;
 
 	TXQ_LOCK_ASSERT_OWNED(txq);
 
 	ndesc = howmany(txpkts->nflits, 8);
 
 	wr = (void *)&eq->desc[eq->pidx];
 	wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR));
 	ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2));
 	if (eq->avail == ndesc) {
 		if (!(eq->flags & EQ_CRFLUSHED)) {
 			ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
 			eq->flags |= EQ_CRFLUSHED;
 		}
 		eq->flags |= EQ_STALLED;
 	}
 	wr->equiq_to_len16 = htobe32(ctrl);
 	wr->plen = htobe16(txpkts->plen);
 	wr->npkt = txpkts->npkt;
 	wr->r3 = wr->type = 0;
 
 	/* Everything else already written */
 
 	txsd = &txq->sdesc[eq->pidx];
 	txsd->desc_used = ndesc;
 
 	KASSERT(eq->avail >= ndesc, ("%s: out of descriptors", __func__));
 
 	eq->pending += ndesc;
 	eq->avail -= ndesc;
 	eq->pidx += ndesc;
 	if (eq->pidx >= eq->cap)
 		eq->pidx -= eq->cap;
 
 	txq->txpkts_pkts += txpkts->npkt;
 	txq->txpkts_wrs++;
 	txpkts->npkt = 0;	/* emptied */
 }
 
 static inline void
 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
     struct txpkts *txpkts, struct mbuf *m, struct sgl *sgl)
 {
 	struct ulp_txpkt *ulpmc;
 	struct ulptx_idata *ulpsc;
 	struct cpl_tx_pkt_core *cpl;
 	struct sge_eq *eq = &txq->eq;
 	uintptr_t flitp, start, end;
 	uint64_t ctrl;
 	caddr_t dst;
 
 	KASSERT(txpkts->npkt > 0, ("%s: txpkts is empty", __func__));
 
 	start = (uintptr_t)eq->desc;
 	end = (uintptr_t)eq->spg;
 
 	/* Checksum offload */
 	ctrl = 0;
 	if (!(m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)))
 		ctrl |= F_TXPKT_IPCSUM_DIS;
 	if (!(m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 |
 	    CSUM_TCP_IPV6 | CSUM_TSO)))
 		ctrl |= F_TXPKT_L4CSUM_DIS;
 	if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP |
 	    CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO))
 		txq->txcsum++;	/* some hardware assistance provided */
 
 	/* VLAN tag insertion */
 	if (m->m_flags & M_VLANTAG) {
 		ctrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m->m_pkthdr.ether_vtag);
 		txq->vlan_insertion++;
 	}
 
 	/*
 	 * The previous packet's SGL must have ended at a 16 byte boundary (this
 	 * is required by the firmware/hardware).  It follows that flitp cannot
 	 * wrap around between the ULPTX master command and ULPTX subcommand (8
 	 * bytes each), and that it can not wrap around in the middle of the
 	 * cpl_tx_pkt_core either.
 	 */
 	flitp = (uintptr_t)txpkts->flitp;
 	KASSERT((flitp & 0xf) == 0,
 	    ("%s: last SGL did not end at 16 byte boundary: %p",
 	    __func__, txpkts->flitp));
 
 	/* ULP master command */
 	ulpmc = (void *)flitp;
 	ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0) |
 	    V_ULP_TXPKT_FID(eq->iqid));
 	ulpmc->len = htonl(howmany(sizeof(*ulpmc) + sizeof(*ulpsc) +
 	    sizeof(*cpl) + 8 * sgl->nflits, 16));
 
 	/* ULP subcommand */
 	ulpsc = (void *)(ulpmc + 1);
 	ulpsc->cmd_more = htobe32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) |
 	    F_ULP_TX_SC_MORE);
 	ulpsc->len = htobe32(sizeof(struct cpl_tx_pkt_core));
 
 	flitp += sizeof(*ulpmc) + sizeof(*ulpsc);
 	if (flitp == end)
 		flitp = start;
 
 	/* CPL_TX_PKT */
 	cpl = (void *)flitp;
 	cpl->ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |
 	    V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
 	cpl->pack = 0;
 	cpl->len = htobe16(m->m_pkthdr.len);
 	cpl->ctrl1 = htobe64(ctrl);
 
 	flitp += sizeof(*cpl);
 	if (flitp == end)
 		flitp = start;
 
 	/* SGL for this frame */
 	dst = (caddr_t)flitp;
 	txpkts->nflits += write_sgl_to_txd(eq, sgl, &dst);
 	txpkts->flitp = (void *)dst;
 
 	KASSERT(((uintptr_t)dst & 0xf) == 0,
 	    ("%s: SGL ends at %p (not a 16 byte boundary)", __func__, dst));
 }
 
 /*
  * If the SGL ends on an address that is not 16 byte aligned, this function will
  * add a 0 filled flit at the end.  It returns 1 in that case.
  */
 static int
 write_sgl_to_txd(struct sge_eq *eq, struct sgl *sgl, caddr_t *to)
 {
 	__be64 *flitp, *end;
 	struct ulptx_sgl *usgl;
 	bus_dma_segment_t *seg;
 	int i, padded;
 
 	KASSERT(sgl->nsegs > 0 && sgl->nflits > 0,
 	    ("%s: bad SGL - nsegs=%d, nflits=%d",
 	    __func__, sgl->nsegs, sgl->nflits));
 
 	KASSERT(((uintptr_t)(*to) & 0xf) == 0,
 	    ("%s: SGL must start at a 16 byte boundary: %p", __func__, *to));
 
 	flitp = (__be64 *)(*to);
 	end = flitp + sgl->nflits;
 	seg = &sgl->seg[0];
 	usgl = (void *)flitp;
 
 	/*
 	 * We start at a 16 byte boundary somewhere inside the tx descriptor
 	 * ring, so we're at least 16 bytes away from the status page.  There is
 	 * no chance of a wrap around in the middle of usgl (which is 16 bytes).
 	 */
 
 	usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
 	    V_ULPTX_NSGE(sgl->nsegs));
 	usgl->len0 = htobe32(seg->ds_len);
 	usgl->addr0 = htobe64(seg->ds_addr);
 	seg++;
 
 	if ((uintptr_t)end <= (uintptr_t)eq->spg) {
 
 		/* Won't wrap around at all */
 
 		for (i = 0; i < sgl->nsegs - 1; i++, seg++) {
 			usgl->sge[i / 2].len[i & 1] = htobe32(seg->ds_len);
 			usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ds_addr);
 		}
 		if (i & 1)
 			usgl->sge[i / 2].len[1] = htobe32(0);
 	} else {
 
 		/* Will wrap somewhere in the rest of the SGL */
 
 		/* 2 flits already written, write the rest flit by flit */
 		flitp = (void *)(usgl + 1);
 		for (i = 0; i < sgl->nflits - 2; i++) {
 			if ((uintptr_t)flitp == (uintptr_t)eq->spg)
 				flitp = (void *)eq->desc;
 			*flitp++ = get_flit(seg, sgl->nsegs - 1, i);
 		}
 		end = flitp;
 	}
 
 	if ((uintptr_t)end & 0xf) {
 		*(uint64_t *)end = 0;
 		end++;
 		padded = 1;
 	} else
 		padded = 0;
 
 	if ((uintptr_t)end == (uintptr_t)eq->spg)
 		*to = (void *)eq->desc;
 	else
 		*to = (void *)end;
 
 	return (padded);
 }
 
 static inline void
 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
 {
 	if (__predict_true((uintptr_t)(*to) + len <= (uintptr_t)eq->spg)) {
 		bcopy(from, *to, len);
 		(*to) += len;
 	} else {
 		int portion = (uintptr_t)eq->spg - (uintptr_t)(*to);
 
 		bcopy(from, *to, portion);
 		from += portion;
 		portion = len - portion;	/* remaining */
 		bcopy(from, (void *)eq->desc, portion);
 		(*to) = (caddr_t)eq->desc + portion;
 	}
 }
 
 static inline void
 ring_eq_db(struct adapter *sc, struct sge_eq *eq)
 {
 	u_int db, pending;
 
 	db = eq->doorbells;
 	pending = eq->pending;
 	if (pending > 1)
 		clrbit(&db, DOORBELL_WCWR);
 	eq->pending = 0;
 	wmb();
 
 	switch (ffs(db) - 1) {
 	case DOORBELL_UDB:
 		*eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(pending));
 		return;
 
 	case DOORBELL_WCWR: {
 		volatile uint64_t *dst, *src;
 		int i;
 
 		/*
 		 * Queues whose 128B doorbell segment fits in the page do not
 		 * use relative qid (udb_qid is always 0).  Only queues with
 		 * doorbell segments can do WCWR.
 		 */
 		KASSERT(eq->udb_qid == 0 && pending == 1,
 		    ("%s: inappropriate doorbell (0x%x, %d, %d) for eq %p",
 		    __func__, eq->doorbells, pending, eq->pidx, eq));
 
 		dst = (volatile void *)((uintptr_t)eq->udb + UDBS_WR_OFFSET -
 		    UDBS_DB_OFFSET);
 		i = eq->pidx ? eq->pidx - 1 : eq->cap - 1;
 		src = (void *)&eq->desc[i];
 		while (src != (void *)&eq->desc[i + 1])
 			*dst++ = *src++;
 		wmb();
 		return;
 	}
 
 	case DOORBELL_UDBWC:
 		*eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(pending));
 		wmb();
 		return;
 
 	case DOORBELL_KDB:
 		t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
 		    V_QID(eq->cntxt_id) | V_PIDX(pending));
 		return;
 	}
 }
 
 static inline int
 reclaimable(struct sge_eq *eq)
 {
 	unsigned int cidx;
 
 	cidx = eq->spg->cidx;	/* stable snapshot */
 	cidx = be16toh(cidx);
 
 	if (cidx >= eq->cidx)
 		return (cidx - eq->cidx);
 	else
 		return (cidx + eq->cap - eq->cidx);
 }
 
 /*
  * There are "can_reclaim" tx descriptors ready to be reclaimed.  Reclaim as
  * many as possible but stop when there are around "n" mbufs to free.
  *
  * The actual number reclaimed is provided as the return value.
  */
 static int
 reclaim_tx_descs(struct sge_txq *txq, int can_reclaim, int n)
 {
 	struct tx_sdesc *txsd;
 	struct tx_maps *txmaps;
 	struct tx_map *txm;
 	unsigned int reclaimed, maps;
 	struct sge_eq *eq = &txq->eq;
 
 	TXQ_LOCK_ASSERT_OWNED(txq);
 
 	if (can_reclaim == 0)
 		can_reclaim = reclaimable(eq);
 
 	maps = reclaimed = 0;
 	while (can_reclaim && maps < n) {
 		int ndesc;
 
 		txsd = &txq->sdesc[eq->cidx];
 		ndesc = txsd->desc_used;
 
 		/* Firmware doesn't return "partial" credits. */
 		KASSERT(can_reclaim >= ndesc,
 		    ("%s: unexpected number of credits: %d, %d",
 		    __func__, can_reclaim, ndesc));
 
 		maps += txsd->credits;
 
 		reclaimed += ndesc;
 		can_reclaim -= ndesc;
 
 		eq->cidx += ndesc;
 		if (__predict_false(eq->cidx >= eq->cap))
 			eq->cidx -= eq->cap;
 	}
 
 	txmaps = &txq->txmaps;
 	txm = &txmaps->maps[txmaps->map_cidx];
 	if (maps)
 		prefetch(txm->m);
 
 	eq->avail += reclaimed;
 	KASSERT(eq->avail < eq->cap,	/* avail tops out at (cap - 1) */
 	    ("%s: too many descriptors available", __func__));
 
 	txmaps->map_avail += maps;
 	KASSERT(txmaps->map_avail <= txmaps->map_total,
 	    ("%s: too many maps available", __func__));
 
 	while (maps--) {
 		struct tx_map *next;
 
 		next = txm + 1;
 		if (__predict_false(txmaps->map_cidx + 1 == txmaps->map_total))
 			next = txmaps->maps;
 		prefetch(next->m);
 
 		bus_dmamap_unload(txq->tx_tag, txm->map);
 		m_freem(txm->m);
 		txm->m = NULL;
 
 		txm = next;
 		if (__predict_false(++txmaps->map_cidx == txmaps->map_total))
 			txmaps->map_cidx = 0;
 	}
 
 	return (reclaimed);
 }
 
 static void
 write_eqflush_wr(struct sge_eq *eq)
 {
 	struct fw_eq_flush_wr *wr;
 
 	EQ_LOCK_ASSERT_OWNED(eq);
 	KASSERT(eq->avail > 0, ("%s: no descriptors left.", __func__));
 	KASSERT(!(eq->flags & EQ_CRFLUSHED), ("%s: flushed already", __func__));
 
 	wr = (void *)&eq->desc[eq->pidx];
 	bzero(wr, sizeof(*wr));
 	wr->opcode = FW_EQ_FLUSH_WR;
 	wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(sizeof(*wr) / 16) |
 	    F_FW_WR_EQUEQ | F_FW_WR_EQUIQ);
 
 	eq->flags |= (EQ_CRFLUSHED | EQ_STALLED);
 	eq->pending++;
 	eq->avail--;
 	if (++eq->pidx == eq->cap)
 		eq->pidx = 0;
 }
 
 static __be64
 get_flit(bus_dma_segment_t *sgl, int nsegs, int idx)
 {
 	int i = (idx / 3) * 2;
 
 	switch (idx % 3) {
 	case 0: {
 		__be64 rc;
 
 		rc = htobe32(sgl[i].ds_len);
 		if (i + 1 < nsegs)
 			rc |= (uint64_t)htobe32(sgl[i + 1].ds_len) << 32;
 
 		return (rc);
 	}
 	case 1:
 		return htobe64(sgl[i].ds_addr);
 	case 2:
 		return htobe64(sgl[i + 1].ds_addr);
 	}
 
 	return (0);
 }
 
 static void
 find_best_refill_source(struct adapter *sc, struct sge_fl *fl, int maxp)
 {
 	int8_t zidx, hwidx, idx;
 	uint16_t region1, region3;
 	int spare, spare_needed, n;
 	struct sw_zone_info *swz;
 	struct hw_buf_info *hwb, *hwb_list = &sc->sge.hw_buf_info[0];
 
 	/*
 	 * Buffer Packing: Look for PAGE_SIZE or larger zone which has a bufsize
 	 * large enough for the max payload and cluster metadata.  Otherwise
 	 * settle for the largest bufsize that leaves enough room in the cluster
 	 * for metadata.
 	 *
 	 * Without buffer packing: Look for the smallest zone which has a
 	 * bufsize large enough for the max payload.  Settle for the largest
 	 * bufsize available if there's nothing big enough for max payload.
 	 */
 	spare_needed = fl->flags & FL_BUF_PACKING ? CL_METADATA_SIZE : 0;
 	swz = &sc->sge.sw_zone_info[0];
 	hwidx = -1;
 	for (zidx = 0; zidx < SW_ZONE_SIZES; zidx++, swz++) {
 		if (swz->size > largest_rx_cluster) {
 			if (__predict_true(hwidx != -1))
 				break;
 
 			/*
 			 * This is a misconfiguration.  largest_rx_cluster is
 			 * preventing us from finding a refill source.  See
 			 * dev.t5nex.<n>.buffer_sizes to figure out why.
 			 */
 			device_printf(sc->dev, "largest_rx_cluster=%u leaves no"
 			    " refill source for fl %p (dma %u).  Ignored.\n",
 			    largest_rx_cluster, fl, maxp);
 		}
 		for (idx = swz->head_hwidx; idx != -1; idx = hwb->next) {
 			hwb = &hwb_list[idx];
 			spare = swz->size - hwb->size;
 			if (spare < spare_needed)
 				continue;
 
 			hwidx = idx;		/* best option so far */
 			if (hwb->size >= maxp) {
 
 				if ((fl->flags & FL_BUF_PACKING) == 0)
 					goto done; /* stop looking (not packing) */
 
 				if (swz->size >= safest_rx_cluster)
 					goto done; /* stop looking (packing) */
 			}
 			break;		/* keep looking, next zone */
 		}
 	}
 done:
 	/* A usable hwidx has been located. */
 	MPASS(hwidx != -1);
 	hwb = &hwb_list[hwidx];
 	zidx = hwb->zidx;
 	swz = &sc->sge.sw_zone_info[zidx];
 	region1 = 0;
 	region3 = swz->size - hwb->size;
 
 	/*
 	 * Stay within this zone and see if there is a better match when mbuf
 	 * inlining is allowed.  Remember that the hwidx's are sorted in
 	 * decreasing order of size (so in increasing order of spare area).
 	 */
 	for (idx = hwidx; idx != -1; idx = hwb->next) {
 		hwb = &hwb_list[idx];
 		spare = swz->size - hwb->size;
 
 		if (allow_mbufs_in_cluster == 0 || hwb->size < maxp)
 			break;
 		if (spare < CL_METADATA_SIZE + MSIZE)
 			continue;
 		n = (spare - CL_METADATA_SIZE) / MSIZE;
 		if (n > howmany(hwb->size, maxp))
 			break;
 
 		hwidx = idx;
 		if (fl->flags & FL_BUF_PACKING) {
 			region1 = n * MSIZE;
 			region3 = spare - region1;
 		} else {
 			region1 = MSIZE;
 			region3 = spare - region1;
 			break;
 		}
 	}
 
 	KASSERT(zidx >= 0 && zidx < SW_ZONE_SIZES,
 	    ("%s: bad zone %d for fl %p, maxp %d", __func__, zidx, fl, maxp));
 	KASSERT(hwidx >= 0 && hwidx <= SGE_FLBUF_SIZES,
 	    ("%s: bad hwidx %d for fl %p, maxp %d", __func__, hwidx, fl, maxp));
 	KASSERT(region1 + sc->sge.hw_buf_info[hwidx].size + region3 ==
 	    sc->sge.sw_zone_info[zidx].size,
 	    ("%s: bad buffer layout for fl %p, maxp %d. "
 		"cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp,
 		sc->sge.sw_zone_info[zidx].size, region1,
 		sc->sge.hw_buf_info[hwidx].size, region3));
 	if (fl->flags & FL_BUF_PACKING || region1 > 0) {
 		KASSERT(region3 >= CL_METADATA_SIZE,
 		    ("%s: no room for metadata.  fl %p, maxp %d; "
 		    "cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp,
 		    sc->sge.sw_zone_info[zidx].size, region1,
 		    sc->sge.hw_buf_info[hwidx].size, region3));
 		KASSERT(region1 % MSIZE == 0,
 		    ("%s: bad mbuf region for fl %p, maxp %d. "
 		    "cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp,
 		    sc->sge.sw_zone_info[zidx].size, region1,
 		    sc->sge.hw_buf_info[hwidx].size, region3));
 	}
 
 	fl->cll_def.zidx = zidx;
 	fl->cll_def.hwidx = hwidx;
 	fl->cll_def.region1 = region1;
 	fl->cll_def.region3 = region3;
 }
 
 static void
 find_safe_refill_source(struct adapter *sc, struct sge_fl *fl)
 {
 	struct sge *s = &sc->sge;
 	struct hw_buf_info *hwb;
 	struct sw_zone_info *swz;
 	int spare;
 	int8_t hwidx;
 
 	if (fl->flags & FL_BUF_PACKING)
 		hwidx = s->safe_hwidx2;	/* with room for metadata */
 	else if (allow_mbufs_in_cluster && s->safe_hwidx2 != -1) {
 		hwidx = s->safe_hwidx2;
 		hwb = &s->hw_buf_info[hwidx];
 		swz = &s->sw_zone_info[hwb->zidx];
 		spare = swz->size - hwb->size;
 
 		/* no good if there isn't room for an mbuf as well */
 		if (spare < CL_METADATA_SIZE + MSIZE)
 			hwidx = s->safe_hwidx1;
 	} else
 		hwidx = s->safe_hwidx1;
 
 	if (hwidx == -1) {
 		/* No fallback source */
 		fl->cll_alt.hwidx = -1;
 		fl->cll_alt.zidx = -1;
 
 		return;
 	}
 
 	hwb = &s->hw_buf_info[hwidx];
 	swz = &s->sw_zone_info[hwb->zidx];
 	spare = swz->size - hwb->size;
 	fl->cll_alt.hwidx = hwidx;
 	fl->cll_alt.zidx = hwb->zidx;
 	if (allow_mbufs_in_cluster)
 		fl->cll_alt.region1 = ((spare - CL_METADATA_SIZE) / MSIZE) * MSIZE;
 	else
 		fl->cll_alt.region1 = 0;
 	fl->cll_alt.region3 = spare - fl->cll_alt.region1;
 }
 
 static void
 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl)
 {
 	mtx_lock(&sc->sfl_lock);
 	FL_LOCK(fl);
 	if ((fl->flags & FL_DOOMED) == 0) {
 		fl->flags |= FL_STARVING;
 		TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
 		callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc);
 	}
 	FL_UNLOCK(fl);
 	mtx_unlock(&sc->sfl_lock);
 }
 
 static int
 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
     struct mbuf *m)
 {
 	const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
 	unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
 	struct adapter *sc = iq->adapter;
 	struct sge *s = &sc->sge;
 	struct sge_eq *eq;
 
 	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
 	    rss->opcode));
 
 	eq = s->eqmap[qid - s->eq_start];
 	EQ_LOCK(eq);
 	KASSERT(eq->flags & EQ_CRFLUSHED,
 	    ("%s: unsolicited egress update", __func__));
 	eq->flags &= ~EQ_CRFLUSHED;
 	eq->egr_update++;
 
 	if (__predict_false(eq->flags & EQ_DOOMED))
 		wakeup_one(eq);
 	else if (eq->flags & EQ_STALLED && can_resume_tx(eq))
 		taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
 	EQ_UNLOCK(eq);
 
 	return (0);
 }
 
 /* handle_fw_msg works for both fw4_msg and fw6_msg because this is valid */
 CTASSERT(offsetof(struct cpl_fw4_msg, data) == \
     offsetof(struct cpl_fw6_msg, data));
 
 static int
 handle_fw_msg(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
 	const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
 
 	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
 	    rss->opcode));
 
 	if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) {
 		const struct rss_header *rss2;
 
 		rss2 = (const struct rss_header *)&cpl->data[0];
 		return (sc->cpl_handler[rss2->opcode](iq, rss2, m));
 	}
 
 	return (sc->fw_msg_handler[cpl->type](sc, &cpl->data[0]));
 }
 
 static int
 sysctl_uint16(SYSCTL_HANDLER_ARGS)
 {
 	uint16_t *id = arg1;
 	int i = *id;
 
 	return sysctl_handle_int(oidp, &i, 0, req);
 }
 
 static int
 sysctl_bufsizes(SYSCTL_HANDLER_ARGS)
 {
 	struct sge *s = arg1;
 	struct hw_buf_info *hwb = &s->hw_buf_info[0];
 	struct sw_zone_info *swz = &s->sw_zone_info[0];
 	int i, rc;
 	struct sbuf sb;
 	char c;
 
 	sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
 	for (i = 0; i < SGE_FLBUF_SIZES; i++, hwb++) {
 		if (hwb->zidx >= 0 && swz[hwb->zidx].size <= largest_rx_cluster)
 			c = '*';
 		else
 			c = '\0';
 
 		sbuf_printf(&sb, "%u%c ", hwb->size, c);
 	}
 	sbuf_trim(&sb);
 	sbuf_finish(&sb);
 	rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
 	sbuf_delete(&sb);
 	return (rc);
 }
Index: stable/10/sys/dev/cxgbe/t4_tracer.c
===================================================================
--- stable/10/sys/dev/cxgbe/t4_tracer.c	(revision 270296)
+++ stable/10/sys/dev/cxgbe/t4_tracer.c	(revision 270297)
@@ -1,518 +1,517 @@
 /*-
  * Copyright (c) 2013 Chelsio Communications, Inc.
  * All rights reserved.
  * Written by: Navdeep Parhar <np@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
 #include "opt_inet6.h"
 
 #include <sys/param.h>
 #include <sys/lock.h>
 #include <sys/types.h>
 #include <sys/mbuf.h>
 #include <sys/socket.h>
 #include <sys/sockio.h>
 #include <sys/sx.h>
 #include <net/bpf.h>
 #include <net/ethernet.h>
 #include <net/if.h>
 #include <net/if_clone.h>
 #include <net/if_types.h>
 
 #include "common/common.h"
 #include "common/t4_msg.h"
 #include "common/t4_regs.h"
 #include "t4_ioctl.h"
 
 /*
  * Locking notes
  * =============
  *
  * An interface cloner is registered during mod_load and it can be used to
  * create or destroy the tracing ifnet for an adapter at any time.  It is
  * possible for the cloned interface to outlive the adapter (adapter disappears
  * in t4_detach but the tracing ifnet may live till mod_unload when removal of
  * the cloner finally destroys any remaining cloned interfaces).  When tracing
  * filters are active, this ifnet is also receiving data.  There are potential
  * bad races between ifnet create, ifnet destroy, ifnet rx, ifnet ioctl,
  * cxgbe_detach/t4_detach, mod_unload.
  *
  * a) The driver selects an iq for tracing (sc->traceq) inside a synch op.  The
  *    iq is destroyed inside a synch op too (and sc->traceq updated).
  * b) The cloner looks for an adapter that matches the name of the ifnet it's
  *    been asked to create, starts a synch op on that adapter, and proceeds only
  *    if the adapter has a tracing iq.
  * c) The cloned ifnet and the adapter are coupled to each other via
  *    ifp->if_softc and sc->ifp.  These can be modified only with the global
  *    t4_trace_lock sx as well as the sc->ifp_lock mutex held.  Holding either
  *    of these will prevent any change.
  *
  * The order in which all the locks involved should be acquired are:
  * t4_list_lock
  * adapter lock
  * (begin synch op and let go of the above two)
  * t4_trace_lock
  * sc->ifp_lock
  */
 
 static struct sx t4_trace_lock;
 static const char *t4_cloner_name = "tXnex";
 static struct if_clone *t4_cloner;
 
 /* tracer ifnet routines.  mostly no-ops. */
 static void tracer_init(void *);
 static int tracer_ioctl(struct ifnet *, unsigned long, caddr_t);
 static int tracer_transmit(struct ifnet *, struct mbuf *);
 static void tracer_qflush(struct ifnet *);
 static int tracer_media_change(struct ifnet *);
 static void tracer_media_status(struct ifnet *, struct ifmediareq *);
 
 /* match name (request/response) */
 struct match_rr {
 	const char *name;
 	int lock;	/* set to 1 to returned sc locked. */
 	struct adapter *sc;
 	int rc;
 };
 
 static void
 match_name(struct adapter *sc, void *arg)
 {
 	struct match_rr *mrr = arg;
 
 	if (strcmp(device_get_nameunit(sc->dev), mrr->name) != 0)
 		return;
 
 	KASSERT(mrr->sc == NULL, ("%s: multiple matches (%p, %p) for %s",
 	    __func__, mrr->sc, sc, mrr->name));
 
 	mrr->sc = sc;
 	if (mrr->lock)
 		mrr->rc = begin_synchronized_op(mrr->sc, NULL, 0, "t4clon");
 	else
 		mrr->rc = 0;
 }
 
 static int
 t4_cloner_match(struct if_clone *ifc, const char *name)
 {
 
 	if (strncmp(name, "t4nex", 5) != 0 &&
 	    strncmp(name, "t5nex", 5) != 0)
 		return (0);
 	if (name[5] < '0' || name[5] > '9')
 		return (0);
 	return (1);
 }
 
 static int
 t4_cloner_create(struct if_clone *ifc, char *name, size_t len, caddr_t params)
 {
 	struct match_rr mrr;
 	struct adapter *sc;
 	struct ifnet *ifp;
 	int rc, unit;
 	const uint8_t lla[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0};
 
 	mrr.name = name;
 	mrr.lock = 1;
 	mrr.sc = NULL;
 	mrr.rc = ENOENT;
 	t4_iterate(match_name, &mrr);
 
 	if (mrr.rc != 0)
 		return (mrr.rc);
 	sc = mrr.sc;
 
 	KASSERT(sc != NULL, ("%s: name (%s) matched but softc is NULL",
 	    __func__, name));
 	ASSERT_SYNCHRONIZED_OP(sc);
 
 	sx_xlock(&t4_trace_lock);
 
 	if (sc->ifp != NULL) {
 		rc = EEXIST;
 		goto done;
 	}
 	if (sc->traceq < 0) {
 		rc = EAGAIN;
 		goto done;
 	}
 
 
 	unit = -1;
 	rc = ifc_alloc_unit(ifc, &unit);
 	if (rc != 0)
 		goto done;
 
 	ifp = if_alloc(IFT_ETHER);
 	if (ifp == NULL) {
 		ifc_free_unit(ifc, unit);
 		rc = ENOMEM;
 		goto done;
 	}
 
 	/* Note that if_xname is not <if_dname><if_dunit>. */
 	strlcpy(ifp->if_xname, name, sizeof(ifp->if_xname));
 	ifp->if_dname = t4_cloner_name;
 	ifp->if_dunit = unit;
 	ifp->if_init = tracer_init;
 	ifp->if_flags = IFF_SIMPLEX | IFF_DRV_RUNNING;
 	ifp->if_ioctl = tracer_ioctl;
 	ifp->if_transmit = tracer_transmit;
 	ifp->if_qflush = tracer_qflush;
 	ifp->if_capabilities = IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
 	ifmedia_init(&sc->media, IFM_IMASK, tracer_media_change,
 	    tracer_media_status);
 	ifmedia_add(&sc->media, IFM_ETHER | IFM_FDX | IFM_NONE, 0, NULL);
 	ifmedia_set(&sc->media, IFM_ETHER | IFM_FDX | IFM_NONE);
 	ether_ifattach(ifp, lla);
-	if_up(ifp);
 
 	mtx_lock(&sc->ifp_lock);
 	ifp->if_softc = sc;
 	sc->ifp = ifp;
 	mtx_unlock(&sc->ifp_lock);
 done:
 	sx_xunlock(&t4_trace_lock);
 	end_synchronized_op(sc, 0);
 	return (rc);
 }
 
 static int
 t4_cloner_destroy(struct if_clone *ifc, struct ifnet *ifp)
 {
 	struct adapter *sc;
 	int unit = ifp->if_dunit;
 
 	sx_xlock(&t4_trace_lock);
 	sc = ifp->if_softc;
 	if (sc != NULL) {
 		mtx_lock(&sc->ifp_lock);
 		sc->ifp = NULL;
 		ifp->if_softc = NULL;
 		mtx_unlock(&sc->ifp_lock);
 		ifmedia_removeall(&sc->media);
 	}
 	ether_ifdetach(ifp);
 	if_free(ifp);
 	ifc_free_unit(ifc, unit);
 	sx_xunlock(&t4_trace_lock);
 
 	return (0);
 }
 
 void
 t4_tracer_modload()
 {
 
 	sx_init(&t4_trace_lock, "T4/T5 tracer lock");
 	t4_cloner = if_clone_advanced(t4_cloner_name, 0, t4_cloner_match,
 	    t4_cloner_create, t4_cloner_destroy);
 }
 
 void
 t4_tracer_modunload()
 {
 
 	if (t4_cloner != NULL) {
 		/*
 		 * The module is being unloaded so the nexus drivers have
 		 * detached.  The tracing interfaces can not outlive the nexus
 		 * (ifp->if_softc is the nexus) and must have been destroyed
 		 * already.  XXX: but if_clone is opaque to us and we can't
 		 * assert LIST_EMPTY(&t4_cloner->ifc_iflist) at this time.
 		 */
 		if_clone_detach(t4_cloner);
 	}
 	sx_destroy(&t4_trace_lock);
 }
 
 void
 t4_tracer_port_detach(struct adapter *sc)
 {
 
 	sx_xlock(&t4_trace_lock);
 	if (sc->ifp != NULL) {
 		mtx_lock(&sc->ifp_lock);
 		sc->ifp->if_softc = NULL;
 		sc->ifp = NULL;
 		mtx_unlock(&sc->ifp_lock);
 	}
 	ifmedia_removeall(&sc->media);
 	sx_xunlock(&t4_trace_lock);
 }
 
 int
 t4_get_tracer(struct adapter *sc, struct t4_tracer *t)
 {
 	int rc, i, enabled;
 	struct trace_params tp;
 
 	if (t->idx >= NTRACE) {
 		t->idx = 0xff;
 		t->enabled = 0;
 		t->valid = 0;
 		return (0);
 	}
 
 	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
 	    "t4gett");
 	if (rc)
 		return (rc);
 
 	for (i = t->idx; i < NTRACE; i++) {
 		if (isset(&sc->tracer_valid, t->idx)) {
 			t4_get_trace_filter(sc, &tp, i, &enabled);
 			t->idx = i;
 			t->enabled = enabled;
 			t->valid = 1;
 			memcpy(&t->tp.data[0], &tp.data[0], sizeof(t->tp.data));
 			memcpy(&t->tp.mask[0], &tp.mask[0], sizeof(t->tp.mask));
 			t->tp.snap_len = tp.snap_len;
 			t->tp.min_len = tp.min_len;
 			t->tp.skip_ofst = tp.skip_ofst;
 			t->tp.skip_len = tp.skip_len;
 			t->tp.invert = tp.invert;
 
 			/* convert channel to port iff 0 <= port < 8. */
 			if (tp.port < 4)
 				t->tp.port = sc->chan_map[tp.port];
 			else if (tp.port < 8)
 				t->tp.port = sc->chan_map[tp.port - 4] + 4;
 			else
 				t->tp.port = tp.port;
 
 			goto done;
 		}
 	}
 
 	t->idx = 0xff;
 	t->enabled = 0;
 	t->valid = 0;
 done:
 	end_synchronized_op(sc, LOCK_HELD);
 
 	return (rc);
 }
 
 int
 t4_set_tracer(struct adapter *sc, struct t4_tracer *t)
 {
 	int rc;
 	struct trace_params tp, *tpp;
 
 	if (t->idx >= NTRACE)
 		return (EINVAL);
 
 	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
 	    "t4sett");
 	if (rc)
 		return (rc);
 
 	/*
 	 * If no tracing filter is specified this time then check if the filter
 	 * at the index is valid anyway because it was set previously.  If so
 	 * then this is a legitimate enable/disable operation.
 	 */
 	if (t->valid == 0) {
 		if (isset(&sc->tracer_valid, t->idx))
 			tpp = NULL;
 		else
 			rc = EINVAL;
 		goto done;
 	}
 
 	if (t->tp.port > 19 || t->tp.snap_len > 9600 ||
 	    t->tp.min_len > M_TFMINPKTSIZE || t->tp.skip_len > M_TFLENGTH ||
 	    t->tp.skip_ofst > M_TFOFFSET) {
 		rc = EINVAL;
 		goto done;
 	}
 
 	memcpy(&tp.data[0], &t->tp.data[0], sizeof(tp.data));
 	memcpy(&tp.mask[0], &t->tp.mask[0], sizeof(tp.mask));
 	tp.snap_len = t->tp.snap_len;
 	tp.min_len = t->tp.min_len;
 	tp.skip_ofst = t->tp.skip_ofst;
 	tp.skip_len = t->tp.skip_len;
 	tp.invert = !!t->tp.invert;
 
 	/* convert port to channel iff 0 <= port < 8. */
 	if (t->tp.port < 4) {
 		if (sc->port[t->tp.port] == NULL) {
 			rc = EINVAL;
 			goto done;
 		}
 		tp.port = sc->port[t->tp.port]->tx_chan;
 	} else if (t->tp.port < 8) {
 		if (sc->port[t->tp.port - 4] == NULL) {
 			rc = EINVAL;
 			goto done;
 		}
 		tp.port = sc->port[t->tp.port - 4]->tx_chan + 4;
 	}
 	tpp = &tp;
 done:
 	if (rc == 0) {
 		rc = -t4_set_trace_filter(sc, tpp, t->idx, t->enabled);
 		if (rc == 0) {
 			if (t->enabled) {
 				setbit(&sc->tracer_valid, t->idx);
 				if (sc->tracer_enabled == 0) {
 					t4_set_reg_field(sc, A_MPS_TRC_CFG,
 					    F_TRCEN, F_TRCEN);
 				}
 				setbit(&sc->tracer_enabled, t->idx);
 			} else {
 				clrbit(&sc->tracer_enabled, t->idx);
 				if (sc->tracer_enabled == 0) {
 					t4_set_reg_field(sc, A_MPS_TRC_CFG,
 					    F_TRCEN, 0);
 				}
 			}
 		}
 	}
 	end_synchronized_op(sc, LOCK_HELD);
 
 	return (rc);
 }
 
 int
 t4_trace_pkt(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
 	struct ifnet *ifp;
 
 	KASSERT(m != NULL, ("%s: no payload with opcode %02x", __func__,
 	    rss->opcode));
 
 	mtx_lock(&sc->ifp_lock);
 	ifp = sc->ifp;
 	if (sc->ifp) {
 		m_adj(m, sizeof(struct cpl_trace_pkt));
 		m->m_pkthdr.rcvif = ifp;
 		ETHER_BPF_MTAP(ifp, m);
 	}
 	mtx_unlock(&sc->ifp_lock);
 	m_freem(m);
 
 	return (0);
 }
 
 int
 t5_trace_pkt(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
 	struct ifnet *ifp;
 
 	KASSERT(m != NULL, ("%s: no payload with opcode %02x", __func__,
 	    rss->opcode));
 
 	mtx_lock(&sc->ifp_lock);
 	ifp = sc->ifp;
 	if (ifp != NULL) {
 		m_adj(m, sizeof(struct cpl_t5_trace_pkt));
 		m->m_pkthdr.rcvif = ifp;
 		ETHER_BPF_MTAP(ifp, m);
 	}
 	mtx_unlock(&sc->ifp_lock);
 	m_freem(m);
 
 	return (0);
 }
 
 
 static void
 tracer_init(void *arg)
 {
 
 	return;
 }
 
 static int
 tracer_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
 {
 	int rc = 0;
 	struct adapter *sc;
 	struct ifreq *ifr = (struct ifreq *)data;
 
 	switch (cmd) {
 	case SIOCSIFMTU:
 	case SIOCSIFFLAGS:
 	case SIOCADDMULTI:
 	case SIOCDELMULTI:
 	case SIOCSIFCAP:
 		break;
 	case SIOCSIFMEDIA:
 	case SIOCGIFMEDIA:
 		sx_xlock(&t4_trace_lock);
 		sc = ifp->if_softc;
 		if (sc == NULL)
 			rc = EIO;
 		else
 			rc = ifmedia_ioctl(ifp, ifr, &sc->media, cmd);
 		sx_xunlock(&t4_trace_lock);
 		break;
 	default:
 		rc = ether_ioctl(ifp, cmd, data);
 	}
 
 	return (rc);
 }
 
 static int
 tracer_transmit(struct ifnet *ifp, struct mbuf *m)
 {
 
 	m_freem(m);
 	return (0);
 }
 
 static void
 tracer_qflush(struct ifnet *ifp)
 {
 
 	return;
 }
 
 static int
 tracer_media_change(struct ifnet *ifp)
 {
 
 	return (EOPNOTSUPP);
 }
 
 static void
 tracer_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 {
 
 	ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE;
 
 	return;
 }
Index: stable/10/sys/dev/cxgbe/tom/t4_cpl_io.c
===================================================================
--- stable/10/sys/dev/cxgbe/tom/t4_cpl_io.c	(revision 270296)
+++ stable/10/sys/dev/cxgbe/tom/t4_cpl_io.c	(revision 270297)
@@ -1,1489 +1,1804 @@
 /*-
  * Copyright (c) 2012 Chelsio Communications, Inc.
  * All rights reserved.
  * Written by: Navdeep Parhar <np@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
 
 #ifdef TCP_OFFLOAD
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/kernel.h>
 #include <sys/ktr.h>
 #include <sys/module.h>
 #include <sys/protosw.h>
 #include <sys/domain.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/sglist.h>
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
 #include <netinet/ip.h>
 #include <netinet/tcp_var.h>
 #define TCPSTATES
 #include <netinet/tcp_fsm.h>
 #include <netinet/tcp_seq.h>
 #include <netinet/toecore.h>
 
 #include "common/common.h"
 #include "common/t4_msg.h"
 #include "common/t4_regs.h"
 #include "common/t4_tcb.h"
 #include "tom/t4_tom_l2t.h"
 #include "tom/t4_tom.h"
 
 VNET_DECLARE(int, tcp_do_autosndbuf);
 #define V_tcp_do_autosndbuf VNET(tcp_do_autosndbuf)
 VNET_DECLARE(int, tcp_autosndbuf_inc);
 #define V_tcp_autosndbuf_inc VNET(tcp_autosndbuf_inc)
 VNET_DECLARE(int, tcp_autosndbuf_max);
 #define V_tcp_autosndbuf_max VNET(tcp_autosndbuf_max)
 VNET_DECLARE(int, tcp_do_autorcvbuf);
 #define V_tcp_do_autorcvbuf VNET(tcp_do_autorcvbuf)
 VNET_DECLARE(int, tcp_autorcvbuf_inc);
 #define V_tcp_autorcvbuf_inc VNET(tcp_autorcvbuf_inc)
 VNET_DECLARE(int, tcp_autorcvbuf_max);
 #define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max)
 
+/*
+ * For ULP connections HW may add headers, e.g., for digests, that aren't part
+ * of the messages sent by the host but that are part of the TCP payload and
+ * therefore consume TCP sequence space.  Tx connection parameters that
+ * operate in TCP sequence space are affected by the HW additions and need to
+ * compensate for them to accurately track TCP sequence numbers. This array
+ * contains the compensating extra lengths for ULP packets.  It is indexed by
+ * a packet's ULP submode.
+ */
+const unsigned int t4_ulp_extra_len[] = {0, 4, 4, 8};
+
+/*
+ * Return the length of any HW additions that will be made to a Tx packet.
+ * Such additions can happen for some types of ULP packets.
+ */
+static inline unsigned int
+ulp_extra_len(struct mbuf *m, int *ulp_mode)
+{
+	struct m_tag    *mtag;
+
+	if ((mtag = m_tag_find(m, CXGBE_ISCSI_MBUF_TAG, NULL)) == NULL)
+		return (0);
+	*ulp_mode = *((int *)(mtag + 1));
+
+	return (t4_ulp_extra_len[*ulp_mode & 3]);
+}
+
 void
 send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp)
 {
 	struct wrqe *wr;
 	struct fw_flowc_wr *flowc;
 	unsigned int nparams = ftxp ? 8 : 6, flowclen;
 	struct port_info *pi = toep->port;
 	struct adapter *sc = pi->adapter;
 	unsigned int pfvf = G_FW_VIID_PFN(pi->viid) << S_FW_VIID_PFN;
 	struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
 
 	KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT),
 	    ("%s: flowc for tid %u sent already", __func__, toep->tid));
 
 	CTR2(KTR_CXGBE, "%s: tid %u", __func__, toep->tid);
 
 	flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval);
 
 	wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq);
 	if (wr == NULL) {
 		/* XXX */
 		panic("%s: allocation failure.", __func__);
 	}
 	flowc = wrtod(wr);
 	memset(flowc, 0, wr->wr_len);
 
 	flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
 	    V_FW_FLOWC_WR_NPARAMS(nparams));
 	flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) |
 	    V_FW_WR_FLOWID(toep->tid));
 
 	flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
 	flowc->mnemval[0].val = htobe32(pfvf);
 	flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
 	flowc->mnemval[1].val = htobe32(pi->tx_chan);
 	flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
 	flowc->mnemval[2].val = htobe32(pi->tx_chan);
 	flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
 	flowc->mnemval[3].val = htobe32(toep->ofld_rxq->iq.abs_id);
 	if (ftxp) {
 		uint32_t sndbuf = min(ftxp->snd_space, sc->tt.sndbuf);
 
 		flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT;
 		flowc->mnemval[4].val = htobe32(ftxp->snd_nxt);
 		flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT;
 		flowc->mnemval[5].val = htobe32(ftxp->rcv_nxt);
 		flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF;
 		flowc->mnemval[6].val = htobe32(sndbuf);
 		flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS;
 		flowc->mnemval[7].val = htobe32(ftxp->mss);
 	} else {
 		flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDBUF;
 		flowc->mnemval[4].val = htobe32(512);
 		flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_MSS;
 		flowc->mnemval[5].val = htobe32(512);
 	}
 
 	txsd->tx_credits = howmany(flowclen, 16);
 	txsd->plen = 0;
 	KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,
 	    ("%s: not enough credits (%d)", __func__, toep->tx_credits));
 	toep->tx_credits -= txsd->tx_credits;
 	if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
 		toep->txsd_pidx = 0;
 	toep->txsd_avail--;
 
 	toep->flags |= TPF_FLOWC_WR_SENT;
         t4_wrq_tx(sc, wr);
 }
 
 void
 send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt)
 {
 	struct wrqe *wr;
 	struct cpl_abort_req *req;
 	int tid = toep->tid;
 	struct inpcb *inp = toep->inp;
 	struct tcpcb *tp = intotcpcb(inp);	/* don't use if INP_DROPPED */
 
 	INP_WLOCK_ASSERT(inp);
 
 	CTR6(KTR_CXGBE, "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x%s",
 	    __func__, toep->tid,
 	    inp->inp_flags & INP_DROPPED ? "inp dropped" :
 	    tcpstates[tp->t_state],
 	    toep->flags, inp->inp_flags,
 	    toep->flags & TPF_ABORT_SHUTDOWN ?
 	    " (abort already in progress)" : "");
 
 	if (toep->flags & TPF_ABORT_SHUTDOWN)
 		return;	/* abort already in progress */
 
 	toep->flags |= TPF_ABORT_SHUTDOWN;
 
 	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
 	    ("%s: flowc_wr not sent for tid %d.", __func__, tid));
 
 	wr = alloc_wrqe(sizeof(*req), toep->ofld_txq);
 	if (wr == NULL) {
 		/* XXX */
 		panic("%s: allocation failure.", __func__);
 	}
 	req = wrtod(wr);
 
 	INIT_TP_WR_MIT_CPL(req, CPL_ABORT_REQ, tid);
 	if (inp->inp_flags & INP_DROPPED)
 		req->rsvd0 = htobe32(snd_nxt);
 	else
 		req->rsvd0 = htobe32(tp->snd_nxt);
 	req->rsvd1 = !(toep->flags & TPF_TX_DATA_SENT);
 	req->cmd = CPL_ABORT_SEND_RST;
 
 	/*
 	 * XXX: What's the correct way to tell that the inp hasn't been detached
 	 * from its socket?  Should I even be flushing the snd buffer here?
 	 */
 	if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
 		struct socket *so = inp->inp_socket;
 
 		if (so != NULL)	/* because I'm not sure.  See comment above */
 			sbflush(&so->so_snd);
 	}
 
 	t4_l2t_send(sc, wr, toep->l2te);
 }
 
 /*
  * Called when a connection is established to translate the TCP options
  * reported by HW to FreeBSD's native format.
  */
 static void
 assign_rxopt(struct tcpcb *tp, unsigned int opt)
 {
 	struct toepcb *toep = tp->t_toe;
 	struct adapter *sc = td_adapter(toep->td);
 
 	INP_LOCK_ASSERT(tp->t_inpcb);
 
 	tp->t_maxseg = tp->t_maxopd = sc->params.mtus[G_TCPOPT_MSS(opt)] - 40;
 
 	if (G_TCPOPT_TSTAMP(opt)) {
 		tp->t_flags |= TF_RCVD_TSTMP;	/* timestamps ok */
 		tp->ts_recent = 0;		/* hmmm */
 		tp->ts_recent_age = tcp_ts_getticks();
 		tp->t_maxseg -= TCPOLEN_TSTAMP_APPA;
 	}
 
 	if (G_TCPOPT_SACK(opt))
 		tp->t_flags |= TF_SACK_PERMIT;	/* should already be set */
 	else
 		tp->t_flags &= ~TF_SACK_PERMIT;	/* sack disallowed by peer */
 
 	if (G_TCPOPT_WSCALE_OK(opt))
 		tp->t_flags |= TF_RCVD_SCALE;
 
 	/* Doing window scaling? */
 	if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
 	    (TF_RCVD_SCALE | TF_REQ_SCALE)) {
 		tp->rcv_scale = tp->request_r_scale;
 		tp->snd_scale = G_TCPOPT_SND_WSCALE(opt);
 	}
 }
 
 /*
  * Completes some final bits of initialization for just established connections
  * and changes their state to TCPS_ESTABLISHED.
  *
  * The ISNs are from after the exchange of SYNs.  i.e., the true ISN + 1.
  */
 void
 make_established(struct toepcb *toep, uint32_t snd_isn, uint32_t rcv_isn,
     uint16_t opt)
 {
 	struct inpcb *inp = toep->inp;
 	struct socket *so = inp->inp_socket;
 	struct tcpcb *tp = intotcpcb(inp);
 	long bufsize;
 	uint32_t iss = be32toh(snd_isn) - 1;	/* true ISS */
 	uint32_t irs = be32toh(rcv_isn) - 1;	/* true IRS */
 	uint16_t tcpopt = be16toh(opt);
 	struct flowc_tx_params ftxp;
 
 	INP_WLOCK_ASSERT(inp);
 	KASSERT(tp->t_state == TCPS_SYN_SENT ||
 	    tp->t_state == TCPS_SYN_RECEIVED,
 	    ("%s: TCP state %s", __func__, tcpstates[tp->t_state]));
 
 	CTR4(KTR_CXGBE, "%s: tid %d, toep %p, inp %p",
 	    __func__, toep->tid, toep, inp);
 
 	tp->t_state = TCPS_ESTABLISHED;
 	tp->t_starttime = ticks;
 	TCPSTAT_INC(tcps_connects);
 
 	tp->irs = irs;
 	tcp_rcvseqinit(tp);
 	tp->rcv_wnd = toep->rx_credits << 10;
 	tp->rcv_adv += tp->rcv_wnd;
 	tp->last_ack_sent = tp->rcv_nxt;
 
 	/*
 	 * If we were unable to send all rx credits via opt0, save the remainder
 	 * in rx_credits so that they can be handed over with the next credit
 	 * update.
 	 */
 	SOCKBUF_LOCK(&so->so_rcv);
 	bufsize = select_rcv_wnd(so);
 	SOCKBUF_UNLOCK(&so->so_rcv);
 	toep->rx_credits = bufsize - tp->rcv_wnd;
 
 	tp->iss = iss;
 	tcp_sendseqinit(tp);
 	tp->snd_una = iss + 1;
 	tp->snd_nxt = iss + 1;
 	tp->snd_max = iss + 1;
 
 	assign_rxopt(tp, tcpopt);
 
 	SOCKBUF_LOCK(&so->so_snd);
 	if (so->so_snd.sb_flags & SB_AUTOSIZE && V_tcp_do_autosndbuf)
 		bufsize = V_tcp_autosndbuf_max;
 	else
 		bufsize = sbspace(&so->so_snd);
 	SOCKBUF_UNLOCK(&so->so_snd);
 
 	ftxp.snd_nxt = tp->snd_nxt;
 	ftxp.rcv_nxt = tp->rcv_nxt;
 	ftxp.snd_space = bufsize;
 	ftxp.mss = tp->t_maxseg;
 	send_flowc_wr(toep, &ftxp);
 
 	soisconnected(so);
 }
 
 static int
 send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits)
 {
 	struct wrqe *wr;
 	struct cpl_rx_data_ack *req;
 	uint32_t dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
 
 	KASSERT(credits >= 0, ("%s: %d credits", __func__, credits));
 
 	wr = alloc_wrqe(sizeof(*req), toep->ctrlq);
 	if (wr == NULL)
 		return (0);
 	req = wrtod(wr);
 
 	INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid);
 	req->credit_dack = htobe32(dack | V_RX_CREDITS(credits));
 
 	t4_wrq_tx(sc, wr);
 	return (credits);
 }
 
 void
 t4_rcvd(struct toedev *tod, struct tcpcb *tp)
 {
 	struct adapter *sc = tod->tod_softc;
 	struct inpcb *inp = tp->t_inpcb;
 	struct socket *so = inp->inp_socket;
 	struct sockbuf *sb = &so->so_rcv;
 	struct toepcb *toep = tp->t_toe;
 	int credits;
 
 	INP_WLOCK_ASSERT(inp);
 
 	SOCKBUF_LOCK(sb);
 	KASSERT(toep->sb_cc >= sb->sb_cc,
 	    ("%s: sb %p has more data (%d) than last time (%d).",
 	    __func__, sb, sb->sb_cc, toep->sb_cc));
-	toep->rx_credits += toep->sb_cc - sb->sb_cc;
-	toep->sb_cc = sb->sb_cc;
+	if (toep->ulp_mode == ULP_MODE_ISCSI) {
+		toep->rx_credits += toep->sb_cc;
+		toep->sb_cc = 0;
+	} else {
+		toep->rx_credits += toep->sb_cc - sb->sb_cc;
+		toep->sb_cc = sb->sb_cc;
+	}
 	credits = toep->rx_credits;
 	SOCKBUF_UNLOCK(sb);
 
 	if (credits > 0 &&
 	    (credits + 16384 >= tp->rcv_wnd || credits >= 15 * 1024)) {
 
 		credits = send_rx_credits(sc, toep, credits);
 		SOCKBUF_LOCK(sb);
 		toep->rx_credits -= credits;
 		SOCKBUF_UNLOCK(sb);
 		tp->rcv_wnd += credits;
 		tp->rcv_adv += credits;
 	}
 }
 
 /*
  * Close a connection by sending a CPL_CLOSE_CON_REQ message.
  */
 static int
 close_conn(struct adapter *sc, struct toepcb *toep)
 {
 	struct wrqe *wr;
 	struct cpl_close_con_req *req;
 	unsigned int tid = toep->tid;
 
 	CTR3(KTR_CXGBE, "%s: tid %u%s", __func__, toep->tid,
 	    toep->flags & TPF_FIN_SENT ? ", IGNORED" : "");
 
 	if (toep->flags & TPF_FIN_SENT)
 		return (0);
 
 	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
 	    ("%s: flowc_wr not sent for tid %u.", __func__, tid));
 
 	wr = alloc_wrqe(sizeof(*req), toep->ofld_txq);
 	if (wr == NULL) {
 		/* XXX */
 		panic("%s: allocation failure.", __func__);
 	}
 	req = wrtod(wr);
 
         req->wr.wr_hi = htonl(V_FW_WR_OP(FW_TP_WR) |
 	    V_FW_WR_IMMDLEN(sizeof(*req) - sizeof(req->wr)));
 	req->wr.wr_mid = htonl(V_FW_WR_LEN16(howmany(sizeof(*req), 16)) |
 	    V_FW_WR_FLOWID(tid));
         req->wr.wr_lo = cpu_to_be64(0);
         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
 	req->rsvd = 0;
 
 	toep->flags |= TPF_FIN_SENT;
 	toep->flags &= ~TPF_SEND_FIN;
 	t4_l2t_send(sc, wr, toep->l2te);
 
 	return (0);
 }
 
 #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16)
 #define MIN_OFLD_TX_CREDITS (howmany(sizeof(struct fw_ofld_tx_data_wr) + 1, 16))
 
 /* Maximum amount of immediate data we could stuff in a WR */
 static inline int
 max_imm_payload(int tx_credits)
 {
 	const int n = 2;	/* Use only up to 2 desc for imm. data WR */
 
 	KASSERT(tx_credits >= 0 &&
 		tx_credits <= MAX_OFLD_TX_CREDITS,
 		("%s: %d credits", __func__, tx_credits));
 
 	if (tx_credits < MIN_OFLD_TX_CREDITS)
 		return (0);
 
 	if (tx_credits >= (n * EQ_ESIZE) / 16)
 		return ((n * EQ_ESIZE) - sizeof(struct fw_ofld_tx_data_wr));
 	else
 		return (tx_credits * 16 - sizeof(struct fw_ofld_tx_data_wr));
 }
 
 /* Maximum number of SGL entries we could stuff in a WR */
 static inline int
 max_dsgl_nsegs(int tx_credits)
 {
 	int nseg = 1;	/* ulptx_sgl has room for 1, rest ulp_tx_sge_pair */
 	int sge_pair_credits = tx_credits - MIN_OFLD_TX_CREDITS;
 
 	KASSERT(tx_credits >= 0 &&
 		tx_credits <= MAX_OFLD_TX_CREDITS,
 		("%s: %d credits", __func__, tx_credits));
 
 	if (tx_credits < MIN_OFLD_TX_CREDITS)
 		return (0);
 
 	nseg += 2 * (sge_pair_credits * 16 / 24);
 	if ((sge_pair_credits * 16) % 24 == 16)
 		nseg++;
 
 	return (nseg);
 }
 
 static inline void
 write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen,
-    unsigned int plen, uint8_t credits, int shove)
+    unsigned int plen, uint8_t credits, int shove, int ulp_mode)
 {
 	struct fw_ofld_tx_data_wr *txwr = dst;
+	unsigned int wr_ulp_mode;
 
 	txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) |
 	    V_FW_WR_IMMDLEN(immdlen));
 	txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
 	    V_FW_WR_LEN16(credits));
+
+	/* for iscsi, the mode & submode setting is per-packet */
+	if (toep->ulp_mode == ULP_MODE_ISCSI)
+		wr_ulp_mode = V_FW_OFLD_TX_DATA_WR_ULPMODE(ulp_mode >> 4) |
+			V_FW_OFLD_TX_DATA_WR_ULPSUBMODE(ulp_mode & 3);
+	else
+		wr_ulp_mode = V_FW_OFLD_TX_DATA_WR_ULPMODE(toep->ulp_mode);
+
 	txwr->lsodisable_to_proxy =
-	    htobe32(V_FW_OFLD_TX_DATA_WR_ULPMODE(toep->ulp_mode) |
+	    htobe32(wr_ulp_mode |
 		V_FW_OFLD_TX_DATA_WR_URGENT(0) |	/* XXX */
 		V_FW_OFLD_TX_DATA_WR_SHOVE(shove));
 	txwr->plen = htobe32(plen);
 }
 
 /*
  * Generate a DSGL from a starting mbuf.  The total number of segments and the
  * maximum segments in any one mbuf are provided.
  */
 static void
 write_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n)
 {
 	struct mbuf *m;
 	struct ulptx_sgl *usgl = dst;
 	int i, j, rc;
 	struct sglist sg;
 	struct sglist_seg segs[n];
 
 	KASSERT(nsegs > 0, ("%s: nsegs 0", __func__));
 
 	sglist_init(&sg, n, segs);
 	usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
 	    V_ULPTX_NSGE(nsegs));
 
 	i = -1;
 	for (m = start; m != stop; m = m->m_next) {
 		rc = sglist_append(&sg, mtod(m, void *), m->m_len);
 		if (__predict_false(rc != 0))
 			panic("%s: sglist_append %d", __func__, rc);
 
 		for (j = 0; j < sg.sg_nseg; i++, j++) {
 			if (i < 0) {
 				usgl->len0 = htobe32(segs[j].ss_len);
 				usgl->addr0 = htobe64(segs[j].ss_paddr);
 			} else {
 				usgl->sge[i / 2].len[i & 1] =
 				    htobe32(segs[j].ss_len);
 				usgl->sge[i / 2].addr[i & 1] =
 				    htobe64(segs[j].ss_paddr);
 			}
 #ifdef INVARIANTS
 			nsegs--;
 #endif
 		}
 		sglist_reset(&sg);
 	}
 	if (i & 1)
 		usgl->sge[i / 2].len[1] = htobe32(0);
 	KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, stop %p",
 	    __func__, nsegs, start, stop));
 }
 
 /*
  * Max number of SGL entries an offload tx work request can have.  This is 41
  * (1 + 40) for a full 512B work request.
  * fw_ofld_tx_data_wr(16B) + ulptx_sgl(16B, 1) + ulptx_sge_pair(480B, 40)
  */
 #define OFLD_SGL_LEN (41)
 
 /*
  * Send data and/or a FIN to the peer.
  *
  * The socket's so_snd buffer consists of a stream of data starting with sb_mb
  * and linked together with m_next.  sb_sndptr, if set, is the last mbuf that
  * was transmitted.
  *
  * drop indicates the number of bytes that should be dropped from the head of
  * the send buffer.  It is an optimization that lets do_fw4_ack avoid creating
  * contention on the send buffer lock (before this change it used to do
  * sowwakeup and then t4_push_frames right after that when recovering from tx
  * stalls).  When drop is set this function MUST drop the bytes and wake up any
  * writers.
  */
-static void
+void
 t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
 {
 	struct mbuf *sndptr, *m, *sb_sndptr;
 	struct fw_ofld_tx_data_wr *txwr;
 	struct wrqe *wr;
 	u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
 	struct inpcb *inp = toep->inp;
 	struct tcpcb *tp = intotcpcb(inp);
 	struct socket *so = inp->inp_socket;
 	struct sockbuf *sb = &so->so_snd;
 	int tx_credits, shove, compl, space, sowwakeup;
 	struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
 
 	INP_WLOCK_ASSERT(inp);
 	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
 	    ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
 
 	KASSERT(toep->ulp_mode == ULP_MODE_NONE ||
 	    toep->ulp_mode == ULP_MODE_TCPDDP ||
 	    toep->ulp_mode == ULP_MODE_RDMA,
 	    ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep));
 
 	/*
 	 * This function doesn't resume by itself.  Someone else must clear the
 	 * flag and call this function.
 	 */
 	if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
 		KASSERT(drop == 0,
 		    ("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
 		return;
 	}
 
 	do {
 		tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
 		max_imm = max_imm_payload(tx_credits);
 		max_nsegs = max_dsgl_nsegs(tx_credits);
 
 		SOCKBUF_LOCK(sb);
 		sowwakeup = drop;
 		if (drop) {
 			sbdrop_locked(sb, drop);
 			drop = 0;
 		}
 		sb_sndptr = sb->sb_sndptr;
 		sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb;
 		plen = 0;
 		nsegs = 0;
 		max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
 		for (m = sndptr; m != NULL; m = m->m_next) {
 			int n = sglist_count(mtod(m, void *), m->m_len);
 
 			nsegs += n;
 			plen += m->m_len;
 
 			/* This mbuf sent us _over_ the nsegs limit, back out */
 			if (plen > max_imm && nsegs > max_nsegs) {
 				nsegs -= n;
 				plen -= m->m_len;
 				if (plen == 0) {
 					/* Too few credits */
 					toep->flags |= TPF_TX_SUSPENDED;
 					if (sowwakeup)
 						sowwakeup_locked(so);
 					else
 						SOCKBUF_UNLOCK(sb);
 					SOCKBUF_UNLOCK_ASSERT(sb);
 					return;
 				}
 				break;
 			}
 
 			if (max_nsegs_1mbuf < n)
 				max_nsegs_1mbuf = n;
 			sb_sndptr = m;	/* new sb->sb_sndptr if all goes well */
 
 			/* This mbuf put us right at the max_nsegs limit */
 			if (plen > max_imm && nsegs == max_nsegs) {
 				m = m->m_next;
 				break;
 			}
 		}
 
 		shove = m == NULL && !(tp->t_flags & TF_MORETOCOME);
 		space = sbspace(sb);
 
 		if (space <= sb->sb_hiwat * 3 / 8 &&
 		    toep->plen_nocompl + plen >= sb->sb_hiwat / 4)
 			compl = 1;
 		else
 			compl = 0;
 
 		if (sb->sb_flags & SB_AUTOSIZE &&
 		    V_tcp_do_autosndbuf &&
 		    sb->sb_hiwat < V_tcp_autosndbuf_max &&
 		    space < sb->sb_hiwat / 8) {
 			int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc,
 			    V_tcp_autosndbuf_max);
 
 			if (!sbreserve_locked(sb, newsize, so, NULL))
 				sb->sb_flags &= ~SB_AUTOSIZE;
 			else
 				sowwakeup = 1;	/* room available */
 		}
 		if (sowwakeup)
 			sowwakeup_locked(so);
 		else
 			SOCKBUF_UNLOCK(sb);
 		SOCKBUF_UNLOCK_ASSERT(sb);
 
 		/* nothing to send */
 		if (plen == 0) {
 			KASSERT(m == NULL,
 			    ("%s: nothing to send, but m != NULL", __func__));
 			break;
 		}
 
 		if (__predict_false(toep->flags & TPF_FIN_SENT))
 			panic("%s: excess tx.", __func__);
 
 		if (plen <= max_imm) {
 
 			/* Immediate data tx */
 
 			wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16),
 					toep->ofld_txq);
 			if (wr == NULL) {
 				/* XXX: how will we recover from this? */
 				toep->flags |= TPF_TX_SUSPENDED;
 				return;
 			}
 			txwr = wrtod(wr);
 			credits = howmany(wr->wr_len, 16);
-			write_tx_wr(txwr, toep, plen, plen, credits, shove);
+			write_tx_wr(txwr, toep, plen, plen, credits, shove, 0);
 			m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
 			nsegs = 0;
 		} else {
 			int wr_len;
 
 			/* DSGL tx */
 
 			wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
 			    ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
 			wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);
 			if (wr == NULL) {
 				/* XXX: how will we recover from this? */
 				toep->flags |= TPF_TX_SUSPENDED;
 				return;
 			}
 			txwr = wrtod(wr);
 			credits = howmany(wr_len, 16);
-			write_tx_wr(txwr, toep, 0, plen, credits, shove);
+			write_tx_wr(txwr, toep, 0, plen, credits, shove, 0);
 			write_tx_sgl(txwr + 1, sndptr, m, nsegs,
 			    max_nsegs_1mbuf);
 			if (wr_len & 0xf) {
 				uint64_t *pad = (uint64_t *)
 				    ((uintptr_t)txwr + wr_len);
 				*pad = 0;
 			}
 		}
 
 		KASSERT(toep->tx_credits >= credits,
 			("%s: not enough credits", __func__));
 
 		toep->tx_credits -= credits;
 		toep->tx_nocompl += credits;
 		toep->plen_nocompl += plen;
 		if (toep->tx_credits <= toep->tx_total * 3 / 8 &&
 		    toep->tx_nocompl >= toep->tx_total / 4)
 			compl = 1;
 
 		if (compl) {
 			txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL);
 			toep->tx_nocompl = 0;
 			toep->plen_nocompl = 0;
 		}
 
 		tp->snd_nxt += plen;
 		tp->snd_max += plen;
 
 		SOCKBUF_LOCK(sb);
 		KASSERT(sb_sndptr, ("%s: sb_sndptr is NULL", __func__));
 		sb->sb_sndptr = sb_sndptr;
 		SOCKBUF_UNLOCK(sb);
 
 		toep->flags |= TPF_TX_DATA_SENT;
 		if (toep->tx_credits < MIN_OFLD_TX_CREDITS)
 			toep->flags |= TPF_TX_SUSPENDED;
 
 		KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
 		txsd->plen = plen;
 		txsd->tx_credits = credits;
 		txsd++;
 		if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
 			toep->txsd_pidx = 0;
 			txsd = &toep->txsd[0];
 		}
 		toep->txsd_avail--;
 
 		t4_l2t_send(sc, wr, toep->l2te);
 	} while (m != NULL);
 
 	/* Send a FIN if requested, but only if there's no more data to send */
 	if (m == NULL && toep->flags & TPF_SEND_FIN)
 		close_conn(sc, toep);
 }
 
+/* Send ULP data over TOE using TX_DATA_WR. We send whole mbuf at once */
+void
+t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
+{
+	struct mbuf *sndptr, *m = NULL;
+	struct fw_ofld_tx_data_wr *txwr;
+	struct wrqe *wr;
+	unsigned int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
+	struct inpcb *inp = toep->inp;
+	struct tcpcb *tp;
+	struct socket *so;
+	struct sockbuf *sb;
+	int tx_credits, ulp_len = 0, ulp_mode = 0, qlen = 0;
+	int shove, compl;
+	struct ofld_tx_sdesc *txsd;
+
+	INP_WLOCK_ASSERT(inp);
+	if (toep->flags & TPF_ABORT_SHUTDOWN)
+		return;
+
+	tp = intotcpcb(inp);
+	so = inp->inp_socket;
+	sb = &so->so_snd;
+	txsd = &toep->txsd[toep->txsd_pidx];
+
+	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
+	    ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
+
+	/*
+	 * This function doesn't resume by itself.  Someone else must clear the
+	 * flag and call this function.
+	 */
+	if (__predict_false(toep->flags & TPF_TX_SUSPENDED))
+		return;
+
+	sndptr = t4_queue_iscsi_callback(so, toep, 1, &qlen);
+	if (!qlen)
+		return;
+
+	do {
+		tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
+		max_imm = max_imm_payload(tx_credits);
+		max_nsegs = max_dsgl_nsegs(tx_credits);
+
+		if (drop) {
+			t4_cpl_iscsi_callback(toep->td, toep, &drop,
+			    CPL_FW4_ACK);
+			drop = 0;
+		}
+
+		plen = 0;
+		nsegs = 0;
+		max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
+		for (m = sndptr; m != NULL; m = m->m_next) {
+			int n = sglist_count(mtod(m, void *), m->m_len);
+
+			nsegs += n;
+			plen += m->m_len;
+
+			/* This mbuf sent us _over_ the nsegs limit, return */
+			if (plen > max_imm && nsegs > max_nsegs) {
+				toep->flags |= TPF_TX_SUSPENDED;
+				return;
+			}
+
+			if (max_nsegs_1mbuf < n)
+				max_nsegs_1mbuf = n;
+
+			/* This mbuf put us right at the max_nsegs limit */
+			if (plen > max_imm && nsegs == max_nsegs) {
+				toep->flags |= TPF_TX_SUSPENDED;
+				return;
+			}
+		}
+
+		shove = m == NULL && !(tp->t_flags & TF_MORETOCOME);
+		/* nothing to send */
+		if (plen == 0) {
+			KASSERT(m == NULL,
+			    ("%s: nothing to send, but m != NULL", __func__));
+			break;
+		}
+
+		if (__predict_false(toep->flags & TPF_FIN_SENT))
+			panic("%s: excess tx.", __func__);
+
+		ulp_len = plen + ulp_extra_len(sndptr, &ulp_mode);
+		if (plen <= max_imm) {
+
+			/* Immediate data tx */
+			wr = alloc_wrqe(roundup(sizeof(*txwr) + plen, 16),
+					toep->ofld_txq);
+			if (wr == NULL) {
+				/* XXX: how will we recover from this? */
+				toep->flags |= TPF_TX_SUSPENDED;
+				return;
+			}
+			txwr = wrtod(wr);
+			credits = howmany(wr->wr_len, 16);
+			write_tx_wr(txwr, toep, plen, ulp_len, credits, shove,
+								ulp_mode);
+			m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
+		} else {
+			int wr_len;
+
+			/* DSGL tx */
+			wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
+			    ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
+			wr = alloc_wrqe(roundup(wr_len, 16), toep->ofld_txq);
+			if (wr == NULL) {
+				/* XXX: how will we recover from this? */
+				toep->flags |= TPF_TX_SUSPENDED;
+				return;
+			}
+			txwr = wrtod(wr);
+			credits = howmany(wr_len, 16);
+			write_tx_wr(txwr, toep, 0, ulp_len, credits, shove,
+								ulp_mode);
+			write_tx_sgl(txwr + 1, sndptr, m, nsegs,
+			    max_nsegs_1mbuf);
+			if (wr_len & 0xf) {
+				uint64_t *pad = (uint64_t *)
+				    ((uintptr_t)txwr + wr_len);
+				*pad = 0;
+			}
+		}
+
+		KASSERT(toep->tx_credits >= credits,
+			("%s: not enough credits", __func__));
+
+		toep->tx_credits -= credits;
+		toep->tx_nocompl += credits;
+		toep->plen_nocompl += plen;
+		if (toep->tx_credits <= toep->tx_total * 3 / 8 &&
+			toep->tx_nocompl >= toep->tx_total / 4)
+			compl = 1;
+
+		if (compl) {
+			txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL);
+			toep->tx_nocompl = 0;
+			toep->plen_nocompl = 0;
+		}
+		tp->snd_nxt += ulp_len;
+		tp->snd_max += ulp_len;
+
+                /* goto next mbuf */
+		sndptr = m = t4_queue_iscsi_callback(so, toep, 2, &qlen);
+
+		toep->flags |= TPF_TX_DATA_SENT;
+		if (toep->tx_credits < MIN_OFLD_TX_CREDITS) {
+			toep->flags |= TPF_TX_SUSPENDED;
+		}
+
+		KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
+		txsd->plen = plen;
+		txsd->tx_credits = credits;
+		txsd++;
+		if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
+			toep->txsd_pidx = 0;
+			txsd = &toep->txsd[0];
+		}
+		toep->txsd_avail--;
+
+		t4_l2t_send(sc, wr, toep->l2te);
+	} while (m != NULL);
+
+	/* Send a FIN if requested, but only if there's no more data to send */
+	if (m == NULL && toep->flags & TPF_SEND_FIN)
+		close_conn(sc, toep);
+}
+
 int
 t4_tod_output(struct toedev *tod, struct tcpcb *tp)
 {
 	struct adapter *sc = tod->tod_softc;
 #ifdef INVARIANTS
 	struct inpcb *inp = tp->t_inpcb;
 #endif
 	struct toepcb *toep = tp->t_toe;
 
 	INP_WLOCK_ASSERT(inp);
 	KASSERT((inp->inp_flags & INP_DROPPED) == 0,
 	    ("%s: inp %p dropped.", __func__, inp));
 	KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
 
 	t4_push_frames(sc, toep, 0);
 
 	return (0);
 }
 
 int
 t4_send_fin(struct toedev *tod, struct tcpcb *tp)
 {
 	struct adapter *sc = tod->tod_softc;
 #ifdef INVARIANTS
 	struct inpcb *inp = tp->t_inpcb;
 #endif
 	struct toepcb *toep = tp->t_toe;
 
 	INP_WLOCK_ASSERT(inp);
 	KASSERT((inp->inp_flags & INP_DROPPED) == 0,
 	    ("%s: inp %p dropped.", __func__, inp));
 	KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
 
 	toep->flags |= TPF_SEND_FIN;
-	if (tp->t_state >= TCPS_ESTABLISHED)
-		t4_push_frames(sc, toep, 0);
+	if (tp->t_state >= TCPS_ESTABLISHED) {
+		if (toep->ulp_mode == ULP_MODE_ISCSI)
+			t4_ulp_push_frames(sc, toep, 0);
+		else
+			t4_push_frames(sc, toep, 0);
+	}
 
 	return (0);
 }
 
 int
 t4_send_rst(struct toedev *tod, struct tcpcb *tp)
 {
 	struct adapter *sc = tod->tod_softc;
 #if defined(INVARIANTS)
 	struct inpcb *inp = tp->t_inpcb;
 #endif
 	struct toepcb *toep = tp->t_toe;
 
 	INP_WLOCK_ASSERT(inp);
 	KASSERT((inp->inp_flags & INP_DROPPED) == 0,
 	    ("%s: inp %p dropped.", __func__, inp));
 	KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
 
 	/* hmmmm */
 	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
 	    ("%s: flowc for tid %u [%s] not sent already",
 	    __func__, toep->tid, tcpstates[tp->t_state]));
 
 	send_reset(sc, toep, 0);
 	return (0);
 }
 
 /*
  * Peer has sent us a FIN.
  */
 static int
 do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
 	const struct cpl_peer_close *cpl = (const void *)(rss + 1);
 	unsigned int tid = GET_TID(cpl);
 	struct toepcb *toep = lookup_tid(sc, tid);
 	struct inpcb *inp = toep->inp;
 	struct tcpcb *tp = NULL;
 	struct socket *so;
 	struct sockbuf *sb;
 #ifdef INVARIANTS
 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
 #endif
 
 	KASSERT(opcode == CPL_PEER_CLOSE,
 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
 
 	if (__predict_false(toep->flags & TPF_SYNQE)) {
 #ifdef INVARIANTS
 		struct synq_entry *synqe = (void *)toep;
 
 		INP_WLOCK(synqe->lctx->inp);
 		if (synqe->flags & TPF_SYNQE_HAS_L2TE) {
 			KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN,
 			    ("%s: listen socket closed but tid %u not aborted.",
 			    __func__, tid));
 		} else {
 			/*
 			 * do_pass_accept_req is still running and will
 			 * eventually take care of this tid.
 			 */
 		}
 		INP_WUNLOCK(synqe->lctx->inp);
 #endif
 		CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid,
 		    toep, toep->flags);
 		return (0);
 	}
 
 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
 
 	INP_INFO_WLOCK(&V_tcbinfo);
 	INP_WLOCK(inp);
 	tp = intotcpcb(inp);
 
 	CTR5(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x, inp %p", __func__,
 	    tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, inp);
 
 	if (toep->flags & TPF_ABORT_SHUTDOWN)
 		goto done;
 
 	tp->rcv_nxt++;	/* FIN */
 
 	so = inp->inp_socket;
 	sb = &so->so_rcv;
 	SOCKBUF_LOCK(sb);
 	if (__predict_false(toep->ddp_flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE))) {
 		m = get_ddp_mbuf(be32toh(cpl->rcv_nxt) - tp->rcv_nxt);
 		tp->rcv_nxt = be32toh(cpl->rcv_nxt);
 		toep->ddp_flags &= ~(DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE);
 
 		KASSERT(toep->sb_cc >= sb->sb_cc,
 		    ("%s: sb %p has more data (%d) than last time (%d).",
 		    __func__, sb, sb->sb_cc, toep->sb_cc));
 		toep->rx_credits += toep->sb_cc - sb->sb_cc;
 #ifdef USE_DDP_RX_FLOW_CONTROL
 		toep->rx_credits -= m->m_len;	/* adjust for F_RX_FC_DDP */
 #endif
 		sbappendstream_locked(sb, m);
 		toep->sb_cc = sb->sb_cc;
 	}
 	socantrcvmore_locked(so);	/* unlocks the sockbuf */
 
 	if (toep->ulp_mode != ULP_MODE_RDMA) {
 		KASSERT(tp->rcv_nxt == be32toh(cpl->rcv_nxt),
 	    		("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt,
 	    		be32toh(cpl->rcv_nxt)));
 	}
 
 	switch (tp->t_state) {
 	case TCPS_SYN_RECEIVED:
 		tp->t_starttime = ticks;
 		/* FALLTHROUGH */ 
 
 	case TCPS_ESTABLISHED:
 		tp->t_state = TCPS_CLOSE_WAIT;
 		break;
 
 	case TCPS_FIN_WAIT_1:
 		tp->t_state = TCPS_CLOSING;
 		break;
 
 	case TCPS_FIN_WAIT_2:
 		tcp_twstart(tp);
 		INP_UNLOCK_ASSERT(inp);	 /* safe, we have a ref on the inp */
 		INP_INFO_WUNLOCK(&V_tcbinfo);
 
 		INP_WLOCK(inp);
 		final_cpl_received(toep);
 		return (0);
 
 	default:
 		log(LOG_ERR, "%s: TID %u received CPL_PEER_CLOSE in state %d\n",
 		    __func__, tid, tp->t_state);
 	}
 done:
 	INP_WUNLOCK(inp);
 	INP_INFO_WUNLOCK(&V_tcbinfo);
 	return (0);
 }
 
 /*
  * Peer has ACK'd our FIN.
  */
 static int
 do_close_con_rpl(struct sge_iq *iq, const struct rss_header *rss,
     struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
 	const struct cpl_close_con_rpl *cpl = (const void *)(rss + 1);
 	unsigned int tid = GET_TID(cpl);
 	struct toepcb *toep = lookup_tid(sc, tid);
 	struct inpcb *inp = toep->inp;
 	struct tcpcb *tp = NULL;
 	struct socket *so = NULL;
 #ifdef INVARIANTS
 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
 #endif
 
 	KASSERT(opcode == CPL_CLOSE_CON_RPL,
 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
 
 	INP_INFO_WLOCK(&V_tcbinfo);
 	INP_WLOCK(inp);
 	tp = intotcpcb(inp);
 
 	CTR4(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x",
 	    __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags);
 
 	if (toep->flags & TPF_ABORT_SHUTDOWN)
 		goto done;
 
 	so = inp->inp_socket;
 	tp->snd_una = be32toh(cpl->snd_nxt) - 1;	/* exclude FIN */
 
 	switch (tp->t_state) {
 	case TCPS_CLOSING:	/* see TCPS_FIN_WAIT_2 in do_peer_close too */
 		tcp_twstart(tp);
 release:
 		INP_UNLOCK_ASSERT(inp);	/* safe, we have a ref on the  inp */
 		INP_INFO_WUNLOCK(&V_tcbinfo);
 
 		INP_WLOCK(inp);
 		final_cpl_received(toep);	/* no more CPLs expected */
 
 		return (0);
 	case TCPS_LAST_ACK:
 		if (tcp_close(tp))
 			INP_WUNLOCK(inp);
 		goto release;
 
 	case TCPS_FIN_WAIT_1:
 		if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
 			soisdisconnected(so);
 		tp->t_state = TCPS_FIN_WAIT_2;
 		break;
 
 	default:
 		log(LOG_ERR,
 		    "%s: TID %u received CPL_CLOSE_CON_RPL in state %s\n",
 		    __func__, tid, tcpstates[tp->t_state]);
 	}
 done:
 	INP_WUNLOCK(inp);
 	INP_INFO_WUNLOCK(&V_tcbinfo);
 	return (0);
 }
 
 void
 send_abort_rpl(struct adapter *sc, struct sge_wrq *ofld_txq, int tid,
     int rst_status)
 {
 	struct wrqe *wr;
 	struct cpl_abort_rpl *cpl;
 
 	wr = alloc_wrqe(sizeof(*cpl), ofld_txq);
 	if (wr == NULL) {
 		/* XXX */
 		panic("%s: allocation failure.", __func__);
 	}
 	cpl = wrtod(wr);
 
 	INIT_TP_WR_MIT_CPL(cpl, CPL_ABORT_RPL, tid);
 	cpl->cmd = rst_status;
 
 	t4_wrq_tx(sc, wr);
 }
 
 static int
 abort_status_to_errno(struct tcpcb *tp, unsigned int abort_reason)
 {
 	switch (abort_reason) {
 	case CPL_ERR_BAD_SYN:
 	case CPL_ERR_CONN_RESET:
 		return (tp->t_state == TCPS_CLOSE_WAIT ? EPIPE : ECONNRESET);
 	case CPL_ERR_XMIT_TIMEDOUT:
 	case CPL_ERR_PERSIST_TIMEDOUT:
 	case CPL_ERR_FINWAIT2_TIMEDOUT:
 	case CPL_ERR_KEEPALIVE_TIMEDOUT:
 		return (ETIMEDOUT);
 	default:
 		return (EIO);
 	}
 }
 
+int
+cpl_not_handled(struct sge_iq *, const struct rss_header *, struct mbuf *);
 /*
+ * tom_cpl_iscsi_callback -
+ * iscsi and tom would share the following cpl messages, so when any of these
+ * message is received, after tom is done with processing it, the messages
+ * needs to be forwarded to iscsi for further processing:
+ * - CPL_SET_TCB_RPL
+ * - CPL_RX_DATA_DDP
+ */
+void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *, void *,
+    unsigned int);
+
+struct mbuf *(*tom_queue_iscsi_callback)(struct socket *, unsigned int, int *);
+/*
+ * Check if the handler function is set for a given CPL
+ * return 0 if the function is NULL or cpl_not_handled, 1 otherwise.
+ */
+int
+t4tom_cpl_handler_registered(struct adapter *sc, unsigned int opcode)
+{
+
+	MPASS(opcode < nitems(sc->cpl_handler));
+
+	return (sc->cpl_handler[opcode] &&
+	    sc->cpl_handler[opcode] != cpl_not_handled);
+}
+
+/*
+ * set the tom_cpl_iscsi_callback function, this function should be used
+ * whenever both toe and iscsi need to process the same cpl msg.
+ */
+void
+t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *, struct socket *,
+    void *, unsigned int))
+{
+
+	tom_cpl_iscsi_callback = fp;
+}
+
+void
+t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *,
+    unsigned int, int *qlen))
+{
+
+	tom_queue_iscsi_callback = fp;
+}
+
+int
+t4_cpl_iscsi_callback(struct tom_data *td, struct toepcb *toep, void *m,
+    unsigned int opcode)
+{
+	struct socket *so;
+
+	if (opcode == CPL_FW4_ACK)
+		so = toep->inp->inp_socket;
+	else {
+		INP_WLOCK(toep->inp);
+		so = toep->inp->inp_socket;
+		INP_WUNLOCK(toep->inp);
+	}
+
+	if (tom_cpl_iscsi_callback && so) {
+		if (toep->ulp_mode == ULP_MODE_ISCSI) {
+			tom_cpl_iscsi_callback(td, so, m, opcode);
+			return (0);
+		}
+	}
+
+	return (1);
+}
+
+struct mbuf *
+t4_queue_iscsi_callback(struct socket *so, struct toepcb *toep,
+    unsigned int cmd, int *qlen)
+{
+
+	if (tom_queue_iscsi_callback && so) {
+		if (toep->ulp_mode == ULP_MODE_ISCSI)
+			return (tom_queue_iscsi_callback(so, cmd, qlen));
+	}
+
+	return (NULL);
+}
+
+/*
  * TCP RST from the peer, timeout, or some other such critical error.
  */
 static int
 do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
 	const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1);
 	unsigned int tid = GET_TID(cpl);
 	struct toepcb *toep = lookup_tid(sc, tid);
 	struct sge_wrq *ofld_txq = toep->ofld_txq;
 	struct inpcb *inp;
 	struct tcpcb *tp;
 #ifdef INVARIANTS
 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
 #endif
 
 	KASSERT(opcode == CPL_ABORT_REQ_RSS,
 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
 
 	if (toep->flags & TPF_SYNQE)
 		return (do_abort_req_synqe(iq, rss, m));
 
 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
 
 	if (negative_advice(cpl->status)) {
 		CTR4(KTR_CXGBE, "%s: negative advice %d for tid %d (0x%x)",
 		    __func__, cpl->status, tid, toep->flags);
 		return (0);	/* Ignore negative advice */
 	}
 
 	inp = toep->inp;
 	INP_INFO_WLOCK(&V_tcbinfo);	/* for tcp_close */
 	INP_WLOCK(inp);
 
 	tp = intotcpcb(inp);
 
 	CTR6(KTR_CXGBE,
 	    "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x, status %d",
 	    __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags,
 	    inp->inp_flags, cpl->status);
 
 	/*
 	 * If we'd initiated an abort earlier the reply to it is responsible for
 	 * cleaning up resources.  Otherwise we tear everything down right here
 	 * right now.  We owe the T4 a CPL_ABORT_RPL no matter what.
 	 */
 	if (toep->flags & TPF_ABORT_SHUTDOWN) {
 		INP_WUNLOCK(inp);
 		goto done;
 	}
 	toep->flags |= TPF_ABORT_SHUTDOWN;
 
 	if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
 		struct socket *so = inp->inp_socket;
 
 		if (so != NULL)
 			so_error_set(so, abort_status_to_errno(tp,
 			    cpl->status));
 		tp = tcp_close(tp);
 		if (tp == NULL)
 			INP_WLOCK(inp);	/* re-acquire */
 	}
 
 	final_cpl_received(toep);
 done:
 	INP_INFO_WUNLOCK(&V_tcbinfo);
 	send_abort_rpl(sc, ofld_txq, tid, CPL_ABORT_NO_RST);
 	return (0);
 }
 
 /*
  * Reply to the CPL_ABORT_REQ (send_reset)
  */
 static int
 do_abort_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
 	const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1);
 	unsigned int tid = GET_TID(cpl);
 	struct toepcb *toep = lookup_tid(sc, tid);
 	struct inpcb *inp = toep->inp;
 #ifdef INVARIANTS
 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
 #endif
 
 	KASSERT(opcode == CPL_ABORT_RPL_RSS,
 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
 
 	if (toep->flags & TPF_SYNQE)
 		return (do_abort_rpl_synqe(iq, rss, m));
 
 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
 
 	CTR5(KTR_CXGBE, "%s: tid %u, toep %p, inp %p, status %d",
 	    __func__, tid, toep, inp, cpl->status);
 
 	KASSERT(toep->flags & TPF_ABORT_SHUTDOWN,
 	    ("%s: wasn't expecting abort reply", __func__));
 
 	INP_WLOCK(inp);
 	final_cpl_received(toep);
 
 	return (0);
 }
 
 static int
 do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
 	const struct cpl_rx_data *cpl = mtod(m, const void *);
 	unsigned int tid = GET_TID(cpl);
 	struct toepcb *toep = lookup_tid(sc, tid);
 	struct inpcb *inp = toep->inp;
 	struct tcpcb *tp;
 	struct socket *so;
 	struct sockbuf *sb;
 	int len;
 	uint32_t ddp_placed = 0;
 
 	if (__predict_false(toep->flags & TPF_SYNQE)) {
 #ifdef INVARIANTS
 		struct synq_entry *synqe = (void *)toep;
 
 		INP_WLOCK(synqe->lctx->inp);
 		if (synqe->flags & TPF_SYNQE_HAS_L2TE) {
 			KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN,
 			    ("%s: listen socket closed but tid %u not aborted.",
 			    __func__, tid));
 		} else {
 			/*
 			 * do_pass_accept_req is still running and will
 			 * eventually take care of this tid.
 			 */
 		}
 		INP_WUNLOCK(synqe->lctx->inp);
 #endif
 		CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid,
 		    toep, toep->flags);
 		m_freem(m);
 		return (0);
 	}
 
 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
 
 	/* strip off CPL header */
 	m_adj(m, sizeof(*cpl));
 	len = m->m_pkthdr.len;
 
 	INP_WLOCK(inp);
 	if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
 		CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
 		    __func__, tid, len, inp->inp_flags);
 		INP_WUNLOCK(inp);
 		m_freem(m);
 		return (0);
 	}
 
 	tp = intotcpcb(inp);
 
 	if (__predict_false(tp->rcv_nxt != be32toh(cpl->seq)))
 		ddp_placed = be32toh(cpl->seq) - tp->rcv_nxt;
 
 	tp->rcv_nxt += len;
 	KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__));
 	tp->rcv_wnd -= len;
 	tp->t_rcvtime = ticks;
 
 	so = inp_inpcbtosocket(inp);
 	sb = &so->so_rcv;
 	SOCKBUF_LOCK(sb);
 
 	if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) {
 		CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)",
 		    __func__, tid, len);
 		m_freem(m);
 		SOCKBUF_UNLOCK(sb);
 		INP_WUNLOCK(inp);
 
 		INP_INFO_WLOCK(&V_tcbinfo);
 		INP_WLOCK(inp);
 		tp = tcp_drop(tp, ECONNRESET);
 		if (tp)
 			INP_WUNLOCK(inp);
 		INP_INFO_WUNLOCK(&V_tcbinfo);
 
 		return (0);
 	}
 
 	/* receive buffer autosize */
 	if (sb->sb_flags & SB_AUTOSIZE &&
 	    V_tcp_do_autorcvbuf &&
 	    sb->sb_hiwat < V_tcp_autorcvbuf_max &&
 	    len > (sbspace(sb) / 8 * 7)) {
 		unsigned int hiwat = sb->sb_hiwat;
 		unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc,
 		    V_tcp_autorcvbuf_max);
 
 		if (!sbreserve_locked(sb, newsize, so, NULL))
 			sb->sb_flags &= ~SB_AUTOSIZE;
 		else
 			toep->rx_credits += newsize - hiwat;
 	}
 
 	if (toep->ulp_mode == ULP_MODE_TCPDDP) {
 		int changed = !(toep->ddp_flags & DDP_ON) ^ cpl->ddp_off;
 
 		if (changed) {
 			if (toep->ddp_flags & DDP_SC_REQ)
 				toep->ddp_flags ^= DDP_ON | DDP_SC_REQ;
 			else {
 				KASSERT(cpl->ddp_off == 1,
 				    ("%s: DDP switched on by itself.",
 				    __func__));
 
 				/* Fell out of DDP mode */
 				toep->ddp_flags &= ~(DDP_ON | DDP_BUF0_ACTIVE |
 				    DDP_BUF1_ACTIVE);
 
 				if (ddp_placed)
 					insert_ddp_data(toep, ddp_placed);
 			}
 		}
 
 		if ((toep->ddp_flags & DDP_OK) == 0 &&
 		    time_uptime >= toep->ddp_disabled + DDP_RETRY_WAIT) {
 			toep->ddp_score = DDP_LOW_SCORE;
 			toep->ddp_flags |= DDP_OK;
 			CTR3(KTR_CXGBE, "%s: tid %u DDP_OK @ %u",
 			    __func__, tid, time_uptime);
 		}
 
 		if (toep->ddp_flags & DDP_ON) {
 
 			/*
 			 * CPL_RX_DATA with DDP on can only be an indicate.  Ask
 			 * soreceive to post a buffer or disable DDP.  The
 			 * payload that arrived in this indicate is appended to
 			 * the socket buffer as usual.
 			 */
 
 #if 0
 			CTR5(KTR_CXGBE,
 			    "%s: tid %u (0x%x) DDP indicate (seq 0x%x, len %d)",
 			    __func__, tid, toep->flags, be32toh(cpl->seq), len);
 #endif
 			sb->sb_flags |= SB_DDP_INDICATE;
 		} else if ((toep->ddp_flags & (DDP_OK|DDP_SC_REQ)) == DDP_OK &&
 		    tp->rcv_wnd > DDP_RSVD_WIN && len >= sc->tt.ddp_thres) {
 
 			/*
 			 * DDP allowed but isn't on (and a request to switch it
 			 * on isn't pending either), and conditions are ripe for
 			 * it to work.  Switch it on.
 			 */
 
 			enable_ddp(sc, toep);
 		}
 	}
 
 	KASSERT(toep->sb_cc >= sb->sb_cc,
 	    ("%s: sb %p has more data (%d) than last time (%d).",
 	    __func__, sb, sb->sb_cc, toep->sb_cc));
 	toep->rx_credits += toep->sb_cc - sb->sb_cc;
 	sbappendstream_locked(sb, m);
 	toep->sb_cc = sb->sb_cc;
 	sorwakeup_locked(so);
 	SOCKBUF_UNLOCK_ASSERT(sb);
 
 	INP_WUNLOCK(inp);
 	return (0);
 }
 
 #define S_CPL_FW4_ACK_OPCODE    24
 #define M_CPL_FW4_ACK_OPCODE    0xff
 #define V_CPL_FW4_ACK_OPCODE(x) ((x) << S_CPL_FW4_ACK_OPCODE)
 #define G_CPL_FW4_ACK_OPCODE(x) \
     (((x) >> S_CPL_FW4_ACK_OPCODE) & M_CPL_FW4_ACK_OPCODE)
 
 #define S_CPL_FW4_ACK_FLOWID    0
 #define M_CPL_FW4_ACK_FLOWID    0xffffff
 #define V_CPL_FW4_ACK_FLOWID(x) ((x) << S_CPL_FW4_ACK_FLOWID)
 #define G_CPL_FW4_ACK_FLOWID(x) \
     (((x) >> S_CPL_FW4_ACK_FLOWID) & M_CPL_FW4_ACK_FLOWID)
 
 #define S_CPL_FW4_ACK_CR        24
 #define M_CPL_FW4_ACK_CR        0xff
 #define V_CPL_FW4_ACK_CR(x)     ((x) << S_CPL_FW4_ACK_CR)
 #define G_CPL_FW4_ACK_CR(x)     (((x) >> S_CPL_FW4_ACK_CR) & M_CPL_FW4_ACK_CR)
 
 #define S_CPL_FW4_ACK_SEQVAL    0
 #define M_CPL_FW4_ACK_SEQVAL    0x1
 #define V_CPL_FW4_ACK_SEQVAL(x) ((x) << S_CPL_FW4_ACK_SEQVAL)
 #define G_CPL_FW4_ACK_SEQVAL(x) \
     (((x) >> S_CPL_FW4_ACK_SEQVAL) & M_CPL_FW4_ACK_SEQVAL)
 #define F_CPL_FW4_ACK_SEQVAL    V_CPL_FW4_ACK_SEQVAL(1U)
 
 static int
 do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
 	const struct cpl_fw4_ack *cpl = (const void *)(rss + 1);
 	unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl)));
 	struct toepcb *toep = lookup_tid(sc, tid);
 	struct inpcb *inp;
 	struct tcpcb *tp;
 	struct socket *so;
 	uint8_t credits = cpl->credits;
 	struct ofld_tx_sdesc *txsd;
 	int plen;
 #ifdef INVARIANTS
 	unsigned int opcode = G_CPL_FW4_ACK_OPCODE(be32toh(OPCODE_TID(cpl)));
 #endif
 
 	/*
 	 * Very unusual case: we'd sent a flowc + abort_req for a synq entry and
 	 * now this comes back carrying the credits for the flowc.
 	 */
 	if (__predict_false(toep->flags & TPF_SYNQE)) {
 		KASSERT(toep->flags & TPF_ABORT_SHUTDOWN,
 		    ("%s: credits for a synq entry %p", __func__, toep));
 		return (0);
 	}
 
 	inp = toep->inp;
 
 	KASSERT(opcode == CPL_FW4_ACK,
 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
 
 	INP_WLOCK(inp);
 
 	if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) {
 		INP_WUNLOCK(inp);
 		return (0);
 	}
 
 	KASSERT((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) == 0,
 	    ("%s: inp_flags 0x%x", __func__, inp->inp_flags));
 
 	tp = intotcpcb(inp);
 
 	if (cpl->flags & CPL_FW4_ACK_FLAGS_SEQVAL) {
 		tcp_seq snd_una = be32toh(cpl->snd_una);
 
 #ifdef INVARIANTS
 		if (__predict_false(SEQ_LT(snd_una, tp->snd_una))) {
 			log(LOG_ERR,
 			    "%s: unexpected seq# %x for TID %u, snd_una %x\n",
 			    __func__, snd_una, toep->tid, tp->snd_una);
 		}
 #endif
 
 		if (tp->snd_una != snd_una) {
 			tp->snd_una = snd_una;
 			tp->ts_recent_age = tcp_ts_getticks();
 		}
 	}
 
 	so = inp->inp_socket;
 	txsd = &toep->txsd[toep->txsd_cidx];
 	plen = 0;
 	while (credits) {
 		KASSERT(credits >= txsd->tx_credits,
 		    ("%s: too many (or partial) credits", __func__));
 		credits -= txsd->tx_credits;
 		toep->tx_credits += txsd->tx_credits;
 		plen += txsd->plen;
 		txsd++;
 		toep->txsd_avail++;
 		KASSERT(toep->txsd_avail <= toep->txsd_total,
 		    ("%s: txsd avail > total", __func__));
 		if (__predict_false(++toep->txsd_cidx == toep->txsd_total)) {
 			txsd = &toep->txsd[0];
 			toep->txsd_cidx = 0;
 		}
 	}
 
 	if (toep->tx_credits == toep->tx_total) {
 		toep->tx_nocompl = 0;
 		toep->plen_nocompl = 0;
 	}
 
 	if (toep->flags & TPF_TX_SUSPENDED &&
 	    toep->tx_credits >= toep->tx_total / 4) {
 		toep->flags &= ~TPF_TX_SUSPENDED;
-		t4_push_frames(sc, toep, plen);
+		if (toep->ulp_mode == ULP_MODE_ISCSI)
+			t4_ulp_push_frames(sc, toep, plen);
+		else
+			t4_push_frames(sc, toep, plen);
 	} else if (plen > 0) {
 		struct sockbuf *sb = &so->so_snd;
 
-		SOCKBUF_LOCK(sb);
-		sbdrop_locked(sb, plen);
-		sowwakeup_locked(so);
-		SOCKBUF_UNLOCK_ASSERT(sb);
+		if (toep->ulp_mode == ULP_MODE_ISCSI)
+			t4_cpl_iscsi_callback(toep->td, toep, &plen,
+			    CPL_FW4_ACK);
+		else {
+			SOCKBUF_LOCK(sb);
+			sbdrop_locked(sb, plen);
+			sowwakeup_locked(so);
+			SOCKBUF_UNLOCK_ASSERT(sb);
+		}
 	}
 
 	INP_WUNLOCK(inp);
 
 	return (0);
 }
 
 static int
 do_set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
 	const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1);
 	unsigned int tid = GET_TID(cpl);
 #ifdef INVARIANTS
 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
 #endif
 
 	KASSERT(opcode == CPL_SET_TCB_RPL,
 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
 
 	if (is_ftid(sc, tid))
 		return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */
+	else {
+		struct toepcb *toep = lookup_tid(sc, tid);
+
+		t4_cpl_iscsi_callback(toep->td, toep, m, CPL_SET_TCB_RPL);
+		return (0);
+	}
 
 	CXGBE_UNIMPLEMENTED(__func__);
 }
 
 void
 t4_set_tcb_field(struct adapter *sc, struct toepcb *toep, int ctrl,
     uint16_t word, uint64_t mask, uint64_t val)
 {
 	struct wrqe *wr;
 	struct cpl_set_tcb_field *req;
 
 	wr = alloc_wrqe(sizeof(*req), ctrl ? toep->ctrlq : toep->ofld_txq);
 	if (wr == NULL) {
 		/* XXX */
 		panic("%s: allocation failure.", __func__);
 	}
 	req = wrtod(wr);
 
 	INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, toep->tid);
 	req->reply_ctrl = htobe16(V_NO_REPLY(1) |
 	    V_QUEUENO(toep->ofld_rxq->iq.abs_id));
 	req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(0));
 	req->mask = htobe64(mask);
 	req->val = htobe64(val);
 
 	t4_wrq_tx(sc, wr);
 }
 
 void
 t4_init_cpl_io_handlers(struct adapter *sc)
 {
 
 	t4_register_cpl_handler(sc, CPL_PEER_CLOSE, do_peer_close);
 	t4_register_cpl_handler(sc, CPL_CLOSE_CON_RPL, do_close_con_rpl);
 	t4_register_cpl_handler(sc, CPL_ABORT_REQ_RSS, do_abort_req);
 	t4_register_cpl_handler(sc, CPL_ABORT_RPL_RSS, do_abort_rpl);
 	t4_register_cpl_handler(sc, CPL_RX_DATA, do_rx_data);
 	t4_register_cpl_handler(sc, CPL_FW4_ACK, do_fw4_ack);
 	t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, do_set_tcb_rpl);
 }
 
 void
 t4_uninit_cpl_io_handlers(struct adapter *sc)
 {
 
 	t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 }
 #endif
Index: stable/10/sys/dev/cxgbe/tom/t4_ddp.c
===================================================================
--- stable/10/sys/dev/cxgbe/tom/t4_ddp.c	(revision 270296)
+++ stable/10/sys/dev/cxgbe/tom/t4_ddp.c	(revision 270297)
@@ -1,1285 +1,1296 @@
 /*-
  * Copyright (c) 2012 Chelsio Communications, Inc.
  * All rights reserved.
  * Written by: Navdeep Parhar <np@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
 
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/ktr.h>
 #include <sys/module.h>
 #include <sys/protosw.h>
 #include <sys/proc.h>
 #include <sys/domain.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/uio.h>
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
 #include <netinet/ip.h>
 #include <netinet/tcp_var.h>
 #define TCPSTATES
 #include <netinet/tcp_fsm.h>
 #include <netinet/toecore.h>
 
 #include <vm/vm.h>
 #include <vm/vm_extern.h>
 #include <vm/vm_param.h>
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
 #include <vm/vm_page.h>
 #include <vm/vm_object.h>
 
 #ifdef TCP_OFFLOAD
 #include "common/common.h"
 #include "common/t4_msg.h"
 #include "common/t4_regs.h"
 #include "common/t4_tcb.h"
 #include "tom/t4_tom.h"
 
 #define PPOD_SZ(n)	((n) * sizeof(struct pagepod))
 #define PPOD_SIZE	(PPOD_SZ(1))
 
 /* XXX: must match A_ULP_RX_TDDP_PSZ */ 
 static int t4_ddp_pgsz[] = {4096, 4096 << 2, 4096 << 4, 4096 << 6};
 
 #if 0
 static void
 t4_dump_tcb(struct adapter *sc, int tid)
 {
 	uint32_t tcb_base, off, i, j;
 
 	/* Dump TCB for the tid */
 	tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
 	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2),
 	    tcb_base + tid * TCB_SIZE);
 	t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2));
 	off = 0;
 	printf("\n");
 	for (i = 0; i < 4; i++) {
 		uint32_t buf[8];
 		for (j = 0; j < 8; j++, off += 4)
 			buf[j] = htonl(t4_read_reg(sc, MEMWIN2_BASE + off));
 
 		printf("%08x %08x %08x %08x %08x %08x %08x %08x\n",
 		    buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
 		    buf[7]);
 	}
 }
 #endif
 
 #define MAX_DDP_BUFFER_SIZE		(M_TCB_RX_DDP_BUF0_LEN)
 static int
 alloc_ppods(struct tom_data *td, int n, struct ppod_region *pr)
 {
 	int ppod;
 
 	KASSERT(n > 0, ("%s: nonsense allocation (%d)", __func__, n));
 
 	mtx_lock(&td->ppod_lock);
 	if (n > td->nppods_free) {
 		mtx_unlock(&td->ppod_lock);
 		return (-1);
 	}
 
 	if (td->nppods_free_head >= n) {
 		td->nppods_free_head -= n;
 		ppod = td->nppods_free_head;
 		TAILQ_INSERT_HEAD(&td->ppods, pr, link);
 	} else {
 		struct ppod_region *p;
 
 		ppod = td->nppods_free_head;
 		TAILQ_FOREACH(p, &td->ppods, link) {
 			ppod += p->used + p->free;
 			if (n <= p->free) {
 				ppod -= n;
 				p->free -= n;
 				TAILQ_INSERT_AFTER(&td->ppods, p, pr, link);
 				goto allocated;
 			}
 		}
 
 		if (__predict_false(ppod != td->nppods)) {
 			panic("%s: ppods TAILQ (%p) corrupt."
 			    "  At %d instead of %d at the end of the queue.",
 			    __func__, &td->ppods, ppod, td->nppods);
 		}
 
 		mtx_unlock(&td->ppod_lock);
 		return (-1);
 	}
 
 allocated:
 	pr->used = n;
 	pr->free = 0;
 	td->nppods_free -= n;
 	mtx_unlock(&td->ppod_lock);
 
 	return (ppod);
 }
 
 static void
 free_ppods(struct tom_data *td, struct ppod_region *pr)
 {
 	struct ppod_region *p;
 
 	KASSERT(pr->used > 0, ("%s: nonsense free (%d)", __func__, pr->used));
 
 	mtx_lock(&td->ppod_lock);
 	p = TAILQ_PREV(pr, ppod_head, link);
 	if (p != NULL)
 		p->free += pr->used + pr->free;
 	else
 		td->nppods_free_head += pr->used + pr->free;
 	td->nppods_free += pr->used;
 	KASSERT(td->nppods_free <= td->nppods,
 	    ("%s: nppods_free (%d) > nppods (%d).  %d freed this time.",
 	    __func__, td->nppods_free, td->nppods, pr->used));
 	TAILQ_REMOVE(&td->ppods, pr, link);
 	mtx_unlock(&td->ppod_lock);
 }
 
 static inline int
 pages_to_nppods(int npages, int ddp_pgsz)
 {
 	int nsegs = npages * PAGE_SIZE / ddp_pgsz;
 
 	return (howmany(nsegs, PPOD_PAGES));
 }
 
 static void
 free_ddp_buffer(struct tom_data *td, struct ddp_buffer *db)
 {
 
 	if (db == NULL)
 		return;
 
 	if (db->pages)
 		free(db->pages, M_CXGBE);
 
 	if (db->nppods > 0)
 		free_ppods(td, &db->ppod_region);
 
 	free(db, M_CXGBE);
 }
 
 void
 release_ddp_resources(struct toepcb *toep)
 {
 	int i;
 
 	for (i = 0; i < nitems(toep->db); i++) {
 		if (toep->db[i] != NULL) {
 			free_ddp_buffer(toep->td, toep->db[i]);
 			toep->db[i] = NULL;
 		}
 	}
 }
 
 /* XXX: handle_ddp_data code duplication */
 void
 insert_ddp_data(struct toepcb *toep, uint32_t n)
 {
 	struct inpcb *inp = toep->inp;
 	struct tcpcb *tp = intotcpcb(inp);
 	struct sockbuf *sb = &inp->inp_socket->so_rcv;
 	struct mbuf *m;
 
 	INP_WLOCK_ASSERT(inp);
 	SOCKBUF_LOCK_ASSERT(sb);
 
 	m = get_ddp_mbuf(n);
 	tp->rcv_nxt += n;
 #ifndef USE_DDP_RX_FLOW_CONTROL
 	KASSERT(tp->rcv_wnd >= n, ("%s: negative window size", __func__));
 	tp->rcv_wnd -= n;
 #endif
 
 	KASSERT(toep->sb_cc >= sb->sb_cc,
 	    ("%s: sb %p has more data (%d) than last time (%d).",
 	    __func__, sb, sb->sb_cc, toep->sb_cc));
 	toep->rx_credits += toep->sb_cc - sb->sb_cc;
 #ifdef USE_DDP_RX_FLOW_CONTROL
 	toep->rx_credits -= n;	/* adjust for F_RX_FC_DDP */
 #endif
 	sbappendstream_locked(sb, m);
 	toep->sb_cc = sb->sb_cc;
 }
 
 /* SET_TCB_FIELD sent as a ULP command looks like this */
 #define LEN__SET_TCB_FIELD_ULP (sizeof(struct ulp_txpkt) + \
     sizeof(struct ulptx_idata) + sizeof(struct cpl_set_tcb_field_core))
 
 /* RX_DATA_ACK sent as a ULP command looks like this */
 #define LEN__RX_DATA_ACK_ULP (sizeof(struct ulp_txpkt) + \
     sizeof(struct ulptx_idata) + sizeof(struct cpl_rx_data_ack_core))
 
 static inline void *
 mk_set_tcb_field_ulp(struct ulp_txpkt *ulpmc, struct toepcb *toep,
     uint64_t word, uint64_t mask, uint64_t val)
 {
 	struct ulptx_idata *ulpsc;
 	struct cpl_set_tcb_field_core *req;
 
 	ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
 	ulpmc->len = htobe32(howmany(LEN__SET_TCB_FIELD_ULP, 16));
 
 	ulpsc = (struct ulptx_idata *)(ulpmc + 1);
 	ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
 	ulpsc->len = htobe32(sizeof(*req));
 
 	req = (struct cpl_set_tcb_field_core *)(ulpsc + 1);
 	OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, toep->tid));
 	req->reply_ctrl = htobe16(V_NO_REPLY(1) |
 	    V_QUEUENO(toep->ofld_rxq->iq.abs_id));
 	req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(0));
         req->mask = htobe64(mask);
         req->val = htobe64(val);
 
 	ulpsc = (struct ulptx_idata *)(req + 1);
 	if (LEN__SET_TCB_FIELD_ULP % 16) {
 		ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
 		ulpsc->len = htobe32(0);
 		return (ulpsc + 1);
 	}
 	return (ulpsc);
 }
 
 static inline void *
 mk_rx_data_ack_ulp(struct ulp_txpkt *ulpmc, struct toepcb *toep)
 {
 	struct ulptx_idata *ulpsc;
 	struct cpl_rx_data_ack_core *req;
 
 	ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
 	ulpmc->len = htobe32(howmany(LEN__RX_DATA_ACK_ULP, 16));
 
 	ulpsc = (struct ulptx_idata *)(ulpmc + 1);
 	ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
 	ulpsc->len = htobe32(sizeof(*req));
 
 	req = (struct cpl_rx_data_ack_core *)(ulpsc + 1);
 	OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tid));
 	req->credit_dack = htobe32(F_RX_MODULATE_RX);
 
 	ulpsc = (struct ulptx_idata *)(req + 1);
 	if (LEN__RX_DATA_ACK_ULP % 16) {
 		ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
 		ulpsc->len = htobe32(0);
 		return (ulpsc + 1);
 	}
 	return (ulpsc);
 }
 
 static inline uint64_t
 select_ddp_flags(struct socket *so, int flags, int db_idx)
 {
 	uint64_t ddp_flags = V_TF_DDP_INDICATE_OUT(0);
 	int waitall = flags & MSG_WAITALL;
 	int nb = so->so_state & SS_NBIO || flags & (MSG_DONTWAIT | MSG_NBIO);
 
 	KASSERT(db_idx == 0 || db_idx == 1,
 	    ("%s: bad DDP buffer index %d", __func__, db_idx));
 
 	if (db_idx == 0) {
 		ddp_flags |= V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_ACTIVE_BUF(0);
 		if (waitall)
 			ddp_flags |= V_TF_DDP_PUSH_DISABLE_0(1);
 		else if (nb)
 			ddp_flags |= V_TF_DDP_BUF0_FLUSH(1);
 		else
 			ddp_flags |= V_TF_DDP_BUF0_FLUSH(0);
 	} else {
 		ddp_flags |= V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_ACTIVE_BUF(1);
 		if (waitall)
 			ddp_flags |= V_TF_DDP_PUSH_DISABLE_1(1);
 		else if (nb)
 			ddp_flags |= V_TF_DDP_BUF1_FLUSH(1);
 		else
 			ddp_flags |= V_TF_DDP_BUF1_FLUSH(0);
 	}
 
 	return (ddp_flags);
 }
 
 static struct wrqe *
 mk_update_tcb_for_ddp(struct adapter *sc, struct toepcb *toep, int db_idx,
     int offset, uint64_t ddp_flags)
 {
 	struct ddp_buffer *db = toep->db[db_idx];
 	struct wrqe *wr;
 	struct work_request_hdr *wrh;
 	struct ulp_txpkt *ulpmc;
 	int len;
 
 	KASSERT(db_idx == 0 || db_idx == 1,
 	    ("%s: bad DDP buffer index %d", __func__, db_idx));
 
 	/*
 	 * We'll send a compound work request that has 3 SET_TCB_FIELDs and an
 	 * RX_DATA_ACK (with RX_MODULATE to speed up delivery).
 	 *
 	 * The work request header is 16B and always ends at a 16B boundary.
 	 * The ULPTX master commands that follow must all end at 16B boundaries
 	 * too so we round up the size to 16.
 	 */
 	len = sizeof(*wrh) + 3 * roundup2(LEN__SET_TCB_FIELD_ULP, 16) +
 	    roundup2(LEN__RX_DATA_ACK_ULP, 16);
 
 	wr = alloc_wrqe(len, toep->ctrlq);
 	if (wr == NULL)
 		return (NULL);
 	wrh = wrtod(wr);
 	INIT_ULPTX_WRH(wrh, len, 1, 0);	/* atomic */
 	ulpmc = (struct ulp_txpkt *)(wrh + 1);
 
 	/* Write the buffer's tag */
 	ulpmc = mk_set_tcb_field_ulp(ulpmc, toep,
 	    W_TCB_RX_DDP_BUF0_TAG + db_idx,
 	    V_TCB_RX_DDP_BUF0_TAG(M_TCB_RX_DDP_BUF0_TAG),
 	    V_TCB_RX_DDP_BUF0_TAG(db->tag));
 
 	/* Update the current offset in the DDP buffer and its total length */
 	if (db_idx == 0)
 		ulpmc = mk_set_tcb_field_ulp(ulpmc, toep,
 		    W_TCB_RX_DDP_BUF0_OFFSET,
 		    V_TCB_RX_DDP_BUF0_OFFSET(M_TCB_RX_DDP_BUF0_OFFSET) |
 		    V_TCB_RX_DDP_BUF0_LEN(M_TCB_RX_DDP_BUF0_LEN),
 		    V_TCB_RX_DDP_BUF0_OFFSET(offset) |
 		    V_TCB_RX_DDP_BUF0_LEN(db->len));
 	else
 		ulpmc = mk_set_tcb_field_ulp(ulpmc, toep,
 		    W_TCB_RX_DDP_BUF1_OFFSET,
 		    V_TCB_RX_DDP_BUF1_OFFSET(M_TCB_RX_DDP_BUF1_OFFSET) |
 		    V_TCB_RX_DDP_BUF1_LEN((u64)M_TCB_RX_DDP_BUF1_LEN << 32),
 		    V_TCB_RX_DDP_BUF1_OFFSET(offset) |
 		    V_TCB_RX_DDP_BUF1_LEN((u64)db->len << 32));
 
 	/* Update DDP flags */
 	ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, W_TCB_RX_DDP_FLAGS,
 	    V_TF_DDP_BUF0_FLUSH(1) | V_TF_DDP_BUF1_FLUSH(1) |
 	    V_TF_DDP_PUSH_DISABLE_0(1) | V_TF_DDP_PUSH_DISABLE_1(1) |
 	    V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_BUF1_VALID(1) |
 	    V_TF_DDP_ACTIVE_BUF(1) | V_TF_DDP_INDICATE_OUT(1), ddp_flags);
 
 	/* Gratuitous RX_DATA_ACK with RX_MODULATE set to speed up delivery. */
 	ulpmc = mk_rx_data_ack_ulp(ulpmc, toep);
 
 	return (wr);
 }
 
 static void
 discourage_ddp(struct toepcb *toep)
 {
 
 	if (toep->ddp_score && --toep->ddp_score == 0) {
 		toep->ddp_flags &= ~DDP_OK;
 		toep->ddp_disabled = time_uptime;
 		CTR3(KTR_CXGBE, "%s: tid %u !DDP_OK @ %u",
 		    __func__, toep->tid, time_uptime);
 	}
 }
 
 static int
 handle_ddp_data(struct toepcb *toep, __be32 ddp_report, __be32 rcv_nxt, int len)
 {
 	uint32_t report = be32toh(ddp_report);
 	unsigned int db_flag;
 	struct inpcb *inp = toep->inp;
 	struct tcpcb *tp;
 	struct socket *so;
 	struct sockbuf *sb;
 	struct mbuf *m;
 
 	db_flag = report & F_DDP_BUF_IDX ? DDP_BUF1_ACTIVE : DDP_BUF0_ACTIVE;
 
 	if (__predict_false(!(report & F_DDP_INV)))
 		CXGBE_UNIMPLEMENTED("DDP buffer still valid");
 
 	INP_WLOCK(inp);
 	so = inp_inpcbtosocket(inp);
 	sb = &so->so_rcv;
 	if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) {
 
 		/*
 		 * XXX: think a bit more.
 		 * tcpcb probably gone, but socket should still be around
 		 * because we always wait for DDP completion in soreceive no
 		 * matter what.  Just wake it up and let it clean up.
 		 */
 
 		CTR5(KTR_CXGBE, "%s: tid %u, seq 0x%x, len %d, inp_flags 0x%x",
 		    __func__, toep->tid, be32toh(rcv_nxt), len, inp->inp_flags);
 		SOCKBUF_LOCK(sb);
 		goto wakeup;
 	}
 
 	tp = intotcpcb(inp);
 	len += be32toh(rcv_nxt) - tp->rcv_nxt;
 	tp->rcv_nxt += len;
 	tp->t_rcvtime = ticks;
 #ifndef USE_DDP_RX_FLOW_CONTROL
 	KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__));
 	tp->rcv_wnd -= len;
 #endif
 	m = get_ddp_mbuf(len);
 
 	SOCKBUF_LOCK(sb);
 	if (report & F_DDP_BUF_COMPLETE)
 		toep->ddp_score = DDP_HIGH_SCORE;
 	else
 		discourage_ddp(toep);
 
 	KASSERT(toep->sb_cc >= sb->sb_cc,
 	    ("%s: sb %p has more data (%d) than last time (%d).",
 	    __func__, sb, sb->sb_cc, toep->sb_cc));
 	toep->rx_credits += toep->sb_cc - sb->sb_cc;
 #ifdef USE_DDP_RX_FLOW_CONTROL
 	toep->rx_credits -= len;	/* adjust for F_RX_FC_DDP */
 #endif
 	sbappendstream_locked(sb, m);
 	toep->sb_cc = sb->sb_cc;
 wakeup:
 	KASSERT(toep->ddp_flags & db_flag,
 	    ("%s: DDP buffer not active. toep %p, ddp_flags 0x%x, report 0x%x",
 	    __func__, toep, toep->ddp_flags, report));
 	toep->ddp_flags &= ~db_flag;
 	sorwakeup_locked(so);
 	SOCKBUF_UNLOCK_ASSERT(sb);
 
 	INP_WUNLOCK(inp);
 	return (0);
 }
 
 #define DDP_ERR (F_DDP_PPOD_MISMATCH | F_DDP_LLIMIT_ERR | F_DDP_ULIMIT_ERR |\
 	 F_DDP_PPOD_PARITY_ERR | F_DDP_PADDING_ERR | F_DDP_OFFSET_ERR |\
 	 F_DDP_INVALID_TAG | F_DDP_COLOR_ERR | F_DDP_TID_MISMATCH |\
 	 F_DDP_INVALID_PPOD | F_DDP_HDRCRC_ERR | F_DDP_DATACRC_ERR)
 
 static int
 do_rx_data_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
 	const struct cpl_rx_data_ddp *cpl = (const void *)(rss + 1);
 	unsigned int tid = GET_TID(cpl);
 	uint32_t vld;
 	struct toepcb *toep = lookup_tid(sc, tid);
+	struct tom_data *td = toep->td;
 
 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
 	KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
 	KASSERT(!(toep->flags & TPF_SYNQE),
 	    ("%s: toep %p claims to be a synq entry", __func__, toep));
 
 	vld = be32toh(cpl->ddpvld);
 	if (__predict_false(vld & DDP_ERR)) {
 		panic("%s: DDP error 0x%x (tid %d, toep %p)",
 		    __func__, vld, tid, toep);
 	}
+	if (toep->ulp_mode == ULP_MODE_ISCSI) {
+		m = m_get(M_NOWAIT, MT_DATA);
+		if (m == NULL)
+			CXGBE_UNIMPLEMENTED("mbuf alloc failure");
+		memcpy(mtod(m, unsigned char *), cpl,
+		    sizeof(struct cpl_rx_data_ddp));
+        	if (!t4_cpl_iscsi_callback(td, toep, m, CPL_RX_DATA_DDP))
+			return (0);
+		m_freem(m);
+        }
 
 	handle_ddp_data(toep, cpl->u.ddp_report, cpl->seq, be16toh(cpl->len));
 
 	return (0);
 }
 
 static int
 do_rx_ddp_complete(struct sge_iq *iq, const struct rss_header *rss,
     struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
 	const struct cpl_rx_ddp_complete *cpl = (const void *)(rss + 1);
 	unsigned int tid = GET_TID(cpl);
 	struct toepcb *toep = lookup_tid(sc, tid);
 
 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
 	KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
 	KASSERT(!(toep->flags & TPF_SYNQE),
 	    ("%s: toep %p claims to be a synq entry", __func__, toep));
 
 	handle_ddp_data(toep, cpl->ddp_report, cpl->rcv_nxt, 0);
 
 	return (0);
 }
 
 void
 enable_ddp(struct adapter *sc, struct toepcb *toep)
 {
 
 	KASSERT((toep->ddp_flags & (DDP_ON | DDP_OK | DDP_SC_REQ)) == DDP_OK,
 	    ("%s: toep %p has bad ddp_flags 0x%x",
 	    __func__, toep, toep->ddp_flags));
 
 	CTR3(KTR_CXGBE, "%s: tid %u (time %u)",
 	    __func__, toep->tid, time_uptime);
 
 	toep->ddp_flags |= DDP_SC_REQ;
 	t4_set_tcb_field(sc, toep, 1, W_TCB_RX_DDP_FLAGS,
 	    V_TF_DDP_OFF(1) | V_TF_DDP_INDICATE_OUT(1) |
 	    V_TF_DDP_BUF0_INDICATE(1) | V_TF_DDP_BUF1_INDICATE(1) |
 	    V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_BUF1_VALID(1),
 	    V_TF_DDP_BUF0_INDICATE(1) | V_TF_DDP_BUF1_INDICATE(1));
 	t4_set_tcb_field(sc, toep, 1, W_TCB_T_FLAGS,
 	    V_TF_RCV_COALESCE_ENABLE(1), 0);
 }
 
 static inline void
 disable_ddp(struct adapter *sc, struct toepcb *toep)
 {
 
 	KASSERT((toep->ddp_flags & (DDP_ON | DDP_SC_REQ)) == DDP_ON,
 	    ("%s: toep %p has bad ddp_flags 0x%x",
 	    __func__, toep, toep->ddp_flags));
 
 	CTR3(KTR_CXGBE, "%s: tid %u (time %u)",
 	    __func__, toep->tid, time_uptime);
 
 	toep->ddp_flags |= DDP_SC_REQ;
 	t4_set_tcb_field(sc, toep, 1, W_TCB_T_FLAGS,
 	    V_TF_RCV_COALESCE_ENABLE(1), V_TF_RCV_COALESCE_ENABLE(1));
 	t4_set_tcb_field(sc, toep, 1, W_TCB_RX_DDP_FLAGS, V_TF_DDP_OFF(1),
 	    V_TF_DDP_OFF(1));
 }
 
 static int
 hold_uio(struct uio *uio, vm_page_t **ppages, int *pnpages)
 {
 	struct vm_map *map;
 	struct iovec *iov;
 	vm_offset_t start, end;
 	vm_page_t *pp;
 	int n;
 
 	KASSERT(uio->uio_iovcnt == 1,
 	    ("%s: uio_iovcnt %d", __func__, uio->uio_iovcnt));
 	KASSERT(uio->uio_td->td_proc == curproc,
 	    ("%s: uio proc (%p) is not curproc (%p)",
 	    __func__, uio->uio_td->td_proc, curproc));
 
 	map = &curproc->p_vmspace->vm_map;
 	iov = &uio->uio_iov[0];
 	start = trunc_page((uintptr_t)iov->iov_base);
 	end = round_page((vm_offset_t)iov->iov_base + iov->iov_len);
 	n = howmany(end - start, PAGE_SIZE);
 
 	if (end - start > MAX_DDP_BUFFER_SIZE)
 		return (E2BIG);
 
 	pp = malloc(n * sizeof(vm_page_t), M_CXGBE, M_NOWAIT);
 	if (pp == NULL)
 		return (ENOMEM);
 
 	if (vm_fault_quick_hold_pages(map, (vm_offset_t)iov->iov_base,
 	    iov->iov_len, VM_PROT_WRITE, pp, n) < 0) {
 		free(pp, M_CXGBE);
 		return (EFAULT);
 	}
 
 	*ppages = pp;
 	*pnpages = n;
 
 	return (0);
 }
 
 static int
 bufcmp(struct ddp_buffer *db, vm_page_t *pages, int npages, int offset, int len)
 {
 	int i;
 
 	if (db == NULL || db->npages != npages || db->offset != offset ||
 	    db->len != len)
 		return (1);
 
 	for (i = 0; i < npages; i++) {
 		if (pages[i]->phys_addr != db->pages[i]->phys_addr)
 			return (1);
 	}
 
 	return (0);
 }
 
 static int
 calculate_hcf(int n1, int n2)
 {
 	int a, b, t;
 
 	if (n1 <= n2) {
 		a = n1;
 		b = n2;
 	} else {
 		a = n2;
 		b = n1;
 	}
 
 	while (a != 0) {
 		t = a;
 		a = b % a;
 		b = t;
 	}
 
 	return (b);
 }
 
 static struct ddp_buffer *
 alloc_ddp_buffer(struct tom_data *td, vm_page_t *pages, int npages, int offset,
     int len)
 {
 	int i, hcf, seglen, idx, ppod, nppods;
 	struct ddp_buffer *db;
 
 	/*
 	 * The DDP page size is unrelated to the VM page size.  We combine
 	 * contiguous physical pages into larger segments to get the best DDP
 	 * page size possible.  This is the largest of the four sizes in
 	 * A_ULP_RX_TDDP_PSZ that evenly divides the HCF of the segment sizes in
 	 * the page list.
 	 */
 	hcf = 0;
 	for (i = 0; i < npages; i++) {
 		seglen = PAGE_SIZE;
 		while (i < npages - 1 &&
 		    pages[i]->phys_addr + PAGE_SIZE == pages[i + 1]->phys_addr) {
 			seglen += PAGE_SIZE;
 			i++;
 		}
 
 		hcf = calculate_hcf(hcf, seglen);
 		if (hcf < t4_ddp_pgsz[1]) {
 			idx = 0;
 			goto have_pgsz;	/* give up, short circuit */
 		}
 	}
 
 	if (hcf % t4_ddp_pgsz[0] != 0) {
 		/* hmmm.  This could only happen when PAGE_SIZE < 4K */
 		KASSERT(PAGE_SIZE < 4096,
 		    ("%s: PAGE_SIZE %d, hcf %d", __func__, PAGE_SIZE, hcf));
 		CTR3(KTR_CXGBE, "%s: PAGE_SIZE %d, hcf %d",
 		    __func__, PAGE_SIZE, hcf);
 		return (NULL);
 	}
 
 	for (idx = nitems(t4_ddp_pgsz) - 1; idx > 0; idx--) {
 		if (hcf % t4_ddp_pgsz[idx] == 0)
 			break;
 	}
 have_pgsz:
 
 	db = malloc(sizeof(*db), M_CXGBE, M_NOWAIT);
 	if (db == NULL) {
 		CTR1(KTR_CXGBE, "%s: malloc failed.", __func__);
 		return (NULL);
 	}
 
 	nppods = pages_to_nppods(npages, t4_ddp_pgsz[idx]);
 	ppod = alloc_ppods(td, nppods, &db->ppod_region);
 	if (ppod < 0) {
 		free(db, M_CXGBE);
 		CTR4(KTR_CXGBE, "%s: no pods, nppods %d, resid %d, pgsz %d",
 		    __func__, nppods, len, t4_ddp_pgsz[idx]);
 		return (NULL);
 	}
 
 	KASSERT(idx <= M_PPOD_PGSZ && ppod <= M_PPOD_TAG,
 	    ("%s: DDP pgsz_idx = %d, ppod = %d", __func__, idx, ppod));
 
 	db->tag = V_PPOD_PGSZ(idx) | V_PPOD_TAG(ppod);
 	db->nppods = nppods;
 	db->npages = npages;
 	db->pages = pages;
 	db->offset = offset;
 	db->len = len;
 
 	CTR6(KTR_CXGBE, "New DDP buffer.  "
 	    "ddp_pgsz %d, ppod 0x%x, npages %d, nppods %d, offset %d, len %d",
 	    t4_ddp_pgsz[idx], ppod, db->npages, db->nppods, db->offset,
 	    db->len);
 
 	return (db);
 }
 
 #define NUM_ULP_TX_SC_IMM_PPODS (256 / PPOD_SIZE)
 
 static int
 write_page_pods(struct adapter *sc, struct toepcb *toep, struct ddp_buffer *db)
 {
 	struct wrqe *wr;
 	struct ulp_mem_io *ulpmc;
 	struct ulptx_idata *ulpsc;
 	struct pagepod *ppod;
 	int i, j, k, n, chunk, len, ddp_pgsz, idx, ppod_addr;
 	uint32_t cmd;
 
 	cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE));
 	if (is_t4(sc))
 		cmd |= htobe32(F_ULP_MEMIO_ORDER);
 	else
 		cmd |= htobe32(F_T5_ULP_MEMIO_IMM);
 	ddp_pgsz = t4_ddp_pgsz[G_PPOD_PGSZ(db->tag)];
 	ppod_addr = sc->vres.ddp.start + G_PPOD_TAG(db->tag) * PPOD_SIZE;
 	for (i = 0; i < db->nppods; ppod_addr += chunk) {
 
 		/* How many page pods are we writing in this cycle */
 		n = min(db->nppods - i, NUM_ULP_TX_SC_IMM_PPODS);
 		chunk = PPOD_SZ(n);
 		len = roundup2(sizeof(*ulpmc) + sizeof(*ulpsc) + chunk, 16);
 
 		wr = alloc_wrqe(len, toep->ctrlq);
 		if (wr == NULL)
 			return (ENOMEM);	/* ok to just bail out */
 		ulpmc = wrtod(wr);
 
 		INIT_ULPTX_WR(ulpmc, len, 0, 0);
 		ulpmc->cmd = cmd;
 		ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32));
 		ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16));
 		ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5));
 
 		ulpsc = (struct ulptx_idata *)(ulpmc + 1);
 		ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
 		ulpsc->len = htobe32(chunk);
 
 		ppod = (struct pagepod *)(ulpsc + 1);
 		for (j = 0; j < n; i++, j++, ppod++) {
 			ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID |
 			    V_PPOD_TID(toep->tid) | db->tag);
 			ppod->len_offset = htobe64(V_PPOD_LEN(db->len) |
 			    V_PPOD_OFST(db->offset));
 			ppod->rsvd = 0;
 			idx = i * PPOD_PAGES * (ddp_pgsz / PAGE_SIZE);
 			for (k = 0; k < nitems(ppod->addr); k++) {
 				if (idx < db->npages) {
 					ppod->addr[k] =
 					    htobe64(db->pages[idx]->phys_addr);
 					idx += ddp_pgsz / PAGE_SIZE;
 				} else
 					ppod->addr[k] = 0;
 #if 0
 				CTR5(KTR_CXGBE,
 				    "%s: tid %d ppod[%d]->addr[%d] = %p",
 				    __func__, toep->tid, i, k,
 				    htobe64(ppod->addr[k]));
 #endif
 			}
 
 		}
 
 		t4_wrq_tx(sc, wr);
 	}
 
 	return (0);
 }
 
 /*
  * Reuse, or allocate (and program the page pods for) a new DDP buffer.  The
  * "pages" array is handed over to this function and should not be used in any
  * way by the caller after that.
  */
 static int
 select_ddp_buffer(struct adapter *sc, struct toepcb *toep, vm_page_t *pages,
     int npages, int db_off, int db_len)
 {
 	struct ddp_buffer *db;
 	struct tom_data *td = sc->tom_softc;
 	int i, empty_slot = -1;
 
 	/* Try to reuse */
 	for (i = 0; i < nitems(toep->db); i++) {
 		if (bufcmp(toep->db[i], pages, npages, db_off, db_len) == 0) {
 			free(pages, M_CXGBE);
 			return (i);	/* pages still held */
 		} else if (toep->db[i] == NULL && empty_slot < 0)
 			empty_slot = i;
 	}
 
 	/* Allocate new buffer, write its page pods. */
 	db = alloc_ddp_buffer(td, pages, npages, db_off, db_len);
 	if (db == NULL) {
 		vm_page_unhold_pages(pages, npages);
 		free(pages, M_CXGBE);
 		return (-1);
 	}
 	if (write_page_pods(sc, toep, db) != 0) {
 		vm_page_unhold_pages(pages, npages);
 		free_ddp_buffer(td, db);
 		return (-1);
 	}
 
 	i = empty_slot;
 	if (i < 0) {
 		i = arc4random() % nitems(toep->db);
 		free_ddp_buffer(td, toep->db[i]);
 	}
 	toep->db[i] = db;
 
 	CTR5(KTR_CXGBE, "%s: tid %d, DDP buffer[%d] = %p (tag 0x%x)",
 	    __func__, toep->tid, i, db, db->tag);
 
 	return (i);
 }
 
 static void
 wire_ddp_buffer(struct ddp_buffer *db)
 {
 	int i;
 	vm_page_t p;
 
 	for (i = 0; i < db->npages; i++) {
 		p = db->pages[i];
 		vm_page_lock(p);
 		vm_page_wire(p);
 		vm_page_unhold(p);
 		vm_page_unlock(p);
 	}
 }
 
 static void
 unwire_ddp_buffer(struct ddp_buffer *db)
 {
 	int i;
 	vm_page_t p;
 
 	for (i = 0; i < db->npages; i++) {
 		p = db->pages[i];
 		vm_page_lock(p);
 		vm_page_unwire(p, 0);
 		vm_page_unlock(p);
 	}
 }
 
 static int
 handle_ddp(struct socket *so, struct uio *uio, int flags, int error)
 {
 	struct sockbuf *sb = &so->so_rcv;
 	struct tcpcb *tp = so_sototcpcb(so);
 	struct toepcb *toep = tp->t_toe;
 	struct adapter *sc = td_adapter(toep->td);
 	vm_page_t *pages;
 	int npages, db_idx, rc, buf_flag;
 	struct ddp_buffer *db;
 	struct wrqe *wr;
 	uint64_t ddp_flags;
 
 	SOCKBUF_LOCK_ASSERT(sb);
 
 #if 0
 	if (sb->sb_cc + sc->tt.ddp_thres > uio->uio_resid) {
 		CTR4(KTR_CXGBE, "%s: sb_cc %d, threshold %d, resid %d",
 		    __func__, sb->sb_cc, sc->tt.ddp_thres, uio->uio_resid);
 	}
 #endif
 
 	/* XXX: too eager to disable DDP, could handle NBIO better than this. */
 	if (sb->sb_cc >= uio->uio_resid || uio->uio_resid < sc->tt.ddp_thres ||
 	    uio->uio_resid > MAX_DDP_BUFFER_SIZE || uio->uio_iovcnt > 1 ||
 	    so->so_state & SS_NBIO || flags & (MSG_DONTWAIT | MSG_NBIO) ||
 	    error || so->so_error || sb->sb_state & SBS_CANTRCVMORE)
 		goto no_ddp;
 
 	/*
 	 * Fault in and then hold the pages of the uio buffers.  We'll wire them
 	 * a bit later if everything else works out.
 	 */
 	SOCKBUF_UNLOCK(sb);
 	if (hold_uio(uio, &pages, &npages) != 0) {
 		SOCKBUF_LOCK(sb);
 		goto no_ddp;
 	}
 	SOCKBUF_LOCK(sb);
 	if (__predict_false(so->so_error || sb->sb_state & SBS_CANTRCVMORE)) {
 		vm_page_unhold_pages(pages, npages);
 		free(pages, M_CXGBE);
 		goto no_ddp;
 	}
 
 	/*
 	 * Figure out which one of the two DDP buffers to use this time.
 	 */
 	db_idx = select_ddp_buffer(sc, toep, pages, npages,
 	    (uintptr_t)uio->uio_iov->iov_base & PAGE_MASK, uio->uio_resid);
 	pages = NULL;	/* handed off to select_ddp_buffer */
 	if (db_idx < 0)
 		goto no_ddp;
 	db = toep->db[db_idx];
 	buf_flag = db_idx == 0 ? DDP_BUF0_ACTIVE : DDP_BUF1_ACTIVE;
 
 	/*
 	 * Build the compound work request that tells the chip where to DMA the
 	 * payload.
 	 */
 	ddp_flags = select_ddp_flags(so, flags, db_idx);
 	wr = mk_update_tcb_for_ddp(sc, toep, db_idx, sb->sb_cc, ddp_flags);
 	if (wr == NULL) {
 		/*
 		 * Just unhold the pages.  The DDP buffer's software state is
 		 * left as-is in the toep.  The page pods were written
 		 * successfully and we may have an opportunity to use it in the
 		 * future.
 		 */
 		vm_page_unhold_pages(db->pages, db->npages);
 		goto no_ddp;
 	}
 
 	/* Wire (and then unhold) the pages, and give the chip the go-ahead. */
 	wire_ddp_buffer(db);
 	t4_wrq_tx(sc, wr);
 	sb->sb_flags &= ~SB_DDP_INDICATE;
 	toep->ddp_flags |= buf_flag;
 
 	/*
 	 * Wait for the DDP operation to complete and then unwire the pages.
 	 * The return code from the sbwait will be the final return code of this
 	 * function.  But we do need to wait for DDP no matter what.
 	 */
 	rc = sbwait(sb);
 	while (toep->ddp_flags & buf_flag) {
 		sb->sb_flags |= SB_WAIT;
 		msleep(&sb->sb_cc, &sb->sb_mtx, PSOCK , "sbwait", 0);
 	}
 	unwire_ddp_buffer(db);
 	return (rc);
 no_ddp:
 	disable_ddp(sc, toep);
 	discourage_ddp(toep);
 	sb->sb_flags &= ~SB_DDP_INDICATE;
 	return (0);
 }
 
 void
 t4_init_ddp(struct adapter *sc, struct tom_data *td)
 {
 	int nppods = sc->vres.ddp.size / PPOD_SIZE;
 
 	td->nppods = nppods;
 	td->nppods_free = nppods;
 	td->nppods_free_head = nppods;
 	TAILQ_INIT(&td->ppods);
 	mtx_init(&td->ppod_lock, "page pods", NULL, MTX_DEF);
 
 	t4_register_cpl_handler(sc, CPL_RX_DATA_DDP, do_rx_data_ddp);
 	t4_register_cpl_handler(sc, CPL_RX_DDP_COMPLETE, do_rx_ddp_complete);
 }
 
 void
 t4_uninit_ddp(struct adapter *sc __unused, struct tom_data *td)
 {
 
 	KASSERT(td->nppods == td->nppods_free,
 	    ("%s: page pods still in use, nppods = %d, free = %d",
 	    __func__, td->nppods, td->nppods_free));
 
 	if (mtx_initialized(&td->ppod_lock))
 		mtx_destroy(&td->ppod_lock);
 }
 
 #define	VNET_SO_ASSERT(so)						\
 	VNET_ASSERT(curvnet != NULL,					\
 	    ("%s:%d curvnet is NULL, so=%p", __func__, __LINE__, (so)));
 #define	SBLOCKWAIT(f)	(((f) & MSG_DONTWAIT) ? 0 : SBL_WAIT)
 static int
 soreceive_rcvoob(struct socket *so, struct uio *uio, int flags)
 {
 
 	CXGBE_UNIMPLEMENTED(__func__);
 }
 
 static char ddp_magic_str[] = "nothing to see here";
 
 struct mbuf *
 get_ddp_mbuf(int len)
 {
 	struct mbuf *m;
 
 	m = m_get(M_NOWAIT, MT_DATA);
 	if (m == NULL)
 		CXGBE_UNIMPLEMENTED("mbuf alloc failure");
 	m->m_len = len;
 	m->m_data = &ddp_magic_str[0];
 
 	return (m);
 }
 
 static inline int
 is_ddp_mbuf(struct mbuf *m)
 {
 
 	return (m->m_data == &ddp_magic_str[0]);
 }
 
 /*
  * Copy an mbuf chain into a uio limited by len if set.
  */
 static int
 m_mbuftouio_ddp(struct uio *uio, struct mbuf *m, int len)
 {
 	int error, length, total;
 	int progress = 0;
 
 	if (len > 0)
 		total = min(uio->uio_resid, len);
 	else
 		total = uio->uio_resid;
 
 	/* Fill the uio with data from the mbufs. */
 	for (; m != NULL; m = m->m_next) {
 		length = min(m->m_len, total - progress);
 
 		if (is_ddp_mbuf(m)) {
 			enum uio_seg segflag = uio->uio_segflg;
 
 			uio->uio_segflg	= UIO_NOCOPY;
 			error = uiomove(mtod(m, void *), length, uio);
 			uio->uio_segflg	= segflag;
 		} else
 			error = uiomove(mtod(m, void *), length, uio);
 		if (error)
 			return (error);
 
 		progress += length;
 	}
 
 	return (0);
 }
 
 /*
  * Based on soreceive_stream() in uipc_socket.c
  */
 int
 t4_soreceive_ddp(struct socket *so, struct sockaddr **psa, struct uio *uio,
     struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
 {
 	int len = 0, error = 0, flags, oresid, ddp_handled = 0;
 	struct sockbuf *sb;
 	struct mbuf *m, *n = NULL;
 
 	/* We only do stream sockets. */
 	if (so->so_type != SOCK_STREAM)
 		return (EINVAL);
 	if (psa != NULL)
 		*psa = NULL;
 	if (controlp != NULL)
 		return (EINVAL);
 	if (flagsp != NULL)
 		flags = *flagsp &~ MSG_EOR;
 	else
 		flags = 0;
 	if (flags & MSG_OOB)
 		return (soreceive_rcvoob(so, uio, flags));
 	if (mp0 != NULL)
 		*mp0 = NULL;
 
 	sb = &so->so_rcv;
 
 	/* Prevent other readers from entering the socket. */
 	error = sblock(sb, SBLOCKWAIT(flags));
 	if (error)
 		goto out;
 	SOCKBUF_LOCK(sb);
 
 	/* Easy one, no space to copyout anything. */
 	if (uio->uio_resid == 0) {
 		error = EINVAL;
 		goto out;
 	}
 	oresid = uio->uio_resid;
 
 	/* We will never ever get anything unless we are or were connected. */
 	if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) {
 		error = ENOTCONN;
 		goto out;
 	}
 
 restart:
 	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
 
 	if (sb->sb_flags & SB_DDP_INDICATE && !ddp_handled) {
 
 		/* uio should be just as it was at entry */
 		KASSERT(oresid == uio->uio_resid,
 		    ("%s: oresid = %d, uio_resid = %zd, sb_cc = %d",
 		    __func__, oresid, uio->uio_resid, sb->sb_cc));
 
 		error = handle_ddp(so, uio, flags, 0);
 		ddp_handled = 1;
 		if (error)
 			goto out;
 	}
 
 	/* Abort if socket has reported problems. */
 	if (so->so_error) {
 		if (sb->sb_cc > 0)
 			goto deliver;
 		if (oresid > uio->uio_resid)
 			goto out;
 		error = so->so_error;
 		if (!(flags & MSG_PEEK))
 			so->so_error = 0;
 		goto out;
 	}
 
 	/* Door is closed.  Deliver what is left, if any. */
 	if (sb->sb_state & SBS_CANTRCVMORE) {
 		if (sb->sb_cc > 0)
 			goto deliver;
 		else
 			goto out;
 	}
 
 	/* Socket buffer is empty and we shall not block. */
 	if (sb->sb_cc == 0 &&
 	    ((so->so_state & SS_NBIO) || (flags & (MSG_DONTWAIT|MSG_NBIO)))) {
 		error = EAGAIN;
 		goto out;
 	}
 
 	/* Socket buffer got some data that we shall deliver now. */
 	if (sb->sb_cc > 0 && !(flags & MSG_WAITALL) &&
 	    ((sb->sb_flags & SS_NBIO) ||
 	     (flags & (MSG_DONTWAIT|MSG_NBIO)) ||
 	     sb->sb_cc >= sb->sb_lowat ||
 	     sb->sb_cc >= uio->uio_resid ||
 	     sb->sb_cc >= sb->sb_hiwat) ) {
 		goto deliver;
 	}
 
 	/* On MSG_WAITALL we must wait until all data or error arrives. */
 	if ((flags & MSG_WAITALL) &&
 	    (sb->sb_cc >= uio->uio_resid || sb->sb_cc >= sb->sb_lowat))
 		goto deliver;
 
 	/*
 	 * Wait and block until (more) data comes in.
 	 * NB: Drops the sockbuf lock during wait.
 	 */
 	error = sbwait(sb);
 	if (error) {
 		if (sb->sb_flags & SB_DDP_INDICATE && !ddp_handled) {
 			(void) handle_ddp(so, uio, flags, 1);
 			ddp_handled = 1;
 		}
 		goto out;
 	}
 	goto restart;
 
 deliver:
 	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
 	KASSERT(sb->sb_cc > 0, ("%s: sockbuf empty", __func__));
 	KASSERT(sb->sb_mb != NULL, ("%s: sb_mb == NULL", __func__));
 
 	if (sb->sb_flags & SB_DDP_INDICATE && !ddp_handled)
 		goto restart;
 
 	/* Statistics. */
 	if (uio->uio_td)
 		uio->uio_td->td_ru.ru_msgrcv++;
 
 	/* Fill uio until full or current end of socket buffer is reached. */
 	len = min(uio->uio_resid, sb->sb_cc);
 	if (mp0 != NULL) {
 		/* Dequeue as many mbufs as possible. */
 		if (!(flags & MSG_PEEK) && len >= sb->sb_mb->m_len) {
 			for (*mp0 = m = sb->sb_mb;
 			     m != NULL && m->m_len <= len;
 			     m = m->m_next) {
 				len -= m->m_len;
 				uio->uio_resid -= m->m_len;
 				sbfree(sb, m);
 				n = m;
 			}
 			sb->sb_mb = m;
 			if (sb->sb_mb == NULL)
 				SB_EMPTY_FIXUP(sb);
 			n->m_next = NULL;
 		}
 		/* Copy the remainder. */
 		if (len > 0) {
 			KASSERT(sb->sb_mb != NULL,
 			    ("%s: len > 0 && sb->sb_mb empty", __func__));
 
 			m = m_copym(sb->sb_mb, 0, len, M_NOWAIT);
 			if (m == NULL)
 				len = 0;	/* Don't flush data from sockbuf. */
 			else
 				uio->uio_resid -= m->m_len;
 			if (*mp0 != NULL)
 				n->m_next = m;
 			else
 				*mp0 = m;
 			if (*mp0 == NULL) {
 				error = ENOBUFS;
 				goto out;
 			}
 		}
 	} else {
 		/* NB: Must unlock socket buffer as uiomove may sleep. */
 		SOCKBUF_UNLOCK(sb);
 		error = m_mbuftouio_ddp(uio, sb->sb_mb, len);
 		SOCKBUF_LOCK(sb);
 		if (error)
 			goto out;
 	}
 	SBLASTRECORDCHK(sb);
 	SBLASTMBUFCHK(sb);
 
 	/*
 	 * Remove the delivered data from the socket buffer unless we
 	 * were only peeking.
 	 */
 	if (!(flags & MSG_PEEK)) {
 		if (len > 0)
 			sbdrop_locked(sb, len);
 
 		/* Notify protocol that we drained some data. */
 		if ((so->so_proto->pr_flags & PR_WANTRCVD) &&
 		    (((flags & MSG_WAITALL) && uio->uio_resid > 0) ||
 		     !(flags & MSG_SOCALLBCK))) {
 			SOCKBUF_UNLOCK(sb);
 			VNET_SO_ASSERT(so);
 			(*so->so_proto->pr_usrreqs->pru_rcvd)(so, flags);
 			SOCKBUF_LOCK(sb);
 		}
 	}
 
 	/*
 	 * For MSG_WAITALL we may have to loop again and wait for
 	 * more data to come in.
 	 */
 	if ((flags & MSG_WAITALL) && uio->uio_resid > 0)
 		goto restart;
 out:
 	SOCKBUF_LOCK_ASSERT(sb);
 	SBLASTRECORDCHK(sb);
 	SBLASTMBUFCHK(sb);
 	SOCKBUF_UNLOCK(sb);
 	sbunlock(sb);
 	return (error);
 }
 
 #endif
Index: stable/10/sys/dev/cxgbe/tom/t4_tom.h
===================================================================
--- stable/10/sys/dev/cxgbe/tom/t4_tom.h	(revision 270296)
+++ stable/10/sys/dev/cxgbe/tom/t4_tom.h	(revision 270297)
@@ -1,292 +1,309 @@
 /*-
  * Copyright (c) 2012 Chelsio Communications, Inc.
  * All rights reserved.
  * Written by: Navdeep Parhar <np@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $FreeBSD$
  *
  */
 
 #ifndef __T4_TOM_H__
 #define __T4_TOM_H__
 
 #define KTR_CXGBE	KTR_SPARE3
 #define LISTEN_HASH_SIZE 32
 
 /*
  * Min receive window.  We want it to be large enough to accommodate receive
  * coalescing, handle jumbo frames, and not trigger sender SWS avoidance.
  */
 #define MIN_RCV_WND (24 * 1024U)
 
 /*
  * Max receive window supported by HW in bytes.  Only a small part of it can
  * be set through option0, the rest needs to be set through RX_DATA_ACK.
  */
 #define MAX_RCV_WND ((1U << 27) - 1)
 
 #define	DDP_RSVD_WIN (16 * 1024U)
 #define	SB_DDP_INDICATE	SB_IN_TOE	/* soreceive must respond to indicate */
 
 #define USE_DDP_RX_FLOW_CONTROL
 
 /* TOE PCB flags */
 enum {
 	TPF_ATTACHED	   = (1 << 0),	/* a tcpcb refers to this toepcb */
 	TPF_FLOWC_WR_SENT  = (1 << 1),	/* firmware flow context WR sent */
 	TPF_TX_DATA_SENT   = (1 << 2),	/* some data sent */
 	TPF_TX_SUSPENDED   = (1 << 3),	/* tx suspended for lack of resources */
 	TPF_SEND_FIN	   = (1 << 4),	/* send FIN after all pending data */
 	TPF_FIN_SENT	   = (1 << 5),	/* FIN has been sent */
 	TPF_ABORT_SHUTDOWN = (1 << 6),	/* connection abort is in progress */
 	TPF_CPL_PENDING    = (1 << 7),	/* haven't received the last CPL */
 	TPF_SYNQE	   = (1 << 8),	/* synq_entry, not really a toepcb */
 	TPF_SYNQE_NEEDFREE = (1 << 9),	/* synq_entry was malloc'd separately */
 	TPF_SYNQE_TCPDDP   = (1 << 10),	/* ulp_mode TCPDDP in toepcb */
 	TPF_SYNQE_EXPANDED = (1 << 11),	/* toepcb ready, tid context updated */
 	TPF_SYNQE_HAS_L2TE = (1 << 12),	/* we've replied to PASS_ACCEPT_REQ */
 };
 
 enum {
 	DDP_OK		= (1 << 0),	/* OK to turn on DDP */
 	DDP_SC_REQ	= (1 << 1),	/* state change (on/off) requested */
 	DDP_ON		= (1 << 2),	/* DDP is turned on */
 	DDP_BUF0_ACTIVE	= (1 << 3),	/* buffer 0 in use (not invalidated) */
 	DDP_BUF1_ACTIVE	= (1 << 4),	/* buffer 1 in use (not invalidated) */
 };
 
 struct ofld_tx_sdesc {
 	uint32_t plen;		/* payload length */
 	uint8_t tx_credits;	/* firmware tx credits (unit is 16B) */
 };
 
 struct ppod_region {
 	TAILQ_ENTRY(ppod_region) link;
 	int used;	/* # of pods used by this region */
 	int free;	/* # of contiguous pods free right after this region */
 };
 
 struct ddp_buffer {
 	uint32_t tag;	/* includes color, page pod addr, and DDP page size */
 	int nppods;
 	int offset;
 	int len;
 	struct ppod_region ppod_region;
 	int npages;
 	vm_page_t *pages;
 };
 
 struct toepcb {
 	TAILQ_ENTRY(toepcb) link; /* toep_list */
 	u_int flags;		/* miscellaneous flags */
 	struct tom_data *td;
 	struct inpcb *inp;	/* backpointer to host stack's PCB */
 	struct port_info *port;	/* physical port */
 	struct sge_wrq *ofld_txq;
 	struct sge_ofld_rxq *ofld_rxq;
 	struct sge_wrq *ctrlq;
 	struct l2t_entry *l2te;	/* L2 table entry used by this connection */
 	struct clip_entry *ce;	/* CLIP table entry used by this tid */
 	int tid;		/* Connection identifier */
 
 	/* tx credit handling */
 	u_int tx_total;		/* total tx WR credits (in 16B units) */
 	u_int tx_credits;	/* tx WR credits (in 16B units) available */
 	u_int tx_nocompl;	/* tx WR credits since last compl request */
 	u_int plen_nocompl;	/* payload since last compl request */
 
 	/* rx credit handling */
 	u_int sb_cc;		/* last noted value of so_rcv->sb_cc */
 	int rx_credits;		/* rx credits (in bytes) to be returned to hw */
 
 	u_int ulp_mode;	/* ULP mode */
 
 	u_int ddp_flags;
 	struct ddp_buffer *db[2];
 	time_t ddp_disabled;
 	uint8_t ddp_score;
 
 	/* Tx software descriptor */
 	uint8_t txsd_total;
 	uint8_t txsd_pidx;
 	uint8_t txsd_cidx;
 	uint8_t txsd_avail;
 	struct ofld_tx_sdesc txsd[];
 };
 
 struct flowc_tx_params {
 	uint32_t snd_nxt;
 	uint32_t rcv_nxt;
 	unsigned int snd_space;
 	unsigned int mss;
 };
 
 #define	DDP_RETRY_WAIT	5	/* seconds to wait before re-enabling DDP */
 #define	DDP_LOW_SCORE	1
 #define	DDP_HIGH_SCORE	3
 
 /*
  * Compressed state for embryonic connections for a listener.  Barely fits in
  * 64B, try not to grow it further.
  */
 struct synq_entry {
 	TAILQ_ENTRY(synq_entry) link;	/* listen_ctx's synq link */
 	int flags;			/* same as toepcb's tp_flags */
 	int tid;
 	struct listen_ctx *lctx;	/* backpointer to listen ctx */
 	struct mbuf *syn;
 	uint32_t iss;
 	uint32_t ts;
 	volatile uintptr_t wr;
 	volatile u_int refcnt;
 	uint16_t l2e_idx;
 	uint16_t rcv_bufsize;
 };
 
 /* listen_ctx flags */
 #define LCTX_RPL_PENDING 1	/* waiting for a CPL_PASS_OPEN_RPL */
 
 struct listen_ctx {
 	LIST_ENTRY(listen_ctx) link;	/* listen hash linkage */
 	volatile int refcount;
 	int stid;
 	struct stid_region stid_region;
 	int flags;
 	struct inpcb *inp;		/* listening socket's inp */
 	struct sge_wrq *ctrlq;
 	struct sge_ofld_rxq *ofld_rxq;
 	struct clip_entry *ce;
 	TAILQ_HEAD(, synq_entry) synq;
 };
 
 TAILQ_HEAD(ppod_head, ppod_region);
 
 struct clip_entry {
 	TAILQ_ENTRY(clip_entry) link;
 	struct in6_addr lip;	/* local IPv6 address */
 	u_int refcount;
 };
 
 TAILQ_HEAD(clip_head, clip_entry);
 struct tom_data {
 	struct toedev tod;
 
 	/* toepcb's associated with this TOE device */
 	struct mtx toep_list_lock;
 	TAILQ_HEAD(, toepcb) toep_list;
 
 	struct mtx lctx_hash_lock;
 	LIST_HEAD(, listen_ctx) *listen_hash;
 	u_long listen_mask;
 	int lctx_count;		/* # of lctx in the hash table */
 
 	struct mtx ppod_lock;
 	int nppods;
 	int nppods_free;	/* # of available ppods */
 	int nppods_free_head;	/* # of available ppods at the begining */
 	struct ppod_head ppods;
 
 	struct mtx clip_table_lock;
 	struct clip_head clip_table;
 	int clip_gen;
 };
 
 static inline struct tom_data *
 tod_td(struct toedev *tod)
 {
 
 	return (__containerof(tod, struct tom_data, tod));
 }
 
 static inline struct adapter *
 td_adapter(struct tom_data *td)
 {
 
 	return (td->tod.tod_softc);
 }
 
 /* t4_tom.c */
 struct toepcb *alloc_toepcb(struct port_info *, int, int, int);
 void free_toepcb(struct toepcb *);
 void offload_socket(struct socket *, struct toepcb *);
 void undo_offload_socket(struct socket *);
 void final_cpl_received(struct toepcb *);
 void insert_tid(struct adapter *, int, void *);
 void *lookup_tid(struct adapter *, int);
 void update_tid(struct adapter *, int, void *);
 void remove_tid(struct adapter *, int);
 void release_tid(struct adapter *, int, struct sge_wrq *);
 int find_best_mtu_idx(struct adapter *, struct in_conninfo *, int);
 u_long select_rcv_wnd(struct socket *);
 int select_rcv_wscale(void);
 uint64_t calc_opt0(struct socket *, struct port_info *, struct l2t_entry *,
     int, int, int, int);
 uint64_t select_ntuple(struct port_info *, struct l2t_entry *);
 void set_tcpddp_ulp_mode(struct toepcb *);
 int negative_advice(int);
 struct clip_entry *hold_lip(struct tom_data *, struct in6_addr *);
 void release_lip(struct tom_data *, struct clip_entry *);
 
 /* t4_connect.c */
 void t4_init_connect_cpl_handlers(struct adapter *);
 int t4_connect(struct toedev *, struct socket *, struct rtentry *,
     struct sockaddr *);
 
 /* t4_listen.c */
 void t4_init_listen_cpl_handlers(struct adapter *);
 int t4_listen_start(struct toedev *, struct tcpcb *);
 int t4_listen_stop(struct toedev *, struct tcpcb *);
 void t4_syncache_added(struct toedev *, void *);
 void t4_syncache_removed(struct toedev *, void *);
 int t4_syncache_respond(struct toedev *, void *, struct mbuf *);
 int do_abort_req_synqe(struct sge_iq *, const struct rss_header *,
     struct mbuf *);
 int do_abort_rpl_synqe(struct sge_iq *, const struct rss_header *,
     struct mbuf *);
 void t4_offload_socket(struct toedev *, void *, struct socket *);
 
 /* t4_cpl_io.c */
 void t4_init_cpl_io_handlers(struct adapter *);
 void t4_uninit_cpl_io_handlers(struct adapter *);
 void send_abort_rpl(struct adapter *, struct sge_wrq *, int , int);
 void send_flowc_wr(struct toepcb *, struct flowc_tx_params *);
 void send_reset(struct adapter *, struct toepcb *, uint32_t);
 void make_established(struct toepcb *, uint32_t, uint32_t, uint16_t);
 void t4_rcvd(struct toedev *, struct tcpcb *);
 int t4_tod_output(struct toedev *, struct tcpcb *);
 int t4_send_fin(struct toedev *, struct tcpcb *);
 int t4_send_rst(struct toedev *, struct tcpcb *);
 void t4_set_tcb_field(struct adapter *, struct toepcb *, int, uint16_t,
     uint64_t, uint64_t);
+void t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop);
 
 /* t4_ddp.c */
 void t4_init_ddp(struct adapter *, struct tom_data *);
 void t4_uninit_ddp(struct adapter *, struct tom_data *);
 int t4_soreceive_ddp(struct socket *, struct sockaddr **, struct uio *,
     struct mbuf **, struct mbuf **, int *);
 struct mbuf *get_ddp_mbuf(int);
 void enable_ddp(struct adapter *, struct toepcb *toep);
 void release_ddp_resources(struct toepcb *toep);
 void insert_ddp_data(struct toepcb *, uint32_t);
+
+/* ULP related */
+#define CXGBE_ISCSI_MBUF_TAG          50
+int t4tom_cpl_handler_registered(struct adapter *, unsigned int);
+void t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *,
+    struct socket *, void *, unsigned int));
+void t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *,
+    unsigned int, int *));
+void t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int);
+int t4_cpl_iscsi_callback(struct tom_data *, struct toepcb *, void *, uint32_t);
+struct mbuf *t4_queue_iscsi_callback(struct socket *, struct toepcb *, uint32_t,
+    int *);
+extern void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *,
+    void *, unsigned int);
+extern struct mbuf *(*tom_queue_iscsi_callback)(struct socket*, unsigned int,
+    int *);
 #endif
Index: stable/10/sys/modules/cxgbe/Makefile
===================================================================
--- stable/10/sys/modules/cxgbe/Makefile	(revision 270296)
+++ stable/10/sys/modules/cxgbe/Makefile	(revision 270297)
@@ -1,21 +1,21 @@
 #
 # $FreeBSD$
 #
 
-SUBDIR = if_cxgbe
-SUBDIR+= t4_firmware
-SUBDIR+= t5_firmware
-SUBDIR+= ${_tom}
-SUBDIR+= ${_iw_cxgbe}
+SUBDIR=	if_cxgbe
+SUBDIR+=	t4_firmware
+SUBDIR+=	t5_firmware
+SUBDIR+=	${_tom}
+SUBDIR+=	${_iw_cxgbe}
 
 .if ${MACHINE_CPUARCH} == "amd64"
-_tom=		tom
+_tom=	tom
 _iw_cxgbe=	iw_cxgbe
 .endif
 
 .if ${MACHINE_CPUARCH} == "i386"
-_tom=		tom
+_tom=	tom
 .endif
 
 
 .include <bsd.subdir.mk>
Index: stable/10/sys/modules/cxgbe/if_cxgbe/Makefile
===================================================================
--- stable/10/sys/modules/cxgbe/if_cxgbe/Makefile	(revision 270296)
+++ stable/10/sys/modules/cxgbe/if_cxgbe/Makefile	(revision 270297)
@@ -1,35 +1,42 @@
 #
 # $FreeBSD$
 #
 
 .include <bsd.own.mk>
 
-CXGBE = ${.CURDIR}/../../../dev/cxgbe
+CXGBE=	${.CURDIR}/../../../dev/cxgbe
 .PATH: ${CXGBE} ${CXGBE}/common
 
-KMOD = if_cxgbe
-SRCS = t4_main.c t4_sge.c t4_l2t.c t4_tracer.c
-SRCS+= t4_hw.c
-SRCS+= device_if.h bus_if.h pci_if.h
-SRCS+= opt_inet.h opt_inet6.h
-SRCS+= opt_ofed.h
+KMOD=	if_cxgbe
+SRCS=	bus_if.h
+SRCS+=	device_if.h
+SRCS+=	opt_inet.h
+SRCS+=	opt_inet6.h
+SRCS+=	opt_ofed.h
+SRCS+=	pci_if.h
+SRCS+=	t4_hw.c
+SRCS+=	t4_l2t.c
+SRCS+=	t4_main.c
+SRCS+=	t4_netmap.c
+SRCS+=	t4_sge.c
+SRCS+=	t4_tracer.c
 
-CFLAGS+= -I${CXGBE}
-
 # Provide the timestamp of a packet in its header mbuf.
 #CFLAGS+= -DT4_PKT_TIMESTAMP
+
+CFLAGS+= -I${CXGBE}
 
 .if !defined(KERNBUILDDIR)
 .if ${MK_INET_SUPPORT} != "no"
 opt_inet.h:
 	@echo "#define INET 1" > ${.TARGET}
 	@echo "#define TCP_OFFLOAD 1" >> ${.TARGET}
 .endif
 
 .if ${MK_INET6_SUPPORT} != "no"
 opt_inet6.h:
 	@echo "#define INET6 1" > ${.TARGET}
 .endif
 .endif
 
 .include <bsd.kmod.mk>
Index: stable/10/sys/modules/cxgbe/iw_cxgbe/Makefile
===================================================================
--- stable/10/sys/modules/cxgbe/iw_cxgbe/Makefile	(revision 270296)
+++ stable/10/sys/modules/cxgbe/iw_cxgbe/Makefile	(revision 270297)
@@ -1,27 +1,43 @@
 # $FreeBSD$
 
 .include <bsd.own.mk>
 
-CXGBE = ${.CURDIR}/../../../dev/cxgbe
+CXGBE=	${.CURDIR}/../../../dev/cxgbe
 .PATH: ${CXGBE}/iw_cxgbe
 
-KMOD=   iw_cxgbe
-SRCS=   device.c cm.c provider.c mem.c cq.c qp.c resource.c ev.c id_table.c
-SRCS+=  bus_if.h device_if.h opt_sched.h pci_if.h pcib_if.h opt_ktr.h
-SRCS+=  opt_inet.h opt_ofed.h vnode_if.h
+KMOD=	iw_cxgbe
+SRCS=	bus_if.h
+SRCS+=	cm.c
+SRCS+=	cq.c
+SRCS+=	device.c
+SRCS+=	device_if.h
+SRCS+=	ev.c
+SRCS+=	id_table.c
+SRCS+=	mem.c
+SRCS+=	opt_inet.h
+SRCS+=	opt_ktr.h
+SRCS+=	opt_ofed.h
+SRCS+=	opt_sched.h
+SRCS+=	pci_if.h
+SRCS+=	pcib_if.h
+SRCS+=	provider.c
+SRCS+=	qp.c
+SRCS+=	resource.c
+SRCS+=	vnode_if.h
+
 CFLAGS+= -I${CXGBE} -I${.CURDIR}/../../../ofed/include -DLINUX_TYPES_DEFINED
 
 .if !defined(KERNBUILDDIR)
 .if ${MK_INET_SUPPORT} != "no"
 opt_inet.h:
 	@echo "#define INET 1" > ${.TARGET}
 	@echo "#define TCP_OFFLOAD 1" >> ${.TARGET}
 .endif
 
 .if ${MK_INET6_SUPPORT} != "no"
 opt_inet6.h:
 	@echo "#define INET6 1" > ${.TARGET}
 .endif
 .endif
 
 .include <bsd.kmod.mk>
Index: stable/10/sys/modules/cxgbe/t4_firmware/Makefile
===================================================================
--- stable/10/sys/modules/cxgbe/t4_firmware/Makefile	(revision 270296)
+++ stable/10/sys/modules/cxgbe/t4_firmware/Makefile	(revision 270297)
@@ -1,27 +1,27 @@
 #
 # $FreeBSD$
 #
 
-T4FW = ${.CURDIR}/../../../dev/cxgbe/firmware
+T4FW=	${.CURDIR}/../../../dev/cxgbe/firmware
 .PATH: ${T4FW}
 
-KMOD = t4fw_cfg
-FIRMWS = ${KMOD}.txt:${KMOD}:1.0.0.0
+KMOD=	t4fw_cfg
+FIRMWS=	${KMOD}.txt:${KMOD}:1.0.0.0
 
 # You can have additional configuration files in the ${T4FW} directory.
 # t4fw_cfg_<name>.txt
 CFG_FILES != cd ${T4FW} && echo ${KMOD}_*.txt
 .for F in ${CFG_FILES}
 .if exists(${F})
-FIRMWS += ${F}:${F:C/.txt//}:1.0.0.0
+FIRMWS+=	${F}:${F:C/.txt//}:1.0.0.0
 .endif
 .endfor
 
-T4FW_VER = 1.11.27.0
-FIRMWS += t4fw.fw:t4fw:${T4FW_VER}
-CLEANFILES += t4fw.fw
+T4FW_VER=	1.11.27.0
+FIRMWS+=	t4fw.fw:t4fw:${T4FW_VER}
+CLEANFILES+=	t4fw.fw
 
 t4fw.fw: t4fw-${T4FW_VER}.bin.uu
 	uudecode -o ${.TARGET} ${.ALLSRC}
 
 .include <bsd.kmod.mk>
Index: stable/10/sys/modules/cxgbe/t5_firmware/Makefile
===================================================================
--- stable/10/sys/modules/cxgbe/t5_firmware/Makefile	(revision 270296)
+++ stable/10/sys/modules/cxgbe/t5_firmware/Makefile	(revision 270297)
@@ -1,27 +1,27 @@
 #
 # $FreeBSD$
 #
 
-T5FW = ${.CURDIR}/../../../dev/cxgbe/firmware
+T5FW=	${.CURDIR}/../../../dev/cxgbe/firmware
 .PATH: ${T5FW}
 
-KMOD = t5fw_cfg
-FIRMWS = ${KMOD}.txt:${KMOD}:1.0.0.0
+KMOD=	t5fw_cfg
+FIRMWS=	${KMOD}.txt:${KMOD}:1.0.0.0
 
 # You can have additional configuration files in the ${T5FW} directory.
 # t5fw_cfg_<name>.txt
 CFG_FILES != cd ${T5FW} && echo ${KMOD}_*.txt
 .for F in ${CFG_FILES}
 .if exists(${F})
-FIRMWS += ${F}:${F:C/.txt//}:1.0.0.0
+FIRMWS+=	${F}:${F:C/.txt//}:1.0.0.0
 .endif
 .endfor
 
-T5FW_VER = 1.11.27.0
-FIRMWS += t5fw.fw:t5fw:${T5FW_VER}
-CLEANFILES += t5fw.fw
+T5FW_VER=	1.11.27.0
+FIRMWS+=	t5fw.fw:t5fw:${T5FW_VER}
+CLEANFILES+=	t5fw.fw
 
 t5fw.fw: t5fw-${T5FW_VER}.bin.uu
 	uudecode -o ${.TARGET} ${.ALLSRC}
 
 .include <bsd.kmod.mk>
Index: stable/10/sys/modules/cxgbe/tom/Makefile
===================================================================
--- stable/10/sys/modules/cxgbe/tom/Makefile	(revision 270296)
+++ stable/10/sys/modules/cxgbe/tom/Makefile	(revision 270297)
@@ -1,30 +1,38 @@
 #
 # $FreeBSD$
 #
 
 .include <bsd.own.mk>
 
-CXGBE = ${.CURDIR}/../../../dev/cxgbe
+CXGBE=	${.CURDIR}/../../../dev/cxgbe
 .PATH: ${CXGBE}/tom
 
-KMOD = t4_tom
-SRCS = t4_tom.c t4_connect.c t4_listen.c t4_cpl_io.c t4_tom_l2t.c t4_ddp.c
-SRCS+= device_if.h bus_if.h pci_if.h
-SRCS+= opt_inet.h opt_inet6.h
+KMOD=	t4_tom
+SRCS=	bus_if.h
+SRCS+=	device_if.h
+SRCS+=	opt_inet.h
+SRCS+=	opt_inet6.h
+SRCS+=	pci_if.h
+SRCS+=	t4_connect.c
+SRCS+=	t4_cpl_io.c
+SRCS+=	t4_ddp.c
+SRCS+=	t4_listen.c
+SRCS+=	t4_tom.c
+SRCS+=	t4_tom_l2t.c
 
 CFLAGS+= -I${CXGBE}
 
 .if !defined(KERNBUILDDIR)
 .if ${MK_INET_SUPPORT} != "no"
 opt_inet.h:
 	@echo "#define INET 1" > ${.TARGET}
 	@echo "#define TCP_OFFLOAD 1" >> ${.TARGET}
 .endif
 
 .if ${MK_INET6_SUPPORT} != "no"
 opt_inet6.h:
 	@echo "#define INET6 1" > ${.TARGET}
 .endif
 .endif
 
 .include <bsd.kmod.mk>
Index: stable/10
===================================================================
--- stable/10	(revision 270296)
+++ stable/10	(revision 270297)

Property changes on: stable/10
___________________________________________________________________
Modified: svn:mergeinfo
## -0,0 +0,1 ##
   Merged /head:r266571,266757,268536,269076,269364,269366,269411,269413,269428,269440,269537,269644,269731