Index: head/sys/conf/files =================================================================== --- head/sys/conf/files (revision 169977) +++ head/sys/conf/files (revision 169978) @@ -1,2202 +1,2204 @@ # $FreeBSD$ # # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and # dependency lines other than the first are silently ignored. # acpi_quirks.h optional acpi \ dependency "$S/tools/acpi_quirks2h.awk $S/dev/acpica/acpi_quirks" \ compile-with "${AWK} -f $S/tools/acpi_quirks2h.awk $S/dev/acpica/acpi_quirks" \ no-obj no-implicit-rule before-depend \ clean "acpi_quirks.h" aicasm optional ahc | ahd \ dependency "$S/dev/aic7xxx/aicasm/*.[chyl]" \ compile-with "CC='${CC}' ${MAKE} -f $S/dev/aic7xxx/aicasm/Makefile MAKESRCPATH=$S/dev/aic7xxx/aicasm" \ no-obj no-implicit-rule \ clean "aicasm* y.tab.h" aic7xxx_seq.h optional ahc \ compile-with "./aicasm ${INCLUDES} -I$S/cam/scsi -I$S/dev/aic7xxx -o aic7xxx_seq.h -r aic7xxx_reg.h -p aic7xxx_reg_print.c -i $S/dev/aic7xxx/aic7xxx_osm.h $S/dev/aic7xxx/aic7xxx.seq" \ no-obj no-implicit-rule before-depend local \ clean "aic7xxx_seq.h" \ dependency "$S/dev/aic7xxx/aic7xxx.{reg,seq} $S/cam/scsi/scsi_message.h aicasm" aic7xxx_reg.h optional ahc \ compile-with "./aicasm ${INCLUDES} -I$S/cam/scsi -I$S/dev/aic7xxx -o aic7xxx_seq.h -r aic7xxx_reg.h -p aic7xxx_reg_print.c -i $S/dev/aic7xxx/aic7xxx_osm.h $S/dev/aic7xxx/aic7xxx.seq" \ no-obj no-implicit-rule before-depend local \ clean "aic7xxx_reg.h" \ dependency "$S/dev/aic7xxx/aic7xxx.{reg,seq} $S/cam/scsi/scsi_message.h aicasm" aic7xxx_reg_print.c optional ahc \ compile-with "./aicasm ${INCLUDES} -I$S/cam/scsi -I$S/dev/aic7xxx -o aic7xxx_seq.h -r aic7xxx_reg.h -p aic7xxx_reg_print.c -i $S/dev/aic7xxx/aic7xxx_osm.h $S/dev/aic7xxx/aic7xxx.seq" \ no-obj no-implicit-rule local \ clean "aic7xxx_reg_print.c" \ dependency "$S/dev/aic7xxx/aic7xxx.{reg,seq} $S/cam/scsi/scsi_message.h aicasm" aic7xxx_reg_print.o optional ahc ahc_reg_pretty_print \ compile-with "${NORMAL_C}" \ no-implicit-rule local aic79xx_seq.h optional ahd pci \ compile-with "./aicasm ${INCLUDES} -I$S/cam/scsi -I$S/dev/aic7xxx -o aic79xx_seq.h -r aic79xx_reg.h -p aic79xx_reg_print.c -i $S/dev/aic7xxx/aic79xx_osm.h $S/dev/aic7xxx/aic79xx.seq" \ no-obj no-implicit-rule before-depend local \ clean "aic79xx_seq.h" \ dependency "$S/dev/aic7xxx/aic79xx.{reg,seq} $S/cam/scsi/scsi_message.h aicasm" aic79xx_reg.h optional ahd pci \ compile-with "./aicasm ${INCLUDES} -I$S/cam/scsi -I$S/dev/aic7xxx -o aic79xx_seq.h -r aic79xx_reg.h -p aic79xx_reg_print.c -i $S/dev/aic7xxx/aic79xx_osm.h $S/dev/aic7xxx/aic79xx.seq" \ no-obj no-implicit-rule before-depend local \ clean "aic79xx_reg.h" \ dependency "$S/dev/aic7xxx/aic79xx.{reg,seq} $S/cam/scsi/scsi_message.h aicasm" aic79xx_reg_print.c optional ahd pci \ compile-with "./aicasm ${INCLUDES} -I$S/cam/scsi -I$S/dev/aic7xxx -o aic79xx_seq.h -r aic79xx_reg.h -p aic79xx_reg_print.c -i $S/dev/aic7xxx/aic79xx_osm.h $S/dev/aic7xxx/aic79xx.seq" \ no-obj no-implicit-rule local \ clean "aic79xx_reg_print.c" \ dependency "$S/dev/aic7xxx/aic79xx.{reg,seq} $S/cam/scsi/scsi_message.h aicasm" aic79xx_reg_print.o optional ahd pci ahd_reg_pretty_print \ compile-with "${NORMAL_C}" \ no-implicit-rule local emu10k1-alsa%diked.h optional snd_emu10k1 | snd_emu10kx \ dependency "$S/tools/emu10k1-mkalsa.sh $S/gnu/dev/sound/pci/emu10k1-alsa.h" \ compile-with "CC='${CC}' AWK=${AWK} sh $S/tools/emu10k1-mkalsa.sh $S/gnu/dev/sound/pci/emu10k1-alsa.h emu10k1-alsa%diked.h" \ no-obj no-implicit-rule before-depend \ clean "emu10k1-alsa%diked.h" p16v-alsa%diked.h optional snd_emu10kx pci \ dependency "$S/tools/emu10k1-mkalsa.sh $S/gnu/dev/sound/pci/p16v-alsa.h" \ compile-with "CC='${CC}' AWK=${AWK} sh $S/tools/emu10k1-mkalsa.sh $S/gnu/dev/sound/pci/p16v-alsa.h p16v-alsa%diked.h" \ no-obj no-implicit-rule before-depend \ clean "p16v-alsa%diked.h" p17v-alsa%diked.h optional snd_emu10kx pci \ dependency "$S/tools/emu10k1-mkalsa.sh $S/gnu/dev/sound/pci/p17v-alsa.h" \ compile-with "CC='${CC}' AWK=${AWK} sh $S/tools/emu10k1-mkalsa.sh $S/gnu/dev/sound/pci/p17v-alsa.h p17v-alsa%diked.h" \ no-obj no-implicit-rule before-depend \ clean "p17v-alsa%diked.h" miidevs.h optional miibus | mii \ dependency "$S/tools/miidevs2h.awk $S/dev/mii/miidevs" \ compile-with "${AWK} -f $S/tools/miidevs2h.awk $S/dev/mii/miidevs" \ no-obj no-implicit-rule before-depend \ clean "miidevs.h" pccarddevs.h standard \ dependency "$S/tools/pccarddevs2h.awk $S/dev/pccard/pccarddevs" \ compile-with "${AWK} -f $S/tools/pccarddevs2h.awk $S/dev/pccard/pccarddevs" \ no-obj no-implicit-rule before-depend \ clean "pccarddevs.h" usbdevs.h optional usb \ dependency "$S/tools/usbdevs2h.awk $S/dev/usb/usbdevs" \ compile-with "${AWK} -f $S/tools/usbdevs2h.awk $S/dev/usb/usbdevs -h" \ no-obj no-implicit-rule before-depend \ clean "usbdevs.h" usbdevs_data.h optional usb \ dependency "$S/tools/usbdevs2h.awk $S/dev/usb/usbdevs" \ compile-with "${AWK} -f $S/tools/usbdevs2h.awk $S/dev/usb/usbdevs -d" \ no-obj no-implicit-rule before-depend \ clean "usbdevs_data.h" cam/cam.c optional scbus cam/cam_periph.c optional scbus cam/cam_queue.c optional scbus cam/cam_sim.c optional scbus cam/cam_xpt.c optional scbus cam/scsi/scsi_all.c optional scbus cam/scsi/scsi_cd.c optional cd cam/scsi/scsi_ch.c optional ch cam/scsi/scsi_da.c optional da cam/scsi/scsi_low.c optional ct | ncv | nsp | stg cam/scsi/scsi_low_pisa.c optional ct | ncv | nsp | stg cam/scsi/scsi_pass.c optional pass cam/scsi/scsi_pt.c optional pt cam/scsi/scsi_sa.c optional sa cam/scsi/scsi_ses.c optional ses cam/scsi/scsi_sg.c optional sg cam/scsi/scsi_targ_bh.c optional targbh cam/scsi/scsi_target.c optional targ coda/coda_fbsd.c optional vcoda coda/coda_namecache.c optional vcoda coda/coda_psdev.c optional vcoda coda/coda_subr.c optional vcoda coda/coda_venus.c optional vcoda coda/coda_vfsops.c optional vcoda coda/coda_vnops.c optional vcoda contrib/altq/altq/altq_cbq.c optional altq \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/altq/altq/altq_cdnr.c optional altq contrib/altq/altq/altq_hfsc.c optional altq \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/altq/altq/altq_priq.c optional altq \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/altq/altq/altq_red.c optional altq \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/altq/altq/altq_rio.c optional altq \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/altq/altq/altq_rmclass.c optional altq contrib/altq/altq/altq_subr.c optional altq \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/dev/acpica/dbcmds.c optional acpi acpi_debug contrib/dev/acpica/dbdisply.c optional acpi acpi_debug contrib/dev/acpica/dbexec.c optional acpi acpi_debug contrib/dev/acpica/dbfileio.c optional acpi acpi_debug contrib/dev/acpica/dbhistry.c optional acpi acpi_debug contrib/dev/acpica/dbinput.c optional acpi acpi_debug contrib/dev/acpica/dbstats.c optional acpi acpi_debug contrib/dev/acpica/dbutils.c optional acpi acpi_debug contrib/dev/acpica/dbxface.c optional acpi acpi_debug contrib/dev/acpica/dmbuffer.c optional acpi acpi_debug contrib/dev/acpica/dmnames.c optional acpi acpi_debug contrib/dev/acpica/dmopcode.c optional acpi acpi_debug contrib/dev/acpica/dmobject.c optional acpi acpi_debug contrib/dev/acpica/dmresrc.c optional acpi acpi_debug contrib/dev/acpica/dmresrcl.c optional acpi acpi_debug contrib/dev/acpica/dmresrcs.c optional acpi acpi_debug contrib/dev/acpica/dmutils.c optional acpi acpi_debug contrib/dev/acpica/dmwalk.c optional acpi acpi_debug contrib/dev/acpica/dsfield.c optional acpi contrib/dev/acpica/dsinit.c optional acpi contrib/dev/acpica/dsmethod.c optional acpi contrib/dev/acpica/dsmthdat.c optional acpi contrib/dev/acpica/dsobject.c optional acpi contrib/dev/acpica/dsopcode.c optional acpi contrib/dev/acpica/dsutils.c optional acpi contrib/dev/acpica/dswexec.c optional acpi contrib/dev/acpica/dswload.c optional acpi contrib/dev/acpica/dswscope.c optional acpi contrib/dev/acpica/dswstate.c optional acpi contrib/dev/acpica/evevent.c optional acpi contrib/dev/acpica/evgpe.c optional acpi contrib/dev/acpica/evgpeblk.c optional acpi contrib/dev/acpica/evmisc.c optional acpi contrib/dev/acpica/evregion.c optional acpi contrib/dev/acpica/evrgnini.c optional acpi contrib/dev/acpica/evsci.c optional acpi contrib/dev/acpica/evxface.c optional acpi contrib/dev/acpica/evxfevnt.c optional acpi contrib/dev/acpica/evxfregn.c optional acpi contrib/dev/acpica/exconfig.c optional acpi contrib/dev/acpica/exconvrt.c optional acpi contrib/dev/acpica/excreate.c optional acpi contrib/dev/acpica/exdump.c optional acpi contrib/dev/acpica/exfield.c optional acpi contrib/dev/acpica/exfldio.c optional acpi contrib/dev/acpica/exmisc.c optional acpi contrib/dev/acpica/exmutex.c optional acpi contrib/dev/acpica/exnames.c optional acpi contrib/dev/acpica/exoparg1.c optional acpi contrib/dev/acpica/exoparg2.c optional acpi contrib/dev/acpica/exoparg3.c optional acpi contrib/dev/acpica/exoparg6.c optional acpi contrib/dev/acpica/exprep.c optional acpi contrib/dev/acpica/exregion.c optional acpi contrib/dev/acpica/exresnte.c optional acpi contrib/dev/acpica/exresolv.c optional acpi contrib/dev/acpica/exresop.c optional acpi contrib/dev/acpica/exstore.c optional acpi contrib/dev/acpica/exstoren.c optional acpi contrib/dev/acpica/exstorob.c optional acpi contrib/dev/acpica/exsystem.c optional acpi contrib/dev/acpica/exutils.c optional acpi contrib/dev/acpica/hwacpi.c optional acpi contrib/dev/acpica/hwgpe.c optional acpi contrib/dev/acpica/hwregs.c optional acpi contrib/dev/acpica/hwsleep.c optional acpi contrib/dev/acpica/hwtimer.c optional acpi contrib/dev/acpica/nsaccess.c optional acpi contrib/dev/acpica/nsalloc.c optional acpi contrib/dev/acpica/nsdump.c optional acpi contrib/dev/acpica/nseval.c optional acpi contrib/dev/acpica/nsinit.c optional acpi contrib/dev/acpica/nsload.c optional acpi contrib/dev/acpica/nsnames.c optional acpi contrib/dev/acpica/nsobject.c optional acpi contrib/dev/acpica/nsparse.c optional acpi contrib/dev/acpica/nssearch.c optional acpi contrib/dev/acpica/nsutils.c optional acpi contrib/dev/acpica/nswalk.c optional acpi contrib/dev/acpica/nsxfeval.c optional acpi contrib/dev/acpica/nsxfname.c optional acpi contrib/dev/acpica/nsxfobj.c optional acpi contrib/dev/acpica/psargs.c optional acpi contrib/dev/acpica/psloop.c optional acpi contrib/dev/acpica/psopcode.c optional acpi contrib/dev/acpica/psparse.c optional acpi contrib/dev/acpica/psscope.c optional acpi contrib/dev/acpica/pstree.c optional acpi contrib/dev/acpica/psutils.c optional acpi contrib/dev/acpica/pswalk.c optional acpi contrib/dev/acpica/psxface.c optional acpi contrib/dev/acpica/rsaddr.c optional acpi contrib/dev/acpica/rscalc.c optional acpi contrib/dev/acpica/rscreate.c optional acpi contrib/dev/acpica/rsdump.c optional acpi contrib/dev/acpica/rsinfo.c optional acpi contrib/dev/acpica/rsio.c optional acpi contrib/dev/acpica/rsirq.c optional acpi contrib/dev/acpica/rslist.c optional acpi contrib/dev/acpica/rsmemory.c optional acpi contrib/dev/acpica/rsmisc.c optional acpi contrib/dev/acpica/rsutils.c optional acpi contrib/dev/acpica/rsxface.c optional acpi contrib/dev/acpica/tbfadt.c optional acpi contrib/dev/acpica/tbfind.c optional acpi contrib/dev/acpica/tbinstal.c optional acpi contrib/dev/acpica/tbutils.c optional acpi contrib/dev/acpica/tbxface.c optional acpi contrib/dev/acpica/tbxfroot.c optional acpi contrib/dev/acpica/utalloc.c optional acpi contrib/dev/acpica/utcache.c optional acpi contrib/dev/acpica/utclib.c optional acpi contrib/dev/acpica/utcopy.c optional acpi contrib/dev/acpica/utdebug.c optional acpi contrib/dev/acpica/utdelete.c optional acpi contrib/dev/acpica/uteval.c optional acpi contrib/dev/acpica/utglobal.c optional acpi contrib/dev/acpica/utinit.c optional acpi contrib/dev/acpica/utmath.c optional acpi contrib/dev/acpica/utmisc.c optional acpi contrib/dev/acpica/utmutex.c optional acpi contrib/dev/acpica/utobject.c optional acpi contrib/dev/acpica/utresrc.c optional acpi contrib/dev/acpica/utstate.c optional acpi contrib/dev/acpica/utxface.c optional acpi contrib/ipfilter/netinet/fil.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_auth.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_fil_freebsd.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_frag.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_log.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_nat.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_proxy.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_state.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_lookup.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_pool.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_htable.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_sync.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/mlfk_ipl.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ngatm/netnatm/api/cc_conn.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_data.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_dump.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_port.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_sig.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_user.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/unisap.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/misc/straddr.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/misc/unimsg_common.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/msg/traffic.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/msg/uni_ie.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/msg/uni_msg.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/saal/saal_sscfu.c optional ngatm_sscfu \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/saal/saal_sscop.c optional ngatm_sscop \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_call.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_coord.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_party.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_print.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_reset.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_uni.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_unimsgcpy.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_verify.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/pf/net/if_pflog.c optional pflog \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/if_pfsync.c optional pfsync \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/pf.c optional pf \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/pf_if.c optional pf \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/pf_subr.c optional pf \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/pf_ioctl.c optional pf \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/pf_norm.c optional pf \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/pf_table.c optional pf \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/pf_osfp.c optional pf \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/netinet/in4_cksum.c optional pf inet crypto/blowfish/bf_ecb.c optional ipsec ipsec_esp crypto/blowfish/bf_skey.c optional crypto | ipsec ipsec_esp crypto/camellia/camellia.c optional crypto | ipsec ipsec_esp crypto/camellia/camellia-api.c optional crypto | ipsec ipsec_esp crypto/des/des_ecb.c optional crypto | ipsec ipsec_esp | netsmb crypto/des/des_setkey.c optional crypto | ipsec ipsec_esp | netsmb crypto/rc4/rc4.c optional netgraph_mppc_encryption crypto/rijndael/rijndael-alg-fst.c optional crypto | geom_bde | \ ipsec | random | wlan_ccmp crypto/rijndael/rijndael-api-fst.c optional geom_bde | random crypto/rijndael/rijndael-api.c optional crypto | ipsec | wlan_ccmp crypto/sha1.c optional carp | crypto | ipsec | \ netgraph_mppc_encryption | sctp crypto/sha2/sha2.c optional crypto | geom_bde | ipsec | random | \ sctp ddb/db_access.c optional ddb ddb/db_break.c optional ddb ddb/db_command.c optional ddb ddb/db_examine.c optional ddb ddb/db_expr.c optional ddb ddb/db_input.c optional ddb ddb/db_lex.c optional ddb ddb/db_main.c optional ddb ddb/db_output.c optional ddb ddb/db_print.c optional ddb ddb/db_ps.c optional ddb ddb/db_run.c optional ddb ddb/db_sym.c optional ddb ddb/db_thread.c optional ddb ddb/db_variables.c optional ddb ddb/db_watch.c optional ddb ddb/db_write_cmd.c optional ddb #dev/dpt/dpt_control.c optional dpt dev/aac/aac.c optional aac dev/aac/aac_cam.c optional aacp aac dev/aac/aac_debug.c optional aac dev/aac/aac_disk.c optional aac dev/aac/aac_linux.c optional aac compat_linux dev/aac/aac_pci.c optional aac pci dev/acpi_support/acpi_aiboost.c optional acpi_aiboost acpi dev/acpi_support/acpi_asus.c optional acpi_asus acpi dev/acpi_support/acpi_fujitsu.c optional acpi_fujitsu acpi dev/acpi_support/acpi_ibm.c optional acpi_ibm acpi dev/acpi_support/acpi_panasonic.c optional acpi_panasonic acpi dev/acpi_support/acpi_sony.c optional acpi_sony acpi dev/acpi_support/acpi_toshiba.c optional acpi_toshiba acpi dev/acpica/Osd/OsdDebug.c optional acpi dev/acpica/Osd/OsdHardware.c optional acpi dev/acpica/Osd/OsdInterrupt.c optional acpi dev/acpica/Osd/OsdMemory.c optional acpi dev/acpica/Osd/OsdSchedule.c optional acpi dev/acpica/Osd/OsdStream.c optional acpi dev/acpica/Osd/OsdSynch.c optional acpi dev/acpica/Osd/OsdTable.c optional acpi dev/acpica/acpi.c optional acpi dev/acpica/acpi_acad.c optional acpi dev/acpica/acpi_battery.c optional acpi dev/acpica/acpi_button.c optional acpi dev/acpica/acpi_cmbat.c optional acpi dev/acpica/acpi_cpu.c optional acpi dev/acpica/acpi_ec.c optional acpi dev/acpica/acpi_hpet.c optional acpi dev/acpica/acpi_isab.c optional acpi isa dev/acpica/acpi_lid.c optional acpi dev/acpica/acpi_package.c optional acpi dev/acpica/acpi_pci.c optional acpi pci dev/acpica/acpi_pci_link.c optional acpi pci dev/acpica/acpi_pcib.c optional acpi pci dev/acpica/acpi_pcib_acpi.c optional acpi pci dev/acpica/acpi_pcib_pci.c optional acpi pci dev/acpica/acpi_perf.c optional acpi dev/acpica/acpi_powerres.c optional acpi dev/acpica/acpi_quirk.c optional acpi dev/acpica/acpi_resource.c optional acpi dev/acpica/acpi_smbat.c optional acpi dev/acpica/acpi_thermal.c optional acpi dev/acpica/acpi_throttle.c optional acpi dev/acpica/acpi_timer.c optional acpi dev/acpica/acpi_video.c optional acpi_video acpi dev/acpica/acpi_dock.c optional acpi_dock acpi dev/adlink/adlink.c optional adlink dev/advansys/adv_eisa.c optional adv eisa dev/advansys/adv_pci.c optional adv pci dev/advansys/advansys.c optional adv dev/advansys/advlib.c optional adv dev/advansys/advmcode.c optional adv dev/advansys/adw_pci.c optional adw pci dev/advansys/adwcam.c optional adw dev/advansys/adwlib.c optional adw dev/advansys/adwmcode.c optional adw dev/aha/aha.c optional aha dev/aha/aha_isa.c optional aha isa dev/aha/aha_mca.c optional aha mca dev/ahb/ahb.c optional ahb eisa dev/aic/aic.c optional aic dev/aic/aic_pccard.c optional aic pccard dev/aic7xxx/ahc_eisa.c optional ahc eisa dev/aic7xxx/ahc_isa.c optional ahc isa dev/aic7xxx/ahc_pci.c optional ahc pci dev/aic7xxx/ahd_pci.c optional ahd pci dev/aic7xxx/aic7770.c optional ahc dev/aic7xxx/aic79xx.c optional ahd pci dev/aic7xxx/aic79xx_osm.c optional ahd pci dev/aic7xxx/aic79xx_pci.c optional ahd pci dev/aic7xxx/aic7xxx.c optional ahc dev/aic7xxx/aic7xxx_93cx6.c optional ahc dev/aic7xxx/aic7xxx_osm.c optional ahc dev/aic7xxx/aic7xxx_pci.c optional ahc pci dev/amd/amd.c optional amd dev/amr/amr.c optional amr dev/amr/amr_cam.c optional amr dev/amr/amr_disk.c optional amr dev/amr/amr_linux.c optional amr compat_linux dev/amr/amr_pci.c optional amr pci dev/an/if_an.c optional an dev/an/if_an_isa.c optional an isa dev/an/if_an_pccard.c optional an pccard dev/an/if_an_pci.c optional an pci dev/asr/asr.c optional asr pci dev/ata/ata_if.m optional ata dev/ata/ata-all.c optional ata dev/ata/ata-card.c optional ata pccard dev/ata/ata-cbus.c optional ata pc98 dev/ata/ata-chipset.c optional ata pci dev/ata/ata-disk.c optional atadisk dev/ata/ata-dma.c optional ata pci dev/ata/ata-isa.c optional ata isa dev/ata/ata-lowlevel.c optional ata dev/ata/ata-pci.c optional ata pci dev/ata/ata-queue.c optional ata dev/ata/ata-raid.c optional ataraid dev/ata/ata-usb.c optional atausb dev/ata/atapi-cam.c optional atapicam dev/ata/atapi-cd.c optional atapicd dev/ata/atapi-fd.c optional atapifd dev/ata/atapi-tape.c optional atapist dev/ath/ah_osdep.c optional ath_hal \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_rate/amrr/amrr.c optional ath_rate_amrr dev/ath/ath_rate/onoe/onoe.c optional ath_rate_onoe \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_rate/sample/sample.c optional ath_rate_sample \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_pci.c optional ath pci \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/awi/am79c930.c optional awi dev/awi/awi.c optional awi dev/awi/if_awi_pccard.c optional awi pccard dev/bce/if_bce.c optional bce dev/bfe/if_bfe.c optional bfe dev/bge/if_bge.c optional bge dev/bktr/bktr_audio.c optional bktr pci dev/bktr/bktr_card.c optional bktr pci dev/bktr/bktr_core.c optional bktr pci dev/bktr/bktr_i2c.c optional bktr pci smbus dev/bktr/bktr_os.c optional bktr pci dev/bktr/bktr_tuner.c optional bktr pci dev/bktr/msp34xx.c optional bktr pci dev/buslogic/bt.c optional bt dev/buslogic/bt_eisa.c optional bt eisa dev/buslogic/bt_isa.c optional bt isa dev/buslogic/bt_mca.c optional bt mca dev/buslogic/bt_pci.c optional bt pci dev/cardbus/cardbus.c optional cardbus dev/cardbus/cardbus_cis.c optional cardbus dev/cardbus/cardbus_device.c optional cardbus dev/ciss/ciss.c optional ciss dev/cm/smc90cx6.c optional cm dev/cnw/if_cnw.c optional cnw pccard dev/cpufreq/ichss.c optional cpufreq dev/cs/if_cs.c optional cs dev/cs/if_cs_isa.c optional cs isa dev/cs/if_cs_pccard.c optional cs pccard dev/cxgb/cxgb_main.c optional cxgb pci +dev/cxgb/cxgb_offload.c optional cxgb pci +dev/cxgb/cxgb_l2t.c optional cxgb pci dev/cxgb/cxgb_lro.c optional cxgb pci dev/cxgb/cxgb_sge.c optional cxgb pci dev/cxgb/common/cxgb_mc5.c optional cxgb pci dev/cxgb/common/cxgb_vsc8211.c optional cxgb pci dev/cxgb/common/cxgb_ael1002.c optional cxgb pci dev/cxgb/common/cxgb_mv88e1xxx.c optional cxgb pci dev/cxgb/common/cxgb_xgmac.c optional cxgb pci dev/cxgb/common/cxgb_t3_hw.c optional cxgb pci dev/cxgb/sys/uipc_mvec.c optional cxgb pci dev/cy/cy.c optional cy dev/cy/cy_isa.c optional cy isa dev/cy/cy_pci.c optional cy pci dev/dc/if_dc.c optional dc pci dev/dc/dcphy.c optional dc pci dev/dc/pnphy.c optional dc pci dev/dcons/dcons.c optional dcons dev/dcons/dcons_crom.c optional dcons_crom dev/dcons/dcons_os.c optional dcons dev/de/if_de.c optional de pci dev/digi/CX.c optional digi_CX dev/digi/CX_PCI.c optional digi_CX_PCI dev/digi/EPCX.c optional digi_EPCX dev/digi/EPCX_PCI.c optional digi_EPCX_PCI dev/digi/Xe.c optional digi_Xe dev/digi/Xem.c optional digi_Xem dev/digi/Xr.c optional digi_Xr dev/digi/digi.c optional digi dev/digi/digi_isa.c optional digi isa dev/digi/digi_pci.c optional digi pci dev/dpt/dpt_eisa.c optional dpt eisa dev/dpt/dpt_pci.c optional dpt pci dev/dpt/dpt_scsi.c optional dpt dev/drm/ati_pcigart.c optional drm dev/drm/drm_agpsupport.c optional drm dev/drm/drm_auth.c optional drm dev/drm/drm_bufs.c optional drm dev/drm/drm_context.c optional drm dev/drm/drm_dma.c optional drm dev/drm/drm_drawable.c optional drm dev/drm/drm_drv.c optional drm dev/drm/drm_fops.c optional drm dev/drm/drm_ioctl.c optional drm dev/drm/drm_irq.c optional drm dev/drm/drm_lock.c optional drm dev/drm/drm_memory.c optional drm dev/drm/drm_pci.c optional drm dev/drm/drm_scatter.c optional drm dev/drm/drm_sysctl.c optional drm dev/drm/drm_vm.c optional drm dev/drm/i915_dma.c optional i915drm dev/drm/i915_drv.c optional i915drm dev/drm/i915_irq.c optional i915drm dev/drm/i915_mem.c optional i915drm dev/drm/mach64_dma.c optional mach64drm dev/drm/mach64_drv.c optional mach64drm dev/drm/mach64_irq.c optional mach64drm dev/drm/mach64_state.c optional mach64drm dev/drm/mga_dma.c optional mgadrm dev/drm/mga_drv.c optional mgadrm dev/drm/mga_irq.c optional mgadrm dev/drm/mga_state.c optional mgadrm \ compile-with "${NORMAL_C} -finline-limit=13500" dev/drm/mga_warp.c optional mgadrm dev/drm/r128_cce.c optional r128drm dev/drm/r128_drv.c optional r128drm dev/drm/r128_irq.c optional r128drm dev/drm/r128_state.c optional r128drm \ compile-with "${NORMAL_C} -finline-limit=13500" dev/drm/r300_cmdbuf.c optional radeondrm dev/drm/radeon_cp.c optional radeondrm dev/drm/radeon_drv.c optional radeondrm dev/drm/radeon_irq.c optional radeondrm dev/drm/radeon_mem.c optional radeondrm dev/drm/radeon_state.c optional radeondrm dev/drm/savage_bci.c optional savagedrm dev/drm/savage_drv.c optional savagedrm dev/drm/savage_state.c optional savagedrm dev/drm/sis_drv.c optional sisdrm dev/drm/sis_ds.c optional sisdrm dev/drm/sis_mm.c optional sisdrm dev/drm/tdfx_drv.c optional tdfxdrm dev/ed/if_ed.c optional ed dev/ed/if_ed_novell.c optional ed dev/ed/if_ed_rtl80x9.c optional ed dev/ed/if_ed_pccard.c optional ed pccard dev/ed/if_ed_pci.c optional ed pci dev/eisa/eisa_if.m standard dev/eisa/eisaconf.c optional eisa dev/em/if_em.c optional em \ compile-with "${NORMAL_C} -I$S/dev/em" dev/em/e1000_80003es2lan.c optional em \ compile-with "${NORMAL_C} -I$S/dev/em" dev/em/e1000_82540.c optional em \ compile-with "${NORMAL_C} -I$S/dev/em" dev/em/e1000_82541.c optional em \ compile-with "${NORMAL_C} -I$S/dev/em" dev/em/e1000_82542.c optional em \ compile-with "${NORMAL_C} -I$S/dev/em" dev/em/e1000_82543.c optional em \ compile-with "${NORMAL_C} -I$S/dev/em" dev/em/e1000_82571.c optional em \ compile-with "${NORMAL_C} -I$S/dev/em" dev/em/e1000_82575.c optional em \ compile-with "${NORMAL_C} -I$S/dev/em" dev/em/e1000_api.c optional em \ compile-with "${NORMAL_C} -I$S/dev/em" dev/em/e1000_ich8lan.c optional em \ compile-with "${NORMAL_C} -I$S/dev/em" dev/em/e1000_mac.c optional em \ compile-with "${NORMAL_C} -I$S/dev/em" dev/em/e1000_manage.c optional em \ compile-with "${NORMAL_C} -I$S/dev/em" dev/em/e1000_nvm.c optional em \ compile-with "${NORMAL_C} -I$S/dev/em" dev/em/e1000_phy.c optional em \ compile-with "${NORMAL_C} -I$S/dev/em" dev/en/if_en_pci.c optional en pci dev/en/midway.c optional en dev/ep/if_ep.c optional ep dev/ep/if_ep_eisa.c optional ep eisa dev/ep/if_ep_isa.c optional ep isa dev/ep/if_ep_mca.c optional ep mca dev/ep/if_ep_pccard.c optional ep pccard dev/esp/ncr53c9x.c optional esp dev/ex/if_ex.c optional ex dev/ex/if_ex_isa.c optional ex isa dev/ex/if_ex_pccard.c optional ex pccard dev/exca/exca.c optional cbb dev/fatm/if_fatm.c optional fatm pci dev/fb/splash.c optional splash dev/fe/if_fe.c optional fe dev/fe/if_fe_pccard.c optional fe pccard dev/firewire/firewire.c optional firewire dev/firewire/fwcrom.c optional firewire dev/firewire/fwdev.c optional firewire dev/firewire/fwdma.c optional firewire dev/firewire/fwmem.c optional firewire dev/firewire/fwohci.c optional firewire dev/firewire/fwohci_pci.c optional firewire pci dev/firewire/if_fwe.c optional fwe dev/firewire/if_fwip.c optional fwip dev/firewire/sbp.c optional sbp dev/firewire/sbp_targ.c optional sbp_targ dev/flash/at45d.c optional at45d dev/fxp/if_fxp.c optional fxp dev/gem/if_gem.c optional gem dev/gem/if_gem_pci.c optional gem pci dev/harp/if_harp.c optional harp pci dev/hatm/if_hatm.c optional hatm pci dev/hatm/if_hatm_intr.c optional hatm pci dev/hatm/if_hatm_ioctl.c optional hatm pci dev/hatm/if_hatm_rx.c optional hatm pci dev/hatm/if_hatm_tx.c optional hatm pci dev/hfa/fore_buffer.c optional hfa dev/hfa/fore_command.c optional hfa dev/hfa/fore_globals.c optional hfa dev/hfa/fore_if.c optional hfa dev/hfa/fore_init.c optional hfa dev/hfa/fore_intr.c optional hfa dev/hfa/fore_output.c optional hfa dev/hfa/fore_receive.c optional hfa dev/hfa/fore_stats.c optional hfa dev/hfa/fore_timer.c optional hfa dev/hfa/fore_transmit.c optional hfa dev/hfa/fore_vcm.c optional hfa #dev/hfa/hfa_eisa.c optional hfa eisa dev/hfa/hfa_freebsd.c optional hfa dev/hfa/hfa_pci.c optional hfa pci #dev/hfa/hfa_sbus.c optional hfa sbus dev/hifn/hifn7751.c optional hifn dev/hme/if_hme.c optional hme dev/hme/if_hme_pci.c optional hme pci dev/hme/if_hme_sbus.c optional hme sbus dev/hptiop/hptiop.c optional hptiop cam dev/hwpmc/hwpmc_logging.c optional hwpmc dev/hwpmc/hwpmc_mod.c optional hwpmc dev/ichsmb/ichsmb.c optional ichsmb dev/ichsmb/ichsmb_pci.c optional ichsmb pci dev/ida/ida.c optional ida dev/ida/ida_disk.c optional ida dev/ida/ida_eisa.c optional ida eisa dev/ida/ida_pci.c optional ida pci dev/ie/if_ie.c optional ie isa nowerror dev/ie/if_ie_isa.c optional ie isa dev/ieee488/ibfoo.c optional pcii | tnt4882 dev/ieee488/pcii.c optional pcii dev/ieee488/tnt4882.c optional tnt4882 dev/ieee488/upd7210.c optional pcii | tnt4882 dev/iicbus/ad7418.c optional ad7418 dev/iicbus/ds1672.c optional ds1672 dev/iicbus/icee.c optional icee dev/iicbus/if_ic.c optional ic dev/iicbus/iic.c optional iic dev/iicbus/iicbb.c optional iicbb dev/iicbus/iicbb_if.m optional iicbb dev/iicbus/iicbus.c optional iicbus dev/iicbus/iicbus_if.m optional iicbus dev/iicbus/iiconf.c optional iicbus dev/iicbus/iicsmb.c optional iicsmb \ dependency "iicbus_if.h" dev/iir/iir.c optional iir dev/iir/iir_ctrl.c optional iir dev/iir/iir_pci.c optional iir pci dev/ips/ips.c optional ips dev/ips/ips_commands.c optional ips dev/ips/ips_disk.c optional ips dev/ips/ips_ioctl.c optional ips dev/ips/ips_pci.c optional ips pci dev/ipw/if_ipw.c optional ipw dev/isp/isp.c optional isp dev/isp/isp_freebsd.c optional isp dev/isp/isp_library.c optional isp dev/isp/isp_pci.c optional isp pci dev/isp/isp_sbus.c optional isp sbus dev/isp/isp_target.c optional isp dev/ispfw/ispfw.c optional ispfw dev/iwi/if_iwi.c optional iwi dev/ixgb/if_ixgb.c optional ixgb dev/ixgb/ixgb_ee.c optional ixgb dev/ixgb/ixgb_hw.c optional ixgb dev/joy/joy.c optional joy dev/joy/joy_isa.c optional joy isa dev/joy/joy_pccard.c optional joy pccard dev/kbdmux/kbdmux.c optional kbdmux dev/le/am7990.c optional le dev/le/am79900.c optional le dev/le/if_le_pci.c optional le pci dev/le/lance.c optional le dev/led/led.c standard dev/lge/if_lge.c optional lge dev/lmc/if_lmc.c optional lmc dev/mc146818/mc146818.c optional mc146818 dev/mca/mca_bus.c optional mca dev/mcd/mcd.c optional mcd isa nowerror dev/mcd/mcd_isa.c optional mcd isa nowerror dev/md/md.c optional md dev/mem/memdev.c optional mem dev/mfi/mfi.c optional mfi dev/mfi/mfi_debug.c optional mfi dev/mfi/mfi_pci.c optional mfi pci dev/mfi/mfi_disk.c optional mfi dev/mfi/mfi_linux.c optional mfi compat_linux dev/mfi/mfi_cam.c optional mfip scbus dev/mii/acphy.c optional miibus | acphy dev/mii/amphy.c optional miibus | amphy dev/mii/bmtphy.c optional miibus | bmtphy dev/mii/brgphy.c optional miibus | brgphy dev/mii/ciphy.c optional miibus | ciphy dev/mii/e1000phy.c optional miibus | e1000phy # XXX only xl cards? dev/mii/exphy.c optional miibus | exphy dev/mii/gentbi.c optional miibus | gentbi # XXX only fxp cards? dev/mii/inphy.c optional miibus | inphy dev/mii/ip1000phy.c optional miibus | ip1000phy dev/mii/lxtphy.c optional miibus | lxtphy dev/mii/mii.c optional miibus | mii dev/mii/mii_physubr.c optional miibus | mii dev/mii/miibus_if.m optional miibus | mii dev/mii/mlphy.c optional miibus | mlphy dev/mii/nsgphy.c optional miibus | nsgphy dev/mii/nsphy.c optional miibus | nsphy dev/mii/pnaphy.c optional miibus | pnaphy dev/mii/qsphy.c optional miibus | qsphy dev/mii/rgephy.c optional miibus | rgephy dev/mii/rlphy.c optional miibus | rlphy dev/mii/rlswitch.c optional rlswitch # XXX rue only? dev/mii/ruephy.c optional miibus | ruephy dev/mii/tdkphy.c optional miibus | tdkphy dev/mii/tlphy.c optional miibus | tlphy dev/mii/ukphy.c optional miibus | mii dev/mii/ukphy_subr.c optional miibus | mii dev/mii/xmphy.c optional miibus | xmphy dev/mk48txx/mk48txx.c optional mk48txx dev/mlx/mlx.c optional mlx dev/mlx/mlx_disk.c optional mlx dev/mlx/mlx_pci.c optional mlx pci dev/mly/mly.c optional mly dev/mmc/mmc.c optional mmc dev/mmc/mmcbr_if.m standard dev/mmc/mmcbus_if.m standard dev/mmc/mmcsd.c optional mmcsd dev/mpt/mpt.c optional mpt dev/mpt/mpt_cam.c optional mpt dev/mpt/mpt_debug.c optional mpt dev/mpt/mpt_pci.c optional mpt pci dev/mpt/mpt_raid.c optional mpt dev/msk/if_msk.c optional msk dev/my/if_my.c optional my dev/ncv/ncr53c500.c optional ncv dev/ncv/ncr53c500_pccard.c optional ncv pccard dev/nge/if_nge.c optional nge dev/nmdm/nmdm.c optional nmdm dev/nsp/nsp.c optional nsp dev/nsp/nsp_pccard.c optional nsp pccard dev/null/null.c standard dev/patm/if_patm.c optional patm pci dev/patm/if_patm_attach.c optional patm pci dev/patm/if_patm_intr.c optional patm pci dev/patm/if_patm_ioctl.c optional patm pci dev/patm/if_patm_rtables.c optional patm pci dev/patm/if_patm_rx.c optional patm pci dev/patm/if_patm_tx.c optional patm pci dev/pbio/pbio.c optional pbio isa dev/pccard/card_if.m standard dev/pccard/pccard.c optional pccard dev/pccard/pccard_cis.c optional pccard dev/pccard/pccard_cis_quirks.c optional pccard dev/pccard/pccard_device.c optional pccard dev/pccard/power_if.m standard dev/pccbb/pccbb.c optional cbb dev/pccbb/pccbb_isa.c optional cbb isa dev/pccbb/pccbb_pci.c optional cbb pci dev/pcf/pcf.c optional pcf dev/pci/eisa_pci.c optional pci eisa dev/pci/fixup_pci.c optional pci dev/pci/hostb_pci.c optional pci dev/pci/ignore_pci.c optional pci dev/pci/isa_pci.c optional pci isa dev/pci/pci.c optional pci dev/pci/pci_if.m standard dev/pci/pci_pci.c optional pci dev/pci/pci_user.c optional pci dev/pci/pcib_if.m standard dev/pci/vga_pci.c optional pci dev/pdq/if_fea.c optional fea eisa dev/pdq/if_fpa.c optional fpa pci dev/pdq/pdq.c optional nowerror fea eisa | fpa pci dev/pdq/pdq_ifsubr.c optional nowerror fea eisa | fpa pci dev/ppbus/if_plip.c optional plip dev/ppbus/immio.c optional vpo dev/ppbus/lpbb.c optional lpbb dev/ppbus/lpt.c optional lpt dev/ppbus/pcfclock.c optional pcfclock dev/ppbus/ppb_1284.c optional ppbus dev/ppbus/ppb_base.c optional ppbus dev/ppbus/ppb_msq.c optional ppbus dev/ppbus/ppbconf.c optional ppbus dev/ppbus/ppbus_if.m optional ppbus dev/ppbus/ppi.c optional ppi dev/ppbus/pps.c optional pps dev/ppbus/vpo.c optional vpo dev/ppbus/vpoio.c optional vpo dev/ppc/ppc.c optional ppc dev/ppc/ppc_acpi.c optional ppc acpi dev/ppc/ppc_isa.c optional ppc isa dev/ppc/ppc_pci.c optional ppc pci dev/ppc/ppc_puc.c optional ppc puc dev/pst/pst-iop.c optional pst dev/pst/pst-pci.c optional pst pci dev/pst/pst-raid.c optional pst dev/puc/puc.c optional puc dev/puc/puc_cfg.c optional puc dev/puc/puc_pccard.c optional puc pccard dev/puc/puc_pci.c optional puc pci dev/puc/pucdata.c optional puc pci dev/ral/rt2560.c optional ral dev/ral/rt2661.c optional ral dev/ral/if_ralrate.c optional ral dev/ral/if_ral_pci.c optional ral pci dev/random/harvest.c standard dev/random/hash.c optional random dev/random/probe.c optional random dev/random/randomdev.c optional random dev/random/randomdev_soft.c optional random dev/random/yarrow.c optional random dev/ray/if_ray.c optional ray pccard dev/rc/rc.c optional rc dev/re/if_re.c optional re dev/rndtest/rndtest.c optional rndtest dev/rp/rp.c optional rp dev/rp/rp_isa.c optional rp isa dev/rp/rp_pci.c optional rp pci dev/safe/safe.c optional safe dev/sbsh/if_sbsh.c optional sbsh dev/scc/scc_if.m optional scc dev/scc/scc_bfe_ebus.c optional scc ebus dev/scc/scc_bfe_sbus.c optional scc fhc | scc sbus dev/scc/scc_core.c optional scc dev/scc/scc_dev_sab82532.c optional scc dev/scc/scc_dev_z8530.c optional scc dev/scd/scd.c optional scd isa dev/scd/scd_isa.c optional scd isa dev/si/si.c optional si dev/si/si2_z280.c optional si dev/si/si3_t225.c optional si dev/si/si_eisa.c optional si eisa dev/si/si_isa.c optional si isa dev/si/si_pci.c optional si pci dev/sk/if_sk.c optional sk pci dev/smbus/smb.c optional smb dev/smbus/smbconf.c optional smbus dev/smbus/smbus.c optional smbus dev/smbus/smbus_if.m optional smbus dev/sn/if_sn.c optional sn dev/sn/if_sn_isa.c optional sn isa dev/sn/if_sn_pccard.c optional sn pccard dev/snp/snp.c optional snp dev/sound/isa/ad1816.c optional snd_ad1816 isa dev/sound/isa/ess.c optional snd_ess isa dev/sound/isa/gusc.c optional snd_gusc isa dev/sound/isa/mss.c optional snd_mss isa dev/sound/isa/sb16.c optional snd_sb16 isa dev/sound/isa/sb8.c optional snd_sb8 isa dev/sound/isa/sbc.c optional snd_sbc isa dev/sound/isa/sndbuf_dma.c optional sound isa dev/sound/pci/als4000.c optional snd_als4000 pci dev/sound/pci/atiixp.c optional snd_atiixp pci #dev/sound/pci/au88x0.c optional snd_au88x0 pci dev/sound/pci/cmi.c optional snd_cmi pci dev/sound/pci/cs4281.c optional snd_cs4281 pci dev/sound/pci/csa.c optional snd_csa pci \ warning "kernel contains GPL contaminated csaimg.h header" dev/sound/pci/csapcm.c optional snd_csa pci dev/sound/pci/ds1.c optional snd_ds1 pci dev/sound/pci/emu10k1.c optional snd_emu10k1 pci \ dependency "emu10k1-alsa%diked.h" \ warning "kernel contains GPL contaminated emu10k1 headers" dev/sound/pci/emu10kx.c optional snd_emu10kx pci \ dependency "emu10k1-alsa%diked.h" \ dependency "p16v-alsa%diked.h" \ dependency "p17v-alsa%diked.h" \ warning "kernel contains GPL contaminated emu10kx headers" dev/sound/pci/emu10kx-pcm.c optional snd_emu10kx pci \ dependency "emu10k1-alsa%diked.h" \ dependency "p16v-alsa%diked.h" \ dependency "p17v-alsa%diked.h" \ warning "kernel contains GPL contaminated emu10kx headers" dev/sound/pci/emu10kx-midi.c optional snd_emu10kx pci \ dependency "emu10k1-alsa%diked.h" \ warning "kernel contains GPL contaminated emu10kx headers" dev/sound/pci/envy24.c optional snd_envy24 pci dev/sound/pci/envy24ht.c optional snd_envy24ht pci dev/sound/pci/es137x.c optional snd_es137x pci dev/sound/pci/fm801.c optional snd_fm801 pci dev/sound/pci/ich.c optional snd_ich pci dev/sound/pci/maestro.c optional snd_maestro pci dev/sound/pci/maestro3.c optional snd_maestro3 pci \ warning "kernel contains GPL contaminated maestro3 headers" dev/sound/pci/neomagic.c optional snd_neomagic pci dev/sound/pci/solo.c optional snd_solo pci dev/sound/pci/spicds.c optional snd_spicds pci dev/sound/pci/t4dwave.c optional snd_t4dwave pci dev/sound/pci/via8233.c optional snd_via8233 pci dev/sound/pci/via82c686.c optional snd_via82c686 pci dev/sound/pci/vibes.c optional snd_vibes pci dev/sound/pci/hda/hdac.c optional snd_hda pci dev/sound/pcm/ac97.c optional sound dev/sound/pcm/ac97_if.m optional sound dev/sound/pcm/ac97_patch.c optional sound dev/sound/pcm/buffer.c optional sound dev/sound/pcm/channel.c optional sound dev/sound/pcm/channel_if.m optional sound dev/sound/pcm/dsp.c optional sound dev/sound/pcm/fake.c optional sound dev/sound/pcm/feeder.c optional sound dev/sound/pcm/feeder_fmt.c optional sound dev/sound/pcm/feeder_if.m optional sound dev/sound/pcm/feeder_rate.c optional sound dev/sound/pcm/feeder_volume.c optional sound dev/sound/pcm/mixer.c optional sound dev/sound/pcm/mixer_if.m optional sound dev/sound/pcm/sndstat.c optional sound dev/sound/pcm/sound.c optional sound dev/sound/pcm/vchan.c optional sound #dev/sound/usb/upcm.c optional snd_upcm usb dev/sound/usb/uaudio.c optional snd_uaudio usb dev/sound/usb/uaudio_pcm.c optional snd_uaudio usb dev/sound/midi/midi.c optional sound dev/sound/midi/mpu401.c optional sound dev/sound/midi/mpu_if.m optional sound dev/sound/midi/mpufoi_if.m optional sound dev/sound/midi/sequencer.c optional sound dev/sound/midi/synth_if.m optional sound dev/spibus/spibus.c optional spibus \ dependency "spibus_if.h" dev/spibus/spibus_if.m optional spibus dev/sr/if_sr.c optional sr dev/sr/if_sr_pci.c optional sr pci dev/stg/tmc18c30.c optional stg dev/stg/tmc18c30_isa.c optional stg isa dev/stg/tmc18c30_pccard.c optional stg pccard dev/stg/tmc18c30_pci.c optional stg pci dev/stg/tmc18c30_subr.c optional stg dev/stge/if_stge.c optional stge dev/streams/streams.c optional streams dev/sym/sym_hipd.c optional sym \ dependency "$S/dev/sym/sym_{conf,defs}.h" dev/syscons/blank/blank_saver.c optional blank_saver dev/syscons/daemon/daemon_saver.c optional daemon_saver dev/syscons/dragon/dragon_saver.c optional dragon_saver dev/syscons/fade/fade_saver.c optional fade_saver dev/syscons/fire/fire_saver.c optional fire_saver dev/syscons/green/green_saver.c optional green_saver dev/syscons/logo/logo.c optional logo_saver dev/syscons/logo/logo_saver.c optional logo_saver dev/syscons/rain/rain_saver.c optional rain_saver dev/syscons/schistory.c optional sc dev/syscons/scmouse.c optional sc dev/syscons/scterm-dumb.c optional sc dev/syscons/scterm.c optional sc dev/syscons/scvidctl.c optional sc dev/syscons/snake/snake_saver.c optional snake_saver dev/syscons/star/star_saver.c optional star_saver dev/syscons/syscons.c optional sc dev/syscons/sysmouse.c optional sc dev/syscons/warp/warp_saver.c optional warp_saver dev/tdfx/tdfx_linux.c optional tdfx_linux tdfx compat_linux dev/tdfx/tdfx_pci.c optional tdfx pci dev/ti/if_ti.c optional ti pci dev/trm/trm.c optional trm dev/twa/tw_cl_init.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_cl_intr.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_cl_io.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_cl_misc.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_osl_cam.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_osl_freebsd.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twe/twe.c optional twe dev/twe/twe_freebsd.c optional twe dev/tx/if_tx.c optional tx dev/txp/if_txp.c optional txp dev/uart/uart_bus_acpi.c optional uart acpi #dev/uart/uart_bus_cbus.c optional uart cbus dev/uart/uart_bus_ebus.c optional uart ebus dev/uart/uart_bus_isa.c optional uart isa dev/uart/uart_bus_pccard.c optional uart pccard dev/uart/uart_bus_pci.c optional uart pci dev/uart/uart_bus_puc.c optional uart puc dev/uart/uart_bus_scc.c optional uart scc dev/uart/uart_core.c optional uart dev/uart/uart_dbg.c optional uart gdb dev/uart/uart_dev_ns8250.c optional uart uart_ns8250 dev/uart/uart_dev_sab82532.c optional uart uart_sab82532 dev/uart/uart_dev_sab82532.c optional uart scc dev/uart/uart_dev_z8530.c optional uart uart_z8530 dev/uart/uart_dev_z8530.c optional uart scc dev/uart/uart_if.m optional uart dev/uart/uart_subr.c optional uart dev/uart/uart_tty.c optional uart dev/ubsec/ubsec.c optional ubsec # # USB support dev/usb/ehci.c optional ehci dev/usb/ehci_pci.c optional ehci pci dev/usb/hid.c optional usb dev/usb/if_aue.c optional aue dev/usb/if_axe.c optional axe dev/usb/if_cdce.c optional cdce dev/usb/if_cue.c optional cue dev/usb/if_kue.c optional kue dev/usb/if_ural.c optional ural dev/usb/if_rue.c optional rue dev/usb/if_rum.c optional rum dev/usb/if_udav.c optional udav dev/usb/ohci.c optional ohci dev/usb/ohci_pci.c optional ohci pci dev/usb/sl811hs.c optional slhci dev/usb/slhci_pccard.c optional slhci pccard dev/usb/uark.c optional uark dev/usb/ubsa.c optional ubsa dev/usb/ubser.c optional ubser dev/usb/ucom.c optional ucom dev/usb/ucycom.c optional ucycom dev/usb/udbp.c optional udbp dev/usb/ufoma.c optional ufoma dev/usb/ufm.c optional ufm dev/usb/uftdi.c optional uftdi dev/usb/ugen.c optional ugen dev/usb/uhci.c optional uhci dev/usb/uhci_pci.c optional uhci pci dev/usb/uhid.c optional uhid dev/usb/uhub.c optional usb dev/usb/uipaq.c optional uipaq dev/usb/ukbd.c optional ukbd dev/usb/ulpt.c optional ulpt dev/usb/umass.c optional umass dev/usb/umct.c optional umct dev/usb/umodem.c optional umodem dev/usb/ums.c optional ums dev/usb/uplcom.c optional uplcom dev/usb/urio.c optional urio dev/usb/usb.c optional usb dev/usb/usb_ethersubr.c optional usb dev/usb/usb_if.m optional usb dev/usb/usb_mem.c optional usb dev/usb/usb_quirks.c optional usb dev/usb/usb_subr.c optional usb dev/usb/usbdi.c optional usb dev/usb/usbdi_util.c optional usb dev/usb/uscanner.c optional uscanner dev/usb/uvisor.c optional uvisor dev/usb/uvscom.c optional uvscom dev/utopia/idtphy.c optional utopia dev/utopia/suni.c optional utopia dev/utopia/utopia.c optional utopia dev/vge/if_vge.c optional vge dev/vkbd/vkbd.c optional vkbd dev/vx/if_vx.c optional vx dev/vx/if_vx_eisa.c optional vx eisa dev/vx/if_vx_pci.c optional vx pci dev/watchdog/watchdog.c standard dev/wds/wd7000.c optional wds isa dev/wi/if_wi.c optional wi dev/wi/if_wi_pccard.c optional wi pccard dev/wi/if_wi_pci.c optional wi pci dev/wl/if_wl.c optional wl isa dev/xe/if_xe.c optional xe dev/xe/if_xe_pccard.c optional xe pccard fs/deadfs/dead_vnops.c standard fs/devfs/devfs_devs.c standard fs/devfs/devfs_rule.c standard fs/devfs/devfs_vfsops.c standard fs/devfs/devfs_vnops.c standard fs/fdescfs/fdesc_vfsops.c optional fdescfs fs/fdescfs/fdesc_vnops.c optional fdescfs fs/fifofs/fifo_vnops.c standard fs/hpfs/hpfs_alsubr.c optional hpfs fs/hpfs/hpfs_lookup.c optional hpfs fs/hpfs/hpfs_subr.c optional hpfs fs/hpfs/hpfs_vfsops.c optional hpfs fs/hpfs/hpfs_vnops.c optional hpfs fs/msdosfs/msdosfs_conv.c optional msdosfs fs/msdosfs/msdosfs_denode.c optional msdosfs fs/msdosfs/msdosfs_fat.c optional msdosfs fs/msdosfs/msdosfs_fileno.c optional msdosfs fs/msdosfs/msdosfs_iconv.c optional msdosfs_iconv fs/msdosfs/msdosfs_lookup.c optional msdosfs fs/msdosfs/msdosfs_vfsops.c optional msdosfs fs/msdosfs/msdosfs_vnops.c optional msdosfs fs/ntfs/ntfs_compr.c optional ntfs fs/ntfs/ntfs_iconv.c optional ntfs_iconv fs/ntfs/ntfs_ihash.c optional ntfs fs/ntfs/ntfs_subr.c optional ntfs fs/ntfs/ntfs_vfsops.c optional ntfs fs/ntfs/ntfs_vnops.c optional ntfs fs/nullfs/null_subr.c optional nullfs fs/nullfs/null_vfsops.c optional nullfs fs/nullfs/null_vnops.c optional nullfs fs/nwfs/nwfs_io.c optional nwfs fs/nwfs/nwfs_ioctl.c optional nwfs fs/nwfs/nwfs_node.c optional nwfs fs/nwfs/nwfs_subr.c optional nwfs fs/nwfs/nwfs_vfsops.c optional nwfs fs/nwfs/nwfs_vnops.c optional nwfs fs/portalfs/portal_vfsops.c optional portalfs fs/portalfs/portal_vnops.c optional portalfs fs/procfs/procfs.c optional procfs fs/procfs/procfs_ctl.c optional procfs fs/procfs/procfs_dbregs.c optional procfs fs/procfs/procfs_fpregs.c optional procfs fs/procfs/procfs_ioctl.c optional procfs fs/procfs/procfs_map.c optional procfs fs/procfs/procfs_mem.c optional procfs fs/procfs/procfs_note.c optional procfs fs/procfs/procfs_regs.c optional procfs fs/procfs/procfs_rlimit.c optional procfs fs/procfs/procfs_status.c optional procfs fs/procfs/procfs_type.c optional procfs fs/pseudofs/pseudofs.c optional pseudofs fs/pseudofs/pseudofs_fileno.c optional pseudofs fs/pseudofs/pseudofs_vncache.c optional pseudofs fs/pseudofs/pseudofs_vnops.c optional pseudofs fs/smbfs/smbfs_io.c optional smbfs fs/smbfs/smbfs_node.c optional smbfs fs/smbfs/smbfs_smb.c optional smbfs fs/smbfs/smbfs_subr.c optional smbfs fs/smbfs/smbfs_vfsops.c optional smbfs fs/smbfs/smbfs_vnops.c optional smbfs fs/udf/osta.c optional udf fs/udf/udf_iconv.c optional udf_iconv fs/udf/udf_vfsops.c optional udf fs/udf/udf_vnops.c optional udf fs/umapfs/umap_subr.c optional umapfs fs/umapfs/umap_vfsops.c optional umapfs fs/umapfs/umap_vnops.c optional umapfs fs/unionfs/union_subr.c optional unionfs fs/unionfs/union_vfsops.c optional unionfs fs/unionfs/union_vnops.c optional unionfs gdb/gdb_cons.c optional gdb gdb/gdb_main.c optional gdb gdb/gdb_packet.c optional gdb geom/bde/g_bde.c optional geom_bde geom/bde/g_bde_crypt.c optional geom_bde geom/bde/g_bde_lock.c optional geom_bde geom/bde/g_bde_work.c optional geom_bde geom/cache/g_cache.c optional geom_cache geom/concat/g_concat.c optional geom_concat geom/eli/g_eli.c optional geom_eli geom/eli/g_eli_crypto.c optional geom_eli geom/eli/g_eli_ctl.c optional geom_eli geom/eli/g_eli_integrity.c optional geom_eli geom/eli/g_eli_key.c optional geom_eli geom/eli/g_eli_privacy.c optional geom_eli geom/eli/pkcs5v2.c optional geom_eli geom/gate/g_gate.c optional geom_gate geom/geom_aes.c optional geom_aes geom/geom_bsd.c optional geom_bsd geom/geom_bsd_enc.c optional geom_bsd geom/geom_ccd.c optional ccd | geom_ccd geom/geom_ctl.c standard geom/geom_dev.c standard geom/geom_disk.c standard geom/geom_dump.c standard geom/geom_event.c standard geom/geom_fox.c optional geom_fox geom/geom_io.c standard geom/geom_kern.c standard geom/geom_mbr.c optional geom_mbr geom/geom_mbr_enc.c optional geom_mbr geom/geom_pc98.c optional geom_pc98 geom/geom_pc98_enc.c optional geom_pc98 geom/geom_slice.c standard geom/geom_subr.c standard geom/geom_sunlabel.c optional geom_sunlabel geom/geom_sunlabel_enc.c optional geom_sunlabel geom/geom_vfs.c standard geom/geom_vol_ffs.c optional geom_vol geom/journal/g_journal.c optional geom_journal geom/journal/g_journal_ufs.c optional geom_journal geom/label/g_label.c optional geom_label geom/label/g_label_ext2fs.c optional geom_label geom/label/g_label_iso9660.c optional geom_label geom/label/g_label_msdosfs.c optional geom_label geom/label/g_label_ntfs.c optional geom_label geom/label/g_label_reiserfs.c optional geom_label geom/label/g_label_ufs.c optional geom_label geom/mirror/g_mirror.c optional geom_mirror geom/mirror/g_mirror_ctl.c optional geom_mirror geom/multipath/g_multipath.c optional geom_multipath geom/nop/g_nop.c optional geom_nop geom/part/g_part.c standard geom/part/g_part_if.m standard geom/part/g_part_apm.c optional geom_part_apm geom/part/g_part_gpt.c optional geom_part_gpt geom/raid3/g_raid3.c optional geom_raid3 geom/raid3/g_raid3_ctl.c optional geom_raid3 geom/shsec/g_shsec.c optional geom_shsec geom/stripe/g_stripe.c optional geom_stripe geom/uzip/g_uzip.c optional geom_uzip geom/zero/g_zero.c optional geom_zero gnu/fs/ext2fs/ext2_alloc.c optional ext2fs \ warning "kernel contains GPL contaminated ext2fs filesystem" gnu/fs/ext2fs/ext2_balloc.c optional ext2fs gnu/fs/ext2fs/ext2_bmap.c optional ext2fs gnu/fs/ext2fs/ext2_inode.c optional ext2fs gnu/fs/ext2fs/ext2_inode_cnv.c optional ext2fs gnu/fs/ext2fs/ext2_linux_balloc.c optional ext2fs gnu/fs/ext2fs/ext2_linux_ialloc.c optional ext2fs gnu/fs/ext2fs/ext2_lookup.c optional ext2fs gnu/fs/ext2fs/ext2_subr.c optional ext2fs gnu/fs/ext2fs/ext2_vfsops.c optional ext2fs gnu/fs/ext2fs/ext2_vnops.c optional ext2fs gnu/fs/reiserfs/reiserfs_hashes.c optional reiserfs \ warning "kernel contains GPL contaminated ReiserFS filesystem" gnu/fs/reiserfs/reiserfs_inode.c optional reiserfs gnu/fs/reiserfs/reiserfs_item_ops.c optional reiserfs gnu/fs/reiserfs/reiserfs_namei.c optional reiserfs gnu/fs/reiserfs/reiserfs_prints.c optional reiserfs gnu/fs/reiserfs/reiserfs_stree.c optional reiserfs gnu/fs/reiserfs/reiserfs_vfsops.c optional reiserfs gnu/fs/reiserfs/reiserfs_vnops.c optional reiserfs # # isdn4bsd device drivers # i4b/driver/i4b_trace.c optional i4btrc i4b/driver/i4b_rbch.c optional i4brbch i4b/driver/i4b_tel.c optional i4btel i4b/driver/i4b_ipr.c optional i4bipr net/slcompress.c optional i4bipr | i4bisppp i4b/driver/i4b_ctl.c optional i4bctl i4b/driver/i4b_ing.c optional i4bing i4b/driver/i4b_isppp.c optional i4bisppp # # isdn4bsd CAPI driver # i4b/capi/capi_l4if.c optional i4bcapi i4b/capi/capi_llif.c optional i4bcapi i4b/capi/capi_msgs.c optional i4bcapi # # isdn4bsd AVM B1/T1 CAPI driver # i4b/capi/iavc/iavc_pci.c optional iavc i4bcapi pci i4b/capi/iavc/iavc_isa.c optional iavc i4bcapi isa i4b/capi/iavc/iavc_lli.c optional iavc i4bcapi i4b/capi/iavc/iavc_card.c optional iavc i4bcapi # # isdn4bsd support # i4b/layer2/i4b_mbuf.c optional i4btrc # # isdn4bsd Q.921 handler # i4b/layer2/i4b_l2.c optional i4bq921 i4b/layer2/i4b_l2fsm.c optional i4bq921 i4b/layer2/i4b_uframe.c optional i4bq921 i4b/layer2/i4b_tei.c optional i4bq921 i4b/layer2/i4b_sframe.c optional i4bq921 i4b/layer2/i4b_iframe.c optional i4bq921 i4b/layer2/i4b_l2timer.c optional i4bq921 i4b/layer2/i4b_util.c optional i4bq921 i4b/layer2/i4b_lme.c optional i4bq921 # # isdn4bsd Q.931 handler # i4b/layer3/i4b_q931.c optional i4bq931 i4b/layer3/i4b_l3fsm.c optional i4bq931 i4b/layer3/i4b_l3timer.c optional i4bq931 i4b/layer3/i4b_l2if.c optional i4bq931 i4b/layer3/i4b_l4if.c optional i4bq931 i4b/layer3/i4b_q932fac.c optional i4bq931 # # isdn4bsd control device driver, interface to isdnd # i4b/layer4/i4b_i4bdrv.c optional i4b i4b/layer4/i4b_l4.c optional i4b i4b/layer4/i4b_l4mgmt.c optional i4b i4b/layer4/i4b_l4timer.c optional i4b # isa/isa_if.m standard isa/isa_common.c optional isa isa/isahint.c optional isa isa/orm.c optional isa isa/pnp.c optional isa isapnp isa/pnpparse.c optional isa isapnp fs/cd9660/cd9660_bmap.c optional cd9660 fs/cd9660/cd9660_lookup.c optional cd9660 fs/cd9660/cd9660_node.c optional cd9660 fs/cd9660/cd9660_rrip.c optional cd9660 fs/cd9660/cd9660_util.c optional cd9660 fs/cd9660/cd9660_vfsops.c optional cd9660 fs/cd9660/cd9660_vnops.c optional cd9660 fs/cd9660/cd9660_iconv.c optional cd9660_iconv kern/bus_if.m standard kern/clock_if.m optional genclock kern/cpufreq_if.m standard kern/device_if.m standard kern/imgact_elf.c standard kern/imgact_shell.c standard kern/inflate.c optional gzip kern/init_main.c standard kern/init_sysent.c standard kern/ksched.c optional _kposix_priority_scheduling kern/kern_acct.c standard kern/kern_alq.c optional alq kern/kern_clock.c standard kern/kern_condvar.c standard kern/kern_conf.c standard kern/kern_cpu.c standard kern/kern_context.c standard kern/kern_descrip.c standard kern/kern_environment.c standard kern/kern_event.c standard kern/kern_exec.c standard kern/kern_exit.c standard kern/kern_fork.c standard kern/kern_idle.c standard kern/kern_intr.c standard kern/kern_jail.c standard kern/kern_kse.c standard kern/kern_kthread.c standard kern/kern_ktr.c optional ktr kern/kern_ktrace.c standard kern/kern_linker.c standard kern/kern_lock.c standard kern/kern_lockf.c standard kern/kern_malloc.c standard kern/kern_mbuf.c standard kern/kern_mib.c standard kern/kern_module.c standard kern/kern_mtxpool.c standard kern/kern_mutex.c standard kern/kern_ntptime.c standard kern/kern_physio.c standard kern/kern_pmc.c standard kern/kern_poll.c optional device_polling kern/kern_priv.c standard kern/kern_proc.c standard kern/kern_prot.c standard kern/kern_resource.c standard kern/kern_rwlock.c standard kern/kern_sema.c standard kern/kern_shutdown.c standard kern/kern_sig.c standard kern/kern_subr.c standard kern/kern_sx.c standard kern/kern_synch.c standard kern/kern_syscalls.c standard kern/kern_sysctl.c standard kern/kern_tc.c standard kern/kern_thr.c standard kern/kern_thread.c standard kern/kern_time.c standard kern/kern_timeout.c standard kern/kern_umtx.c standard kern/kern_uuid.c standard kern/kern_xxx.c standard kern/link_elf.c standard kern/linker_if.m standard kern/md4c.c optional netsmb kern/md5c.c standard kern/p1003_1b.c standard kern/posix4_mib.c standard kern/sched_4bsd.c optional sched_4bsd kern/sched_core.c optional sched_core kern/sched_ule.c optional sched_ule kern/serdev_if.m standard kern/subr_acl_posix1e.c standard kern/subr_autoconf.c standard kern/subr_blist.c standard kern/subr_bus.c standard kern/subr_clock.c standard kern/subr_devstat.c standard kern/subr_disk.c standard kern/subr_eventhandler.c standard kern/subr_fattime.c standard kern/subr_firmware.c optional firmware kern/subr_hints.c standard kern/subr_kdb.c standard kern/subr_kobj.c standard kern/subr_lock.c standard kern/subr_log.c standard kern/subr_mbpool.c optional libmbpool kern/subr_mchain.c optional libmchain kern/subr_module.c standard kern/subr_msgbuf.c standard kern/subr_param.c standard kern/subr_pcpu.c standard kern/subr_power.c standard kern/subr_prf.c standard kern/subr_prof.c standard kern/subr_rman.c standard kern/subr_rtc.c optional genclock kern/subr_sbuf.c standard kern/subr_scanf.c standard kern/subr_sleepqueue.c standard kern/subr_smp.c standard kern/subr_stack.c optional ddb kern/subr_taskqueue.c standard kern/subr_trap.c standard kern/subr_turnstile.c standard kern/subr_unit.c standard kern/subr_witness.c optional witness kern/sys_generic.c standard kern/sys_pipe.c standard kern/sys_process.c standard kern/sys_socket.c standard kern/syscalls.c optional witness | invariants kern/sysv_ipc.c standard kern/sysv_msg.c optional sysvmsg kern/sysv_sem.c optional sysvsem kern/sysv_shm.c optional sysvshm kern/tty.c standard kern/tty_compat.c optional compat_43tty kern/tty_conf.c standard kern/tty_cons.c standard kern/tty_pty.c optional pty kern/tty_pts.c optional pty kern/tty_subr.c standard kern/tty_tty.c standard kern/uipc_accf.c optional inet kern/uipc_cow.c optional zero_copy_sockets kern/uipc_debug.c optional ddb kern/uipc_domain.c standard kern/uipc_mbuf.c standard kern/uipc_mbuf2.c standard kern/uipc_mqueue.c optional p1003_1b_mqueue kern/uipc_sem.c optional p1003_1b_semaphores kern/uipc_sockbuf.c standard kern/uipc_socket.c standard kern/uipc_syscalls.c standard kern/uipc_usrreq.c standard kern/vfs_acl.c standard kern/vfs_aio.c optional vfs_aio kern/vfs_bio.c standard kern/vfs_cache.c standard kern/vfs_cluster.c standard kern/vfs_default.c standard kern/vfs_export.c standard kern/vfs_extattr.c standard kern/vfs_hash.c standard kern/vfs_init.c standard kern/vfs_lookup.c standard kern/vfs_mount.c standard kern/vfs_subr.c standard kern/vfs_syscalls.c standard kern/vfs_vnops.c standard # # These files in libkern/ are those needed by all architectures. Some # of the files in libkern/ are only needed on some architectures, e.g., # libkern/divdi3.c is needed by i386 but not alpha. Also, some of these # routines may be optimized for a particular platform. In either case, # the file should be moved to conf/files. from here. # libkern/arc4random.c standard libkern/bcd.c standard libkern/bsearch.c standard libkern/crc32.c standard libkern/fnmatch.c standard libkern/gets.c standard libkern/iconv.c optional libiconv libkern/iconv_converter_if.m optional libiconv libkern/iconv_xlat.c optional libiconv libkern/iconv_xlat16.c optional libiconv libkern/index.c standard libkern/inet_ntoa.c standard libkern/mcount.c optional profiling-routine libkern/qsort.c standard libkern/qsort_r.c standard libkern/random.c standard libkern/rindex.c standard libkern/scanc.c standard libkern/skpc.c standard libkern/strcasecmp.c standard libkern/strcat.c standard libkern/strcmp.c standard libkern/strcpy.c standard libkern/strdup.c standard libkern/strlcat.c standard libkern/strlcpy.c standard libkern/strlen.c standard libkern/strncmp.c standard libkern/strncpy.c standard libkern/strsep.c standard libkern/strspn.c standard libkern/strstr.c standard libkern/strtol.c standard libkern/strtoq.c standard libkern/strtoul.c standard libkern/strtouq.c standard libkern/strvalid.c standard net/bpf.c standard net/bpf_jitter.c optional bpf_jitter net/bpf_filter.c optional bpf | netgraph_bpf net/bridgestp.c optional if_bridge net/bsd_comp.c optional ppp_bsdcomp net/ieee8023ad_lacp.c optional lagg net/if.c standard net/if_arcsubr.c optional arcnet net/if_atmsubr.c optional atm net/if_bridge.c optional if_bridge net/if_clone.c standard net/if_disc.c optional disc net/if_edsc.c optional edsc net/if_ef.c optional ef net/if_enc.c optional enc net/if_ethersubr.c optional ether net/if_faith.c optional faith net/if_fddisubr.c optional fddi net/if_fwsubr.c optional fwip net/if_gif.c optional gif net/if_gre.c optional gre net/if_iso88025subr.c optional token net/if_lagg.c optional lagg net/if_loop.c optional loop net/if_media.c standard net/if_mib.c standard net/if_ppp.c optional ppp net/if_sl.c optional sl net/if_spppfr.c optional i4bisppp | sppp | netgraph_sppp net/if_spppsubr.c optional i4bisppp | sppp | netgraph_sppp net/if_stf.c optional stf net/if_tun.c optional tun net/if_tap.c optional tap net/if_vlan.c optional vlan net/mppcc.c optional netgraph_mppc_compression net/mppcd.c optional netgraph_mppc_compression net/netisr.c standard net/ppp_deflate.c optional ppp_deflate net/ppp_tty.c optional ppp net/pfil.c optional ether | inet net/radix.c standard net/raw_cb.c standard net/raw_usrreq.c standard net/route.c standard net/rtsock.c standard net/slcompress.c optional netgraph_vjc | ppp | sl | sppp | \ netgraph_sppp net/zlib.c optional crypto | geom_uzip | ipsec | \ ppp_deflate | netgraph_deflate net80211/ieee80211.c optional wlan net80211/ieee80211_acl.c optional wlan_acl net80211/ieee80211_amrr.c optional wlan_amrr net80211/ieee80211_crypto.c optional wlan net80211/ieee80211_crypto_ccmp.c optional wlan_ccmp net80211/ieee80211_crypto_none.c optional wlan net80211/ieee80211_crypto_tkip.c optional wlan_tkip net80211/ieee80211_crypto_wep.c optional wlan_wep net80211/ieee80211_freebsd.c optional wlan net80211/ieee80211_input.c optional wlan net80211/ieee80211_ioctl.c optional wlan net80211/ieee80211_node.c optional wlan net80211/ieee80211_output.c optional wlan net80211/ieee80211_proto.c optional wlan net80211/ieee80211_xauth.c optional wlan_xauth netatalk/aarp.c optional netatalk netatalk/at_control.c optional netatalk netatalk/at_proto.c optional netatalk netatalk/at_rmx.c optional netatalkdebug netatalk/ddp_input.c optional netatalk netatalk/ddp_output.c optional netatalk netatalk/ddp_pcb.c optional netatalk netatalk/ddp_usrreq.c optional netatalk netatm/atm_aal5.c optional atm_core netatm/atm_cm.c optional atm_core netatm/atm_device.c optional atm_core netatm/atm_if.c optional atm_core netatm/atm_proto.c optional atm_core netatm/atm_signal.c optional atm_core netatm/atm_socket.c optional atm_core netatm/atm_subr.c optional atm_core netatm/atm_usrreq.c optional atm_core netatm/ipatm/ipatm_event.c optional atm_ip atm_core netatm/ipatm/ipatm_if.c optional atm_ip atm_core netatm/ipatm/ipatm_input.c optional atm_ip atm_core netatm/ipatm/ipatm_load.c optional atm_ip atm_core netatm/ipatm/ipatm_output.c optional atm_ip atm_core netatm/ipatm/ipatm_usrreq.c optional atm_ip atm_core netatm/ipatm/ipatm_vcm.c optional atm_ip atm_core netatm/sigpvc/sigpvc_if.c optional atm_sigpvc atm_core netatm/sigpvc/sigpvc_subr.c optional atm_sigpvc atm_core netatm/spans/spans_arp.c optional atm_spans atm_core \ dependency "spans_xdr.h" netatm/spans/spans_cls.c optional atm_spans atm_core netatm/spans/spans_if.c optional atm_spans atm_core netatm/spans/spans_kxdr.c optional atm_spans atm_core netatm/spans/spans_msg.c optional atm_spans atm_core netatm/spans/spans_print.c optional atm_spans atm_core netatm/spans/spans_proto.c optional atm_spans atm_core netatm/spans/spans_subr.c optional atm_spans atm_core netatm/spans/spans_util.c optional atm_spans atm_core spans_xdr.h optional atm_spans atm_core \ before-depend \ dependency "$S/netatm/spans/spans_xdr.x" \ compile-with "rpcgen -h -C $S/netatm/spans/spans_xdr.x | grep -v rpc/rpc.h > spans_xdr.h" \ clean "spans_xdr.h" \ no-obj no-implicit-rule spans_xdr.c optional atm_spans atm_core \ before-depend \ dependency "$S/netatm/spans/spans_xdr.x" \ compile-with "rpcgen -c -C $S/netatm/spans/spans_xdr.x | grep -v rpc/rpc.h > spans_xdr.c" \ clean "spans_xdr.c" \ no-obj no-implicit-rule local spans_xdr.o optional atm_spans atm_core \ dependency "$S/netatm/spans/spans_xdr.x" \ compile-with "${NORMAL_C}" \ no-implicit-rule local netatm/uni/q2110_sigaa.c optional atm_uni atm_core netatm/uni/q2110_sigcpcs.c optional atm_uni atm_core netatm/uni/q2110_subr.c optional atm_uni atm_core netatm/uni/qsaal1_sigaa.c optional atm_uni atm_core netatm/uni/qsaal1_sigcpcs.c optional atm_uni atm_core netatm/uni/qsaal1_subr.c optional atm_uni atm_core netatm/uni/sscf_uni.c optional atm_uni atm_core netatm/uni/sscf_uni_lower.c optional atm_uni atm_core netatm/uni/sscf_uni_upper.c optional atm_uni atm_core netatm/uni/sscop.c optional atm_uni atm_core netatm/uni/sscop_lower.c optional atm_uni atm_core netatm/uni/sscop_pdu.c optional atm_uni atm_core netatm/uni/sscop_sigaa.c optional atm_uni atm_core netatm/uni/sscop_sigcpcs.c optional atm_uni atm_core netatm/uni/sscop_subr.c optional atm_uni atm_core netatm/uni/sscop_timer.c optional atm_uni atm_core netatm/uni/sscop_upper.c optional atm_uni atm_core netatm/uni/uni_load.c optional atm_uni atm_core netatm/uni/uniarp.c optional atm_uni atm_core netatm/uni/uniarp_cache.c optional atm_uni atm_core netatm/uni/uniarp_input.c optional atm_uni atm_core netatm/uni/uniarp_output.c optional atm_uni atm_core netatm/uni/uniarp_timer.c optional atm_uni atm_core netatm/uni/uniarp_vcm.c optional atm_uni atm_core netatm/uni/uniip.c optional atm_uni atm_core netatm/uni/unisig_decode.c optional atm_uni atm_core netatm/uni/unisig_encode.c optional atm_uni atm_core netatm/uni/unisig_if.c optional atm_uni atm_core netatm/uni/unisig_mbuf.c optional atm_uni atm_core netatm/uni/unisig_msg.c optional atm_uni atm_core netatm/uni/unisig_print.c optional atm_uni atm_core netatm/uni/unisig_proto.c optional atm_uni atm_core netatm/uni/unisig_sigmgr_state.c optional atm_uni atm_core netatm/uni/unisig_subr.c optional atm_uni atm_core netatm/uni/unisig_util.c optional atm_uni atm_core netatm/uni/unisig_vc_state.c optional atm_uni atm_core netgraph/atm/atmpif/ng_atmpif.c optional netgraph_atm_atmpif netgraph/atm/atmpif/ng_atmpif_harp.c optional netgraph_atm_atmpif netgraph/atm/ccatm/ng_ccatm.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/atm/ng_atm.c optional ngatm_atm netgraph/atm/ngatmbase.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/atm/sscfu/ng_sscfu.c optional ngatm_sscfu \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/atm/sscop/ng_sscop.c optional ngatm_sscop \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/atm/uni/ng_uni.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/bluetooth/common/ng_bluetooth.c optional netgraph_bluetooth netgraph/bluetooth/drivers/bt3c/ng_bt3c_pccard.c optional netgraph_bluetooth_bt3c netgraph/bluetooth/drivers/h4/ng_h4.c optional netgraph_bluetooth_h4 netgraph/bluetooth/drivers/ubt/ng_ubt.c optional netgraph_bluetooth_ubt netgraph/bluetooth/drivers/ubtbcmfw/ubtbcmfw.c optional netgraph_bluetooth_ubtbcmfw netgraph/bluetooth/hci/ng_hci_cmds.c optional netgraph_bluetooth_hci netgraph/bluetooth/hci/ng_hci_evnt.c optional netgraph_bluetooth_hci netgraph/bluetooth/hci/ng_hci_main.c optional netgraph_bluetooth_hci netgraph/bluetooth/hci/ng_hci_misc.c optional netgraph_bluetooth_hci netgraph/bluetooth/hci/ng_hci_ulpi.c optional netgraph_bluetooth_hci netgraph/bluetooth/l2cap/ng_l2cap_cmds.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_evnt.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_llpi.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_main.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_misc.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_ulpi.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/socket/ng_btsocket.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_hci_raw.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_l2cap.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_l2cap_raw.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_rfcomm.c optional netgraph_bluetooth_socket netgraph/netflow/netflow.c optional netgraph_netflow netgraph/netflow/ng_netflow.c optional netgraph_netflow netgraph/ng_UI.c optional netgraph_UI netgraph/ng_async.c optional netgraph_async netgraph/ng_atmllc.c optional netgraph_atmllc netgraph/ng_base.c optional netgraph netgraph/ng_bpf.c optional netgraph_bpf netgraph/ng_bridge.c optional netgraph_bridge netgraph/ng_car.c optional netgraph_car netgraph/ng_cisco.c optional netgraph_cisco netgraph/ng_deflate.c optional netgraph_deflate netgraph/ng_device.c optional netgraph_device netgraph/ng_echo.c optional netgraph_echo netgraph/ng_eiface.c optional netgraph_eiface netgraph/ng_ether.c optional netgraph_ether netgraph/ng_fec.c optional netgraph_fec netgraph/ng_frame_relay.c optional netgraph_frame_relay netgraph/ng_gif.c optional netgraph_gif netgraph/ng_gif_demux.c optional netgraph_gif_demux netgraph/ng_hole.c optional netgraph_hole netgraph/ng_iface.c optional netgraph_iface netgraph/ng_ip_input.c optional netgraph_ip_input netgraph/ng_ipfw.c optional netgraph_ipfw netgraph/ng_ksocket.c optional netgraph_ksocket netgraph/ng_l2tp.c optional netgraph_l2tp netgraph/ng_lmi.c optional netgraph_lmi netgraph/ng_mppc.c optional netgraph_mppc_compression | \ netgraph_mppc_encryption netgraph/ng_nat.c optional netgraph_nat netgraph/ng_one2many.c optional netgraph_one2many netgraph/ng_parse.c optional netgraph netgraph/ng_ppp.c optional netgraph_ppp netgraph/ng_pppoe.c optional netgraph_pppoe netgraph/ng_pptpgre.c optional netgraph_pptpgre netgraph/ng_pred1.c optional netgraph_pred1 netgraph/ng_rfc1490.c optional netgraph_rfc1490 netgraph/ng_socket.c optional netgraph_socket netgraph/ng_split.c optional netgraph_split netgraph/ng_sppp.c optional netgraph_sppp netgraph/ng_tag.c optional netgraph_tag netgraph/ng_tcpmss.c optional netgraph_tcpmss netgraph/ng_tee.c optional netgraph_tee netgraph/ng_tty.c optional netgraph_tty netgraph/ng_vjc.c optional netgraph_vjc netinet/accf_data.c optional accept_filter_data netinet/accf_http.c optional accept_filter_http netinet/if_atm.c optional atm netinet/if_ether.c optional ether netinet/igmp.c optional inet netinet/in.c optional inet netinet/ip_carp.c optional carp netinet/in_gif.c optional gif inet netinet/ip_gre.c optional gre inet netinet/ip_id.c optional inet netinet/in_pcb.c optional inet netinet/in_proto.c optional inet \ compile-with "${NORMAL_C} -I$S/contrib/pf" netinet/in_rmx.c optional inet netinet/ip_divert.c optional ipdivert netinet/ip_dummynet.c optional dummynet netinet/ip_ecn.c optional inet | inet6 netinet/ip_encap.c optional inet | inet6 netinet/ip_fastfwd.c optional inet netinet/ip_fw2.c optional ipfirewall netinet/ip_fw_pfil.c optional ipfirewall netinet/ip_icmp.c optional inet netinet/ip_input.c optional inet netinet/ip_ipsec.c optional ipsec netinet/ip_ipsec.c optional fast_ipsec netinet/ip_mroute.c optional mrouting inet | mrouting inet6 netinet/ip_options.c optional inet netinet/ip_output.c optional inet netinet/raw_ip.c optional inet netinet/sctp_asconf.c optional inet inet6 sctp netinet/sctp_auth.c optional inet inet6 sctp netinet/sctp_bsd_addr.c optional inet inet6 sctp netinet/sctp_crc32.c optional inet inet6 sctp netinet/sctp_indata.c optional inet inet6 sctp netinet/sctp_input.c optional inet inet6 sctp netinet/sctp_output.c optional inet inet6 sctp netinet/sctp_pcb.c optional inet inet6 sctp netinet/sctp_peeloff.c optional inet inet6 sctp netinet/sctp_sysctl.c optional inet inet6 sctp netinet/sctp_timer.c optional inet inet6 sctp netinet/sctp_usrreq.c optional inet inet6 sctp netinet/sctputil.c optional inet inet6 sctp netinet/tcp_debug.c optional tcpdebug netinet/tcp_hostcache.c optional inet netinet/tcp_input.c optional inet netinet/tcp_output.c optional inet netinet/tcp_reass.c optional inet netinet/tcp_sack.c optional inet netinet/tcp_subr.c optional inet netinet/tcp_syncache.c optional inet netinet/tcp_timer.c optional inet netinet/tcp_timewait.c optional inet netinet/tcp_usrreq.c optional inet netinet/udp_usrreq.c optional inet netinet/libalias/alias.c optional libalias | netgraph_nat netinet/libalias/alias_db.c optional libalias | netgraph_nat netinet/libalias/alias_mod.c optional libalias | netgraph_nat netinet/libalias/alias_proxy.c optional libalias | netgraph_nat netinet/libalias/alias_util.c optional libalias | netgraph_nat netinet6/ah_aesxcbcmac.c optional ipsec netinet6/ah_core.c optional ipsec netinet6/ah_input.c optional ipsec netinet6/ah_output.c optional ipsec netinet6/dest6.c optional inet6 netinet6/esp_aesctr.c optional ipsec ipsec_esp netinet6/esp_core.c optional ipsec ipsec_esp netinet6/esp_input.c optional ipsec ipsec_esp netinet6/esp_output.c optional ipsec ipsec_esp netinet6/esp_rijndael.c optional ipsec ipsec_esp netinet6/esp_camellia.c optional ipsec ipsec_esp netinet6/frag6.c optional inet6 netinet6/icmp6.c optional inet6 netinet6/in6.c optional inet6 netinet6/in6_cksum.c optional inet6 netinet6/in6_gif.c optional gif inet6 netinet6/in6_ifattach.c optional inet6 netinet6/in6_pcb.c optional inet6 netinet6/in6_proto.c optional inet6 netinet6/in6_rmx.c optional inet6 netinet6/in6_src.c optional inet6 netinet6/ip6_forward.c optional inet6 netinet6/ip6_id.c optional inet6 netinet6/ip6_input.c optional inet6 netinet6/ip6_mroute.c optional mrouting inet6 netinet6/ip6_output.c optional inet6 netinet6/ipcomp_core.c optional ipsec netinet6/ipcomp_input.c optional ipsec netinet6/ipcomp_output.c optional ipsec netinet6/ipsec.c optional ipsec netinet6/mld6.c optional inet6 netinet6/nd6.c optional inet6 netinet6/nd6_nbr.c optional inet6 netinet6/nd6_rtr.c optional inet6 netinet6/raw_ip6.c optional inet6 netinet6/route6.c optional inet6 netinet6/scope6.c optional inet6 netinet6/sctp6_usrreq.c optional inet6 sctp netinet6/udp6_output.c optional inet6 netinet6/udp6_usrreq.c optional inet6 netipsec/ipsec.c optional fast_ipsec netipsec/ipsec_input.c optional fast_ipsec netipsec/ipsec_mbuf.c optional fast_ipsec netipsec/ipsec_output.c optional fast_ipsec netipsec/key.c optional fast_ipsec netipsec/key_debug.c optional fast_ipsec netipsec/keysock.c optional fast_ipsec netipsec/xform_ah.c optional fast_ipsec netipsec/xform_esp.c optional fast_ipsec netipsec/xform_ipcomp.c optional fast_ipsec netipsec/xform_ipip.c optional fast_ipsec netipsec/xform_tcp.c optional fast_ipsec tcp_signature netipx/ipx.c optional ipx netipx/ipx_cksum.c optional ipx netipx/ipx_input.c optional ipx netipx/ipx_ip.c optional ipx ipxip netipx/ipx_outputfl.c optional ipx netipx/ipx_pcb.c optional ipx netipx/ipx_proto.c optional ipx netipx/ipx_usrreq.c optional ipx netipx/spx_debug.c optional ipx netipx/spx_usrreq.c optional ipx netkey/key.c optional ipsec netkey/key_debug.c optional ipsec netkey/keydb.c optional ipsec netkey/keysock.c optional ipsec netnatm/natm.c optional natm netnatm/natm_pcb.c optional natm netnatm/natm_proto.c optional natm netncp/ncp_conn.c optional ncp netncp/ncp_crypt.c optional ncp netncp/ncp_login.c optional ncp netncp/ncp_mod.c optional ncp netncp/ncp_ncp.c optional ncp netncp/ncp_nls.c optional ncp netncp/ncp_rq.c optional ncp netncp/ncp_sock.c optional ncp netncp/ncp_subr.c optional ncp netsmb/smb_conn.c optional netsmb netsmb/smb_crypt.c optional netsmb netsmb/smb_dev.c optional netsmb netsmb/smb_iod.c optional netsmb netsmb/smb_rq.c optional netsmb netsmb/smb_smb.c optional netsmb netsmb/smb_subr.c optional netsmb netsmb/smb_trantcp.c optional netsmb netsmb/smb_usr.c optional netsmb nfs/nfs_common.c optional nfsclient | nfsserver nfs4client/nfs4_dev.c optional nfsclient nfs4client/nfs4_idmap.c optional nfsclient nfs4client/nfs4_socket.c optional nfsclient nfs4client/nfs4_subs.c optional nfsclient nfs4client/nfs4_vfs_subs.c optional nfsclient nfs4client/nfs4_vfsops.c optional nfsclient nfs4client/nfs4_vn_subs.c optional nfsclient nfs4client/nfs4_vnops.c optional nfsclient nfsclient/bootp_subr.c optional bootp nfsclient nfsclient/krpc_subr.c optional bootp nfsclient nfsclient/nfs_bio.c optional nfsclient nfsclient/nfs_diskless.c optional nfsclient nfs_root nfsclient/nfs_node.c optional nfsclient nfsclient/nfs_socket.c optional nfsclient nfsclient/nfs_subs.c optional nfsclient nfsclient/nfs_nfsiod.c optional nfsclient nfsclient/nfs_vfsops.c optional nfsclient nfsclient/nfs_vnops.c optional nfsclient nfsclient/nfs_lock.c optional nfsclient nfsserver/nfs_serv.c optional nfsserver nfsserver/nfs_srvsock.c optional nfsserver nfsserver/nfs_srvcache.c optional nfsserver nfsserver/nfs_srvsubs.c optional nfsserver nfsserver/nfs_syscalls.c optional nfsserver # crypto support opencrypto/cast.c optional crypto | ipsec ipsec_esp opencrypto/criov.c optional crypto opencrypto/crypto.c optional crypto opencrypto/cryptodev.c optional cryptodev opencrypto/cryptodev_if.m optional crypto opencrypto/cryptosoft.c optional crypto opencrypto/deflate.c optional crypto opencrypto/rmd160.c optional crypto | ipsec opencrypto/skipjack.c optional crypto opencrypto/xform.c optional crypto pci/agp.c optional agp pci pci/agp_if.m optional agp pci pci/alpm.c optional alpm pci pci/amdpm.c optional amdpm pci | nfpm pci pci/amdsmb.c optional amdsmb pci pci/if_mn.c optional mn pci pci/if_pcn.c optional pcn pci pci/if_rl.c optional rl pci pci/if_sf.c optional sf pci pci/if_sis.c optional sis pci pci/if_ste.c optional ste pci pci/if_tl.c optional tl pci pci/if_vr.c optional vr pci pci/if_wb.c optional wb pci pci/if_xl.c optional xl pci pci/intpm.c optional intpm pci pci/ncr.c optional ncr pci pci/nfsmb.c optional nfsmb pci pci/viapm.c optional viapm pci pci/xrpu.c optional xrpu pci rpc/rpcclnt.c optional nfsclient security/audit/audit.c optional audit security/audit/audit_arg.c optional audit security/audit/audit_bsm.c optional audit security/audit/audit_bsm_klib.c optional audit security/audit/audit_bsm_token.c optional audit security/audit/audit_pipe.c optional audit security/audit/audit_syscalls.c standard security/audit/audit_trigger.c optional audit security/audit/audit_worker.c optional audit security/mac/mac_audit.c optional mac audit security/mac/mac_framework.c optional mac security/mac/mac_inet.c optional mac inet security/mac/mac_label.c optional mac security/mac/mac_net.c optional mac security/mac/mac_pipe.c optional mac security/mac/mac_posix_sem.c optional mac security/mac/mac_priv.c optional mac security/mac/mac_process.c optional mac security/mac/mac_socket.c optional mac security/mac/mac_syscalls.c standard security/mac/mac_system.c optional mac security/mac/mac_sysv_msg.c optional mac security/mac/mac_sysv_sem.c optional mac security/mac/mac_sysv_shm.c optional mac security/mac/mac_vfs.c optional mac security/mac_biba/mac_biba.c optional mac_biba security/mac_bsdextended/mac_bsdextended.c optional mac_bsdextended security/mac_ifoff/mac_ifoff.c optional mac_ifoff security/mac_lomac/mac_lomac.c optional mac_lomac security/mac_mls/mac_mls.c optional mac_mls security/mac_none/mac_none.c optional mac_none security/mac_partition/mac_partition.c optional mac_partition security/mac_portacl/mac_portacl.c optional mac_portacl security/mac_seeotheruids/mac_seeotheruids.c optional mac_seeotheruids security/mac_stub/mac_stub.c optional mac_stub security/mac_test/mac_test.c optional mac_test ufs/ffs/ffs_alloc.c optional ffs ufs/ffs/ffs_balloc.c optional ffs ufs/ffs/ffs_inode.c optional ffs ufs/ffs/ffs_snapshot.c optional ffs ufs/ffs/ffs_softdep.c optional ffs ufs/ffs/ffs_subr.c optional ffs ufs/ffs/ffs_tables.c optional ffs ufs/ffs/ffs_vfsops.c optional ffs ufs/ffs/ffs_vnops.c optional ffs ufs/ffs/ffs_rawread.c optional directio ufs/ufs/ufs_acl.c optional ffs ufs/ufs/ufs_bmap.c optional ffs ufs/ufs/ufs_dirhash.c optional ffs ufs/ufs/ufs_extattr.c optional ffs ufs/ufs/ufs_gjournal.c optional ffs ufs/ufs/ufs_inode.c optional ffs ufs/ufs/ufs_lookup.c optional ffs ufs/ufs/ufs_quota.c optional ffs ufs/ufs/ufs_vfsops.c optional ffs ufs/ufs/ufs_vnops.c optional ffs vm/default_pager.c standard vm/device_pager.c standard vm/phys_pager.c standard vm/redzone.c optional DEBUG_REDZONE vm/swap_pager.c standard vm/uma_core.c standard vm/uma_dbg.c standard vm/vm_contig.c standard vm/memguard.c optional DEBUG_MEMGUARD vm/vm_fault.c standard vm/vm_glue.c standard vm/vm_init.c standard vm/vm_kern.c standard vm/vm_map.c standard vm/vm_meter.c standard vm/vm_mmap.c standard vm/vm_object.c standard vm/vm_page.c standard vm/vm_pageout.c standard vm/vm_pageq.c standard vm/vm_pager.c standard vm/vm_unix.c standard vm/vm_zeroidle.c standard vm/vnode_pager.c standard # gnu/fs/xfs/xfs_alloc.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" \ warning "kernel contains GPL contaminated xfs filesystem" gnu/fs/xfs/xfs_alloc_btree.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_bit.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_bmap.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_bmap_btree.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_btree.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_buf_item.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_da_btree.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir2.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir2_block.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir2_data.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir2_leaf.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir2_node.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir2_sf.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir2_trace.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir_leaf.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_error.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_extfree_item.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_fsops.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_ialloc.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_ialloc_btree.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_inode.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_inode_item.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_iocore.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_itable.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dfrag.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_log.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_log_recover.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_mount.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_rename.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_trans.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_trans_ail.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_trans_buf.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_trans_extfree.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_trans_inode.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_trans_item.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_utils.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_vfsops.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_vnodeops.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_rw.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_attr_leaf.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_attr.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dmops.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_qmops.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_iget.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_freebsd_iget.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_mountops.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_vnops.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_frw.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_buf.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_globals.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_dmistubs.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_super.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_stats.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_vfs.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_vnode.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_sysctl.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_fs_subr.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_ioctl.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/support/debug.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/support/ktrace.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/support/mrlock.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/support/uuid.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/support/kmem.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_iomap.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_behavior.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" Index: head/sys/dev/cxgb/t3fw-3.2.bin.gz.uu =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: head/sys/dev/cxgb/t3fw-3.2.bin.gz.uu ___________________________________________________________________ Deleted: svn:mime-type ## -1 +0,0 ## -application/octet-stream \ No newline at end of property Index: head/sys/dev/cxgb/common/cxgb_common.h =================================================================== --- head/sys/dev/cxgb/common/cxgb_common.h (revision 169977) +++ head/sys/dev/cxgb/common/cxgb_common.h (revision 169978) @@ -1,722 +1,729 @@ /************************************************************************** Copyright (c) 2007, Chelsio Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the Chelsio Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. $FreeBSD$ ***************************************************************************/ #ifndef __CHELSIO_COMMON_H #define __CHELSIO_COMMON_H #include enum { MAX_NPORTS = 2, /* max # of ports */ MAX_FRAME_SIZE = 10240, /* max MAC frame size, including header + FCS */ EEPROMSIZE = 8192, /* Serial EEPROM size */ RSS_TABLE_SIZE = 64, /* size of RSS lookup and mapping tables */ TCB_SIZE = 128, /* TCB size */ NMTUS = 16, /* size of MTU table */ NCCTRL_WIN = 32, /* # of congestion control windows */ NTX_SCHED = 8, /* # of HW Tx scheduling queues */ + TP_TMR_RES = 200, /* TP timer resolution in usec */ }; #define MAX_RX_COALESCING_LEN 16224U enum { PAUSE_RX = 1 << 0, PAUSE_TX = 1 << 1, PAUSE_AUTONEG = 1 << 2 }; enum { - SUPPORTED_OFFLOAD = 1 << 24, SUPPORTED_IRQ = 1 << 25 }; enum { /* adapter interrupt-maintained statistics */ STAT_ULP_CH0_PBL_OOB, STAT_ULP_CH1_PBL_OOB, STAT_PCI_CORR_ECC, IRQ_NUM_STATS /* keep last */ }; enum { - FW_VERSION_MAJOR = 3, - FW_VERSION_MINOR = 2, + FW_VERSION_MAJOR = 4, + FW_VERSION_MINOR = 0, FW_VERSION_MICRO = 0 }; enum { SGE_QSETS = 8, /* # of SGE Tx/Rx/RspQ sets */ SGE_RXQ_PER_SET = 2, /* # of Rx queues per set */ SGE_TXQ_PER_SET = 3 /* # of Tx queues per set */ }; enum sge_context_type { /* SGE egress context types */ SGE_CNTXT_RDMA = 0, SGE_CNTXT_ETH = 2, SGE_CNTXT_OFLD = 4, SGE_CNTXT_CTRL = 5 }; enum { AN_PKT_SIZE = 32, /* async notification packet size */ IMMED_PKT_SIZE = 48 /* packet size for immediate data */ }; struct sg_ent { /* SGE scatter/gather entry */ u32 len[2]; u64 addr[2]; }; #ifndef SGE_NUM_GENBITS /* Must be 1 or 2 */ # define SGE_NUM_GENBITS 2 #endif #define TX_DESC_FLITS 16U #define WR_FLITS (TX_DESC_FLITS + 1 - SGE_NUM_GENBITS) struct cphy; struct mdio_ops { int (*read)(adapter_t *adapter, int phy_addr, int mmd_addr, int reg_addr, unsigned int *val); int (*write)(adapter_t *adapter, int phy_addr, int mmd_addr, int reg_addr, unsigned int val); }; struct adapter_info { unsigned char nports; /* # of ports */ unsigned char phy_base_addr; /* MDIO PHY base address */ unsigned char mdien; unsigned char mdiinv; unsigned int gpio_out; /* GPIO output settings */ unsigned int gpio_intr; /* GPIO IRQ enable mask */ unsigned long caps; /* adapter capabilities */ const struct mdio_ops *mdio_ops; /* MDIO operations */ const char *desc; /* product description */ }; struct port_type_info { void (*phy_prep)(struct cphy *phy, adapter_t *adapter, int phy_addr, const struct mdio_ops *ops); unsigned int caps; const char *desc; }; struct mc5_stats { unsigned long parity_err; unsigned long active_rgn_full; unsigned long nfa_srch_err; unsigned long unknown_cmd; unsigned long reqq_parity_err; unsigned long dispq_parity_err; unsigned long del_act_empty; }; struct mc7_stats { unsigned long corr_err; unsigned long uncorr_err; unsigned long parity_err; unsigned long addr_err; }; struct mac_stats { u64 tx_octets; /* total # of octets in good frames */ u64 tx_octets_bad; /* total # of octets in error frames */ u64 tx_frames; /* all good frames */ u64 tx_mcast_frames; /* good multicast frames */ u64 tx_bcast_frames; /* good broadcast frames */ u64 tx_pause; /* # of transmitted pause frames */ u64 tx_deferred; /* frames with deferred transmissions */ u64 tx_late_collisions; /* # of late collisions */ u64 tx_total_collisions; /* # of total collisions */ u64 tx_excess_collisions; /* frame errors from excessive collissions */ u64 tx_underrun; /* # of Tx FIFO underruns */ u64 tx_len_errs; /* # of Tx length errors */ u64 tx_mac_internal_errs; /* # of internal MAC errors on Tx */ u64 tx_excess_deferral; /* # of frames with excessive deferral */ u64 tx_fcs_errs; /* # of frames with bad FCS */ u64 tx_frames_64; /* # of Tx frames in a particular range */ u64 tx_frames_65_127; u64 tx_frames_128_255; u64 tx_frames_256_511; u64 tx_frames_512_1023; u64 tx_frames_1024_1518; u64 tx_frames_1519_max; u64 rx_octets; /* total # of octets in good frames */ u64 rx_octets_bad; /* total # of octets in error frames */ u64 rx_frames; /* all good frames */ u64 rx_mcast_frames; /* good multicast frames */ u64 rx_bcast_frames; /* good broadcast frames */ u64 rx_pause; /* # of received pause frames */ u64 rx_fcs_errs; /* # of received frames with bad FCS */ u64 rx_align_errs; /* alignment errors */ u64 rx_symbol_errs; /* symbol errors */ u64 rx_data_errs; /* data errors */ u64 rx_sequence_errs; /* sequence errors */ u64 rx_runt; /* # of runt frames */ u64 rx_jabber; /* # of jabber frames */ u64 rx_short; /* # of short frames */ u64 rx_too_long; /* # of oversized frames */ u64 rx_mac_internal_errs; /* # of internal MAC errors on Rx */ u64 rx_frames_64; /* # of Rx frames in a particular range */ u64 rx_frames_65_127; u64 rx_frames_128_255; u64 rx_frames_256_511; u64 rx_frames_512_1023; u64 rx_frames_1024_1518; u64 rx_frames_1519_max; u64 rx_cong_drops; /* # of Rx drops due to SGE congestion */ unsigned long tx_fifo_parity_err; unsigned long rx_fifo_parity_err; unsigned long tx_fifo_urun; unsigned long rx_fifo_ovfl; unsigned long serdes_signal_loss; unsigned long xaui_pcs_ctc_err; unsigned long xaui_pcs_align_change; unsigned long num_toggled; /* # times toggled TxEn due to stuck TX */ unsigned long num_resets; /* # times reset due to stuck TX */ }; struct tp_mib_stats { u32 ipInReceive_hi; u32 ipInReceive_lo; u32 ipInHdrErrors_hi; u32 ipInHdrErrors_lo; u32 ipInAddrErrors_hi; u32 ipInAddrErrors_lo; u32 ipInUnknownProtos_hi; u32 ipInUnknownProtos_lo; u32 ipInDiscards_hi; u32 ipInDiscards_lo; u32 ipInDelivers_hi; u32 ipInDelivers_lo; u32 ipOutRequests_hi; u32 ipOutRequests_lo; u32 ipOutDiscards_hi; u32 ipOutDiscards_lo; u32 ipOutNoRoutes_hi; u32 ipOutNoRoutes_lo; u32 ipReasmTimeout; u32 ipReasmReqds; u32 ipReasmOKs; u32 ipReasmFails; u32 reserved[8]; u32 tcpActiveOpens; u32 tcpPassiveOpens; u32 tcpAttemptFails; u32 tcpEstabResets; u32 tcpOutRsts; u32 tcpCurrEstab; u32 tcpInSegs_hi; u32 tcpInSegs_lo; u32 tcpOutSegs_hi; u32 tcpOutSegs_lo; u32 tcpRetransSeg_hi; u32 tcpRetransSeg_lo; u32 tcpInErrs_hi; u32 tcpInErrs_lo; u32 tcpRtoMin; u32 tcpRtoMax; }; struct tp_params { unsigned int nchan; /* # of channels */ unsigned int pmrx_size; /* total PMRX capacity */ unsigned int pmtx_size; /* total PMTX capacity */ unsigned int cm_size; /* total CM capacity */ unsigned int chan_rx_size; /* per channel Rx size */ unsigned int chan_tx_size; /* per channel Tx size */ unsigned int rx_pg_size; /* Rx page size */ unsigned int tx_pg_size; /* Tx page size */ unsigned int rx_num_pgs; /* # of Rx pages */ unsigned int tx_num_pgs; /* # of Tx pages */ unsigned int ntimer_qs; /* # of timer queues */ unsigned int dack_re; /* DACK timer resolution */ }; struct qset_params { /* SGE queue set parameters */ unsigned int polling; /* polling/interrupt service for rspq */ unsigned int coalesce_nsecs; /* irq coalescing timer */ unsigned int rspq_size; /* # of entries in response queue */ unsigned int fl_size; /* # of entries in regular free list */ unsigned int jumbo_size; /* # of entries in jumbo free list */ unsigned int txq_size[SGE_TXQ_PER_SET]; /* Tx queue sizes */ unsigned int cong_thres; /* FL congestion threshold */ unsigned int vector; /* Interrupt (line or vector) number */ }; struct sge_params { unsigned int max_pkt_size; /* max offload pkt size */ struct qset_params qset[SGE_QSETS]; }; struct mc5_params { unsigned int mode; /* selects MC5 width */ unsigned int nservers; /* size of server region */ unsigned int nfilters; /* size of filter region */ unsigned int nroutes; /* size of routing region */ }; /* Default MC5 region sizes */ enum { DEFAULT_NSERVERS = 512, DEFAULT_NFILTERS = 128 }; /* MC5 modes, these must be non-0 */ enum { MC5_MODE_144_BIT = 1, MC5_MODE_72_BIT = 2 }; +/* MC5 min active region size */ +enum { MC5_MIN_TIDS = 16 }; + struct vpd_params { unsigned int cclk; unsigned int mclk; unsigned int uclk; unsigned int mdc; unsigned int mem_timing; u8 eth_base[6]; u8 port_type[MAX_NPORTS]; unsigned short xauicfg[2]; }; struct pci_params { unsigned int vpd_cap_addr; unsigned int pcie_cap_addr; unsigned short speed; unsigned char width; unsigned char variant; }; enum { PCI_VARIANT_PCI, PCI_VARIANT_PCIX_MODE1_PARITY, PCI_VARIANT_PCIX_MODE1_ECC, PCI_VARIANT_PCIX_266_MODE2, PCI_VARIANT_PCIE }; struct adapter_params { struct sge_params sge; struct mc5_params mc5; struct tp_params tp; struct vpd_params vpd; struct pci_params pci; const struct adapter_info *info; #ifdef CONFIG_CHELSIO_T3_CORE unsigned short mtus[NMTUS]; unsigned short a_wnd[NCCTRL_WIN]; unsigned short b_wnd[NCCTRL_WIN]; #endif unsigned int nports; /* # of ethernet ports */ unsigned int stats_update_period; /* MAC stats accumulation period */ unsigned int linkpoll_period; /* link poll period in 0.1s */ unsigned int rev; /* chip revision */ + unsigned int offload; }; enum { /* chip revisions */ T3_REV_A = 0, T3_REV_B = 2, T3_REV_B2 = 3, }; struct trace_params { u32 sip; u32 sip_mask; u32 dip; u32 dip_mask; u16 sport; u16 sport_mask; u16 dport; u16 dport_mask; u32 vlan:12; u32 vlan_mask:12; u32 intf:4; u32 intf_mask:4; u8 proto; u8 proto_mask; }; struct link_config { unsigned int supported; /* link capabilities */ unsigned int advertising; /* advertised capabilities */ unsigned short requested_speed; /* speed user has requested */ unsigned short speed; /* actual link speed */ unsigned char requested_duplex; /* duplex user has requested */ unsigned char duplex; /* actual link duplex */ unsigned char requested_fc; /* flow control user has requested */ unsigned char fc; /* actual link flow control */ unsigned char autoneg; /* autonegotiating? */ unsigned int link_ok; /* link up? */ }; #define SPEED_INVALID 0xffff #define DUPLEX_INVALID 0xff struct mc5 { adapter_t *adapter; unsigned int tcam_size; unsigned char part_type; unsigned char parity_enabled; unsigned char mode; struct mc5_stats stats; }; static inline unsigned int t3_mc5_size(const struct mc5 *p) { return p->tcam_size; } struct mc7 { adapter_t *adapter; /* backpointer to adapter */ unsigned int size; /* memory size in bytes */ unsigned int width; /* MC7 interface width */ unsigned int offset; /* register address offset for MC7 instance */ const char *name; /* name of MC7 instance */ struct mc7_stats stats; /* MC7 statistics */ }; static inline unsigned int t3_mc7_size(const struct mc7 *p) { return p->size; } struct cmac { adapter_t *adapter; unsigned int offset; unsigned int nucast; /* # of address filters for unicast MACs */ - unsigned int tcnt; - unsigned int xcnt; + unsigned int tx_tcnt; + unsigned int tx_xcnt; + u64 tx_mcnt; + unsigned int rx_xcnt; + u64 rx_mcnt; unsigned int toggle_cnt; unsigned int txen; struct mac_stats stats; }; enum { MAC_DIRECTION_RX = 1, MAC_DIRECTION_TX = 2, MAC_RXFIFO_SIZE = 32768 }; /* IEEE 802.3ae specified MDIO devices */ enum { MDIO_DEV_PMA_PMD = 1, MDIO_DEV_WIS = 2, MDIO_DEV_PCS = 3, MDIO_DEV_XGXS = 4 }; /* PHY loopback direction */ enum { PHY_LOOPBACK_TX = 1, PHY_LOOPBACK_RX = 2 }; /* PHY interrupt types */ enum { cphy_cause_link_change = 1, cphy_cause_fifo_error = 2 }; /* PHY operations */ struct cphy_ops { void (*destroy)(struct cphy *phy); int (*reset)(struct cphy *phy, int wait); int (*intr_enable)(struct cphy *phy); int (*intr_disable)(struct cphy *phy); int (*intr_clear)(struct cphy *phy); int (*intr_handler)(struct cphy *phy); int (*autoneg_enable)(struct cphy *phy); int (*autoneg_restart)(struct cphy *phy); int (*advertise)(struct cphy *phy, unsigned int advertise_map); int (*set_loopback)(struct cphy *phy, int mmd, int dir, int enable); int (*set_speed_duplex)(struct cphy *phy, int speed, int duplex); int (*get_link_status)(struct cphy *phy, int *link_ok, int *speed, int *duplex, int *fc); int (*power_down)(struct cphy *phy, int enable); }; /* A PHY instance */ struct cphy { int addr; /* PHY address */ adapter_t *adapter; /* associated adapter */ unsigned long fifo_errors; /* FIFO over/under-flows */ const struct cphy_ops *ops; /* PHY operations */ int (*mdio_read)(adapter_t *adapter, int phy_addr, int mmd_addr, int reg_addr, unsigned int *val); int (*mdio_write)(adapter_t *adapter, int phy_addr, int mmd_addr, int reg_addr, unsigned int val); }; /* Convenience MDIO read/write wrappers */ static inline int mdio_read(struct cphy *phy, int mmd, int reg, unsigned int *valp) { return phy->mdio_read(phy->adapter, phy->addr, mmd, reg, valp); } static inline int mdio_write(struct cphy *phy, int mmd, int reg, unsigned int val) { return phy->mdio_write(phy->adapter, phy->addr, mmd, reg, val); } /* Convenience initializer */ static inline void cphy_init(struct cphy *phy, adapter_t *adapter, int phy_addr, struct cphy_ops *phy_ops, const struct mdio_ops *mdio_ops) { phy->adapter = adapter; phy->addr = phy_addr; phy->ops = phy_ops; if (mdio_ops) { phy->mdio_read = mdio_ops->read; phy->mdio_write = mdio_ops->write; } } /* Accumulate MAC statistics every 180 seconds. For 1G we multiply by 10. */ #define MAC_STATS_ACCUM_SECS 180 #define XGM_REG(reg_addr, idx) \ ((reg_addr) + (idx) * (XGMAC0_1_BASE_ADDR - XGMAC0_0_BASE_ADDR)) struct addr_val_pair { unsigned int reg_addr; unsigned int val; }; #include #ifndef PCI_VENDOR_ID_CHELSIO # define PCI_VENDOR_ID_CHELSIO 0x1425 #endif #define for_each_port(adapter, iter) \ for (iter = 0; iter < (adapter)->params.nports; ++iter) #define adapter_info(adap) ((adap)->params.info) static inline int uses_xaui(const adapter_t *adap) { return adapter_info(adap)->caps & SUPPORTED_AUI; } static inline int is_10G(const adapter_t *adap) { return adapter_info(adap)->caps & SUPPORTED_10000baseT_Full; } static inline int is_offload(const adapter_t *adap) { #ifdef CONFIG_CHELSIO_T3_CORE - return adapter_info(adap)->caps & SUPPORTED_OFFLOAD; + return adap->params.offload; #else return 0; #endif } static inline unsigned int core_ticks_per_usec(const adapter_t *adap) { return adap->params.vpd.cclk / 1000; } static inline unsigned int dack_ticks_to_usec(const adapter_t *adap, unsigned int ticks) { return (ticks << adap->params.tp.dack_re) / core_ticks_per_usec(adap); } static inline unsigned int is_pcie(const adapter_t *adap) { return adap->params.pci.variant == PCI_VARIANT_PCIE; } void t3_set_reg_field(adapter_t *adap, unsigned int addr, u32 mask, u32 val); void t3_read_indirect(adapter_t *adap, unsigned int addr_reg, unsigned int data_reg, u32 *vals, unsigned int nregs, unsigned int start_idx); void t3_write_regs(adapter_t *adapter, const struct addr_val_pair *p, int n, unsigned int offset); int t3_wait_op_done_val(adapter_t *adapter, int reg, u32 mask, int polarity, int attempts, int delay, u32 *valp); static inline int t3_wait_op_done(adapter_t *adapter, int reg, u32 mask, int polarity, int attempts, int delay) { return t3_wait_op_done_val(adapter, reg, mask, polarity, attempts, delay, NULL); } int t3_mdio_change_bits(struct cphy *phy, int mmd, int reg, unsigned int clear, unsigned int set); int t3_phy_reset(struct cphy *phy, int mmd, int wait); int t3_phy_advertise(struct cphy *phy, unsigned int advert); int t3_set_phy_speed_duplex(struct cphy *phy, int speed, int duplex); void t3_intr_enable(adapter_t *adapter); void t3_intr_disable(adapter_t *adapter); void t3_intr_clear(adapter_t *adapter); void t3_port_intr_enable(adapter_t *adapter, int idx); void t3_port_intr_disable(adapter_t *adapter, int idx); void t3_port_intr_clear(adapter_t *adapter, int idx); int t3_slow_intr_handler(adapter_t *adapter); int t3_phy_intr_handler(adapter_t *adapter); void t3_link_changed(adapter_t *adapter, int port_id); int t3_link_start(struct cphy *phy, struct cmac *mac, struct link_config *lc); const struct adapter_info *t3_get_adapter_info(unsigned int board_id); int t3_seeprom_read(adapter_t *adapter, u32 addr, u32 *data); int t3_seeprom_write(adapter_t *adapter, u32 addr, u32 data); int t3_seeprom_wp(adapter_t *adapter, int enable); int t3_read_flash(adapter_t *adapter, unsigned int addr, unsigned int nwords, u32 *data, int byte_oriented); int t3_load_fw(adapter_t *adapter, const u8 *fw_data, unsigned int size); int t3_get_fw_version(adapter_t *adapter, u32 *vers); int t3_check_fw_version(adapter_t *adapter); int t3_init_hw(adapter_t *adapter, u32 fw_params); void mac_prep(struct cmac *mac, adapter_t *adapter, int index); void early_hw_init(adapter_t *adapter, const struct adapter_info *ai); int t3_reset_adapter(adapter_t *adapter); int t3_prep_adapter(adapter_t *adapter, const struct adapter_info *ai, int reset); void t3_led_ready(adapter_t *adapter); void t3_fatal_err(adapter_t *adapter); void t3_set_vlan_accel(adapter_t *adapter, unsigned int ports, int on); void t3_config_rss(adapter_t *adapter, unsigned int rss_config, const u8 *cpus, const u16 *rspq); int t3_read_rss(adapter_t *adapter, u8 *lkup, u16 *map); int t3_mps_set_active_ports(adapter_t *adap, unsigned int port_mask); void t3_port_failover(adapter_t *adapter, int port); void t3_failover_done(adapter_t *adapter, int port); void t3_failover_clear(adapter_t *adapter); int t3_cim_ctl_blk_read(adapter_t *adap, unsigned int addr, unsigned int n, unsigned int *valp); int t3_mc7_bd_read(struct mc7 *mc7, unsigned int start, unsigned int n, u64 *buf); int t3_mac_reset(struct cmac *mac); void t3b_pcs_reset(struct cmac *mac); int t3_mac_enable(struct cmac *mac, int which); int t3_mac_disable(struct cmac *mac, int which); int t3_mac_set_mtu(struct cmac *mac, unsigned int mtu); int t3_mac_set_rx_mode(struct cmac *mac, struct t3_rx_mode *rm); int t3_mac_set_address(struct cmac *mac, unsigned int idx, u8 addr[6]); int t3_mac_set_num_ucast(struct cmac *mac, int n); const struct mac_stats *t3_mac_update_stats(struct cmac *mac); int t3_mac_set_speed_duplex_fc(struct cmac *mac, int speed, int duplex, int fc); int t3b2_mac_watchdog_task(struct cmac *mac); void t3_mc5_prep(adapter_t *adapter, struct mc5 *mc5, int mode); int t3_mc5_init(struct mc5 *mc5, unsigned int nservers, unsigned int nfilters, unsigned int nroutes); void t3_mc5_intr_handler(struct mc5 *mc5); int t3_read_mc5_range(const struct mc5 *mc5, unsigned int start, unsigned int n, u32 *buf); #ifdef CONFIG_CHELSIO_T3_CORE int t3_tp_set_coalescing_size(adapter_t *adap, unsigned int size, int psh); void t3_tp_set_max_rxsize(adapter_t *adap, unsigned int size); void t3_tp_set_offload_mode(adapter_t *adap, int enable); void t3_tp_get_mib_stats(adapter_t *adap, struct tp_mib_stats *tps); void t3_load_mtus(adapter_t *adap, unsigned short mtus[NMTUS], unsigned short alpha[NCCTRL_WIN], unsigned short beta[NCCTRL_WIN], unsigned short mtu_cap); void t3_read_hw_mtus(adapter_t *adap, unsigned short mtus[NMTUS]); void t3_get_cong_cntl_tab(adapter_t *adap, unsigned short incr[NMTUS][NCCTRL_WIN]); void t3_config_trace_filter(adapter_t *adapter, const struct trace_params *tp, int filter_index, int invert, int enable); int t3_config_sched(adapter_t *adap, unsigned int kbps, int sched); int t3_set_sched_ipg(adapter_t *adap, int sched, unsigned int ipg); void t3_get_tx_sched(adapter_t *adap, unsigned int sched, unsigned int *kbps, unsigned int *ipg); void t3_read_pace_tbl(adapter_t *adap, unsigned int pace_vals[NTX_SCHED]); void t3_set_pace_tbl(adapter_t *adap, unsigned int *pace_vals, unsigned int start, unsigned int n); #endif void t3_sge_prep(adapter_t *adap, struct sge_params *p); void t3_sge_init(adapter_t *adap, struct sge_params *p); int t3_sge_init_ecntxt(adapter_t *adapter, unsigned int id, int gts_enable, enum sge_context_type type, int respq, u64 base_addr, unsigned int size, unsigned int token, int gen, unsigned int cidx); int t3_sge_init_flcntxt(adapter_t *adapter, unsigned int id, int gts_enable, u64 base_addr, unsigned int size, unsigned int esize, unsigned int cong_thres, int gen, unsigned int cidx); int t3_sge_init_rspcntxt(adapter_t *adapter, unsigned int id, int irq_vec_idx, u64 base_addr, unsigned int size, unsigned int fl_thres, int gen, unsigned int cidx); int t3_sge_init_cqcntxt(adapter_t *adapter, unsigned int id, u64 base_addr, unsigned int size, int rspq, int ovfl_mode, unsigned int credits, unsigned int credit_thres); int t3_sge_enable_ecntxt(adapter_t *adapter, unsigned int id, int enable); int t3_sge_disable_fl(adapter_t *adapter, unsigned int id); int t3_sge_disable_rspcntxt(adapter_t *adapter, unsigned int id); int t3_sge_disable_cqcntxt(adapter_t *adapter, unsigned int id); int t3_sge_read_ecntxt(adapter_t *adapter, unsigned int id, u32 data[4]); int t3_sge_read_fl(adapter_t *adapter, unsigned int id, u32 data[4]); int t3_sge_read_cq(adapter_t *adapter, unsigned int id, u32 data[4]); int t3_sge_read_rspq(adapter_t *adapter, unsigned int id, u32 data[4]); int t3_sge_cqcntxt_op(adapter_t *adapter, unsigned int id, unsigned int op, unsigned int credits); void t3_mv88e1xxx_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr, const struct mdio_ops *mdio_ops); void t3_vsc8211_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr, const struct mdio_ops *mdio_ops); void t3_ael1002_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr, const struct mdio_ops *mdio_ops); void t3_ael1006_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr, const struct mdio_ops *mdio_ops); void t3_qt2045_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr, const struct mdio_ops *mdio_ops); void t3_xaui_direct_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr, const struct mdio_ops *mdio_ops); #endif /* __CHELSIO_COMMON_H */ Index: head/sys/dev/cxgb/common/cxgb_mc5.c =================================================================== --- head/sys/dev/cxgb/common/cxgb_mc5.c (revision 169977) +++ head/sys/dev/cxgb/common/cxgb_mc5.c (revision 169978) @@ -1,474 +1,477 @@ /************************************************************************** Copyright (c) 2007, Chelsio Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the Chelsio Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ***************************************************************************/ #include __FBSDID("$FreeBSD$"); #include #include enum { IDT75P52100 = 4, IDT75N43102 = 5 }; /* DBGI command mode */ enum { DBGI_MODE_MBUS = 0, DBGI_MODE_IDT52100 = 5 }; /* IDT 75P52100 commands */ #define IDT_CMD_READ 0 #define IDT_CMD_WRITE 1 #define IDT_CMD_SEARCH 2 #define IDT_CMD_LEARN 3 /* IDT LAR register address and value for 144-bit mode (low 32 bits) */ #define IDT_LAR_ADR0 0x180006 #define IDT_LAR_MODE144 0xffff0000 /* IDT SCR and SSR addresses (low 32 bits) */ #define IDT_SCR_ADR0 0x180000 #define IDT_SSR0_ADR0 0x180002 #define IDT_SSR1_ADR0 0x180004 /* IDT GMR base address (low 32 bits) */ #define IDT_GMR_BASE_ADR0 0x180020 /* IDT data and mask array base addresses (low 32 bits) */ #define IDT_DATARY_BASE_ADR0 0 #define IDT_MSKARY_BASE_ADR0 0x80000 /* IDT 75N43102 commands */ #define IDT4_CMD_SEARCH144 3 #define IDT4_CMD_WRITE 4 #define IDT4_CMD_READ 5 /* IDT 75N43102 SCR address (low 32 bits) */ #define IDT4_SCR_ADR0 0x3 /* IDT 75N43102 GMR base addresses (low 32 bits) */ #define IDT4_GMR_BASE0 0x10 #define IDT4_GMR_BASE1 0x20 #define IDT4_GMR_BASE2 0x30 /* IDT 75N43102 data and mask array base addresses (low 32 bits) */ #define IDT4_DATARY_BASE_ADR0 0x1000000 #define IDT4_MSKARY_BASE_ADR0 0x2000000 #define MAX_WRITE_ATTEMPTS 5 #define MAX_ROUTES 2048 /* * Issue a command to the TCAM and wait for its completion. The address and * any data required by the command must have been setup by the caller. */ static int mc5_cmd_write(adapter_t *adapter, u32 cmd) { t3_write_reg(adapter, A_MC5_DB_DBGI_REQ_CMD, cmd); return t3_wait_op_done(adapter, A_MC5_DB_DBGI_RSP_STATUS, F_DBGIRSPVALID, 1, MAX_WRITE_ATTEMPTS, 1); } static inline void dbgi_wr_addr3(adapter_t *adapter, u32 v1, u32 v2, u32 v3) { t3_write_reg(adapter, A_MC5_DB_DBGI_REQ_ADDR0, v1); t3_write_reg(adapter, A_MC5_DB_DBGI_REQ_ADDR1, v2); t3_write_reg(adapter, A_MC5_DB_DBGI_REQ_ADDR2, v3); } static inline void dbgi_wr_data3(adapter_t *adapter, u32 v1, u32 v2, u32 v3) { t3_write_reg(adapter, A_MC5_DB_DBGI_REQ_DATA0, v1); t3_write_reg(adapter, A_MC5_DB_DBGI_REQ_DATA1, v2); t3_write_reg(adapter, A_MC5_DB_DBGI_REQ_DATA2, v3); } static inline void dbgi_rd_rsp3(adapter_t *adapter, u32 *v1, u32 *v2, u32 *v3) { *v1 = t3_read_reg(adapter, A_MC5_DB_DBGI_RSP_DATA0); *v2 = t3_read_reg(adapter, A_MC5_DB_DBGI_RSP_DATA1); *v3 = t3_read_reg(adapter, A_MC5_DB_DBGI_RSP_DATA2); } /* * Write data to the TCAM register at address (0, 0, addr_lo) using the TCAM * command cmd. The data to be written must have been set up by the caller. * Returns -1 on failure, 0 on success. */ static int mc5_write(adapter_t *adapter, u32 addr_lo, u32 cmd) { t3_write_reg(adapter, A_MC5_DB_DBGI_REQ_ADDR0, addr_lo); if (mc5_cmd_write(adapter, cmd) == 0) return 0; CH_ERR(adapter, "MC5 timeout writing to TCAM address 0x%x\n", addr_lo); return -1; } static int init_mask_data_array(struct mc5 *mc5, u32 mask_array_base, u32 data_array_base, u32 write_cmd, int addr_shift) { unsigned int i; adapter_t *adap = mc5->adapter; /* * We need the size of the TCAM data and mask arrays in terms of * 72-bit entries. */ unsigned int size72 = mc5->tcam_size; unsigned int server_base = t3_read_reg(adap, A_MC5_DB_SERVER_INDEX); if (mc5->mode == MC5_MODE_144_BIT) { size72 *= 2; /* 1 144-bit entry is 2 72-bit entries */ server_base *= 2; } /* Clear the data array */ dbgi_wr_data3(adap, 0, 0, 0); for (i = 0; i < size72; i++) if (mc5_write(adap, data_array_base + (i << addr_shift), write_cmd)) return -1; /* Initialize the mask array. */ dbgi_wr_data3(adap, 0xffffffff, 0xffffffff, 0xff); for (i = 0; i < size72; i++) { if (i == server_base) /* entering server or routing region */ t3_write_reg(adap, A_MC5_DB_DBGI_REQ_DATA0, mc5->mode == MC5_MODE_144_BIT ? 0xfffffff9 : 0xfffffffd); if (mc5_write(adap, mask_array_base + (i << addr_shift), write_cmd)) return -1; } return 0; } static int init_idt52100(struct mc5 *mc5) { int i; adapter_t *adap = mc5->adapter; t3_write_reg(adap, A_MC5_DB_RSP_LATENCY, V_RDLAT(0x15) | V_LRNLAT(0x15) | V_SRCHLAT(0x15)); t3_write_reg(adap, A_MC5_DB_PART_ID_INDEX, 2); /* * Use GMRs 14-15 for ELOOKUP, GMRs 12-13 for SYN lookups, and * GMRs 8-9 for ACK- and AOPEN searches. */ t3_write_reg(adap, A_MC5_DB_POPEN_DATA_WR_CMD, IDT_CMD_WRITE); t3_write_reg(adap, A_MC5_DB_POPEN_MASK_WR_CMD, IDT_CMD_WRITE); t3_write_reg(adap, A_MC5_DB_AOPEN_SRCH_CMD, IDT_CMD_SEARCH); t3_write_reg(adap, A_MC5_DB_AOPEN_LRN_CMD, IDT_CMD_LEARN); t3_write_reg(adap, A_MC5_DB_SYN_SRCH_CMD, IDT_CMD_SEARCH | 0x6000); t3_write_reg(adap, A_MC5_DB_SYN_LRN_CMD, IDT_CMD_LEARN); t3_write_reg(adap, A_MC5_DB_ACK_SRCH_CMD, IDT_CMD_SEARCH); t3_write_reg(adap, A_MC5_DB_ACK_LRN_CMD, IDT_CMD_LEARN); t3_write_reg(adap, A_MC5_DB_ILOOKUP_CMD, IDT_CMD_SEARCH); t3_write_reg(adap, A_MC5_DB_ELOOKUP_CMD, IDT_CMD_SEARCH | 0x7000); t3_write_reg(adap, A_MC5_DB_DATA_WRITE_CMD, IDT_CMD_WRITE); t3_write_reg(adap, A_MC5_DB_DATA_READ_CMD, IDT_CMD_READ); /* Set DBGI command mode for IDT TCAM. */ t3_write_reg(adap, A_MC5_DB_DBGI_CONFIG, DBGI_MODE_IDT52100); /* Set up LAR */ dbgi_wr_data3(adap, IDT_LAR_MODE144, 0, 0); if (mc5_write(adap, IDT_LAR_ADR0, IDT_CMD_WRITE)) goto err; /* Set up SSRs */ dbgi_wr_data3(adap, 0xffffffff, 0xffffffff, 0); if (mc5_write(adap, IDT_SSR0_ADR0, IDT_CMD_WRITE) || mc5_write(adap, IDT_SSR1_ADR0, IDT_CMD_WRITE)) goto err; /* Set up GMRs */ for (i = 0; i < 32; ++i) { if (i >= 12 && i < 15) dbgi_wr_data3(adap, 0xfffffff9, 0xffffffff, 0xff); else if (i == 15) dbgi_wr_data3(adap, 0xfffffff9, 0xffff8007, 0xff); else dbgi_wr_data3(adap, 0xffffffff, 0xffffffff, 0xff); if (mc5_write(adap, IDT_GMR_BASE_ADR0 + i, IDT_CMD_WRITE)) goto err; } /* Set up SCR */ dbgi_wr_data3(adap, 1, 0, 0); if (mc5_write(adap, IDT_SCR_ADR0, IDT_CMD_WRITE)) goto err; return init_mask_data_array(mc5, IDT_MSKARY_BASE_ADR0, IDT_DATARY_BASE_ADR0, IDT_CMD_WRITE, 0); err: return -EIO; } static int init_idt43102(struct mc5 *mc5) { int i; adapter_t *adap = mc5->adapter; t3_write_reg(adap, A_MC5_DB_RSP_LATENCY, adap->params.rev == 0 ? V_RDLAT(0xd) | V_SRCHLAT(0x11) : V_RDLAT(0xd) | V_SRCHLAT(0x12)); /* * Use GMRs 24-25 for ELOOKUP, GMRs 20-21 for SYN lookups, and no mask * for ACK- and AOPEN searches. */ t3_write_reg(adap, A_MC5_DB_POPEN_DATA_WR_CMD, IDT4_CMD_WRITE); t3_write_reg(adap, A_MC5_DB_POPEN_MASK_WR_CMD, IDT4_CMD_WRITE); t3_write_reg(adap, A_MC5_DB_AOPEN_SRCH_CMD, IDT4_CMD_SEARCH144 | 0x3800); t3_write_reg(adap, A_MC5_DB_SYN_SRCH_CMD, IDT4_CMD_SEARCH144); t3_write_reg(adap, A_MC5_DB_ACK_SRCH_CMD, IDT4_CMD_SEARCH144 | 0x3800); t3_write_reg(adap, A_MC5_DB_ILOOKUP_CMD, IDT4_CMD_SEARCH144 | 0x3800); t3_write_reg(adap, A_MC5_DB_ELOOKUP_CMD, IDT4_CMD_SEARCH144 | 0x800); t3_write_reg(adap, A_MC5_DB_DATA_WRITE_CMD, IDT4_CMD_WRITE); t3_write_reg(adap, A_MC5_DB_DATA_READ_CMD, IDT4_CMD_READ); t3_write_reg(adap, A_MC5_DB_PART_ID_INDEX, 3); /* Set DBGI command mode for IDT TCAM. */ t3_write_reg(adap, A_MC5_DB_DBGI_CONFIG, DBGI_MODE_IDT52100); /* Set up GMRs */ dbgi_wr_data3(adap, 0xffffffff, 0xffffffff, 0xff); for (i = 0; i < 7; ++i) if (mc5_write(adap, IDT4_GMR_BASE0 + i, IDT4_CMD_WRITE)) goto err; for (i = 0; i < 4; ++i) if (mc5_write(adap, IDT4_GMR_BASE2 + i, IDT4_CMD_WRITE)) goto err; dbgi_wr_data3(adap, 0xfffffff9, 0xffffffff, 0xff); if (mc5_write(adap, IDT4_GMR_BASE1, IDT4_CMD_WRITE) || mc5_write(adap, IDT4_GMR_BASE1 + 1, IDT4_CMD_WRITE) || mc5_write(adap, IDT4_GMR_BASE1 + 4, IDT4_CMD_WRITE)) goto err; dbgi_wr_data3(adap, 0xfffffff9, 0xffff8007, 0xff); if (mc5_write(adap, IDT4_GMR_BASE1 + 5, IDT4_CMD_WRITE)) goto err; /* Set up SCR */ dbgi_wr_data3(adap, 0xf0000000, 0, 0); if (mc5_write(adap, IDT4_SCR_ADR0, IDT4_CMD_WRITE)) goto err; return init_mask_data_array(mc5, IDT4_MSKARY_BASE_ADR0, IDT4_DATARY_BASE_ADR0, IDT4_CMD_WRITE, 1); err: return -EIO; } /* Put MC5 in DBGI mode. */ static inline void mc5_dbgi_mode_enable(const struct mc5 *mc5) { t3_write_reg(mc5->adapter, A_MC5_DB_CONFIG, V_TMMODE(mc5->mode == MC5_MODE_72_BIT) | F_DBGIEN); } /* Put MC5 in M-Bus mode. */ static void mc5_dbgi_mode_disable(const struct mc5 *mc5) { t3_write_reg(mc5->adapter, A_MC5_DB_CONFIG, V_TMMODE(mc5->mode == MC5_MODE_72_BIT) | V_COMPEN(mc5->mode == MC5_MODE_72_BIT) | V_PRTYEN(mc5->parity_enabled) | F_MBUSEN); } /* * Initialization that requires the OS and protocol layers to already * be intialized goes here. */ int t3_mc5_init(struct mc5 *mc5, unsigned int nservers, unsigned int nfilters, unsigned int nroutes) { u32 cfg; int err; unsigned int tcam_size = mc5->tcam_size; adapter_t *adap = mc5->adapter; + if (tcam_size == 0) + return 0; + if (nroutes > MAX_ROUTES || nroutes + nservers + nfilters > tcam_size) return -EINVAL; /* Reset the TCAM */ cfg = t3_read_reg(adap, A_MC5_DB_CONFIG) & ~F_TMMODE; cfg |= V_TMMODE(mc5->mode == MC5_MODE_72_BIT) | F_TMRST; t3_write_reg(adap, A_MC5_DB_CONFIG, cfg); if (t3_wait_op_done(adap, A_MC5_DB_CONFIG, F_TMRDY, 1, 500, 0)) { CH_ERR(adap, "TCAM reset timed out\n"); return -1; } t3_write_reg(adap, A_MC5_DB_ROUTING_TABLE_INDEX, tcam_size - nroutes); t3_write_reg(adap, A_MC5_DB_FILTER_TABLE, tcam_size - nroutes - nfilters); t3_write_reg(adap, A_MC5_DB_SERVER_INDEX, tcam_size - nroutes - nfilters - nservers); mc5->parity_enabled = 1; /* All the TCAM addresses we access have only the low 32 bits non 0 */ t3_write_reg(adap, A_MC5_DB_DBGI_REQ_ADDR1, 0); t3_write_reg(adap, A_MC5_DB_DBGI_REQ_ADDR2, 0); mc5_dbgi_mode_enable(mc5); switch (mc5->part_type) { case IDT75P52100: err = init_idt52100(mc5); break; case IDT75N43102: err = init_idt43102(mc5); break; default: CH_ERR(adap, "Unsupported TCAM type %d\n", mc5->part_type); err = -EINVAL; break; } mc5_dbgi_mode_disable(mc5); return err; } /* * read_mc5_range - dump a part of the memory managed by MC5 * @mc5: the MC5 handle * @start: the start address for the dump * @n: number of 72-bit words to read * @buf: result buffer * * Read n 72-bit words from MC5 memory from the given start location. */ int t3_read_mc5_range(const struct mc5 *mc5, unsigned int start, unsigned int n, u32 *buf) { u32 read_cmd; int err = 0; adapter_t *adap = mc5->adapter; if (mc5->part_type == IDT75P52100) read_cmd = IDT_CMD_READ; else if (mc5->part_type == IDT75N43102) read_cmd = IDT4_CMD_READ; else return -EINVAL; mc5_dbgi_mode_enable(mc5); while (n--) { t3_write_reg(adap, A_MC5_DB_DBGI_REQ_ADDR0, start++); if (mc5_cmd_write(adap, read_cmd)) { err = -EIO; break; } dbgi_rd_rsp3(adap, buf + 2, buf + 1, buf); buf += 3; } mc5_dbgi_mode_disable(mc5); return 0; } #define MC5_INT_FATAL (F_PARITYERR | F_REQQPARERR | F_DISPQPARERR) /* * MC5 interrupt handler */ void t3_mc5_intr_handler(struct mc5 *mc5) { adapter_t *adap = mc5->adapter; u32 cause = t3_read_reg(adap, A_MC5_DB_INT_CAUSE); if ((cause & F_PARITYERR) && mc5->parity_enabled) { CH_ALERT(adap, "MC5 parity error\n"); mc5->stats.parity_err++; } if (cause & F_REQQPARERR) { CH_ALERT(adap, "MC5 request queue parity error\n"); mc5->stats.reqq_parity_err++; } if (cause & F_DISPQPARERR) { CH_ALERT(adap, "MC5 dispatch queue parity error\n"); mc5->stats.dispq_parity_err++; } if (cause & F_ACTRGNFULL) mc5->stats.active_rgn_full++; if (cause & F_NFASRCHFAIL) mc5->stats.nfa_srch_err++; if (cause & F_UNKNOWNCMD) mc5->stats.unknown_cmd++; if (cause & F_DELACTEMPTY) mc5->stats.del_act_empty++; if (cause & MC5_INT_FATAL) t3_fatal_err(adap); t3_write_reg(adap, A_MC5_DB_INT_CAUSE, cause); } void __devinit t3_mc5_prep(adapter_t *adapter, struct mc5 *mc5, int mode) { #define K * 1024 static unsigned int tcam_part_size[] = { /* in K 72-bit entries */ 64 K, 128 K, 256 K, 32 K }; #undef K u32 cfg = t3_read_reg(adapter, A_MC5_DB_CONFIG); mc5->adapter = adapter; mc5->mode = (unsigned char) mode; mc5->part_type = (unsigned char) G_TMTYPE(cfg); if (cfg & F_TMTYPEHI) mc5->part_type |= 4; mc5->tcam_size = tcam_part_size[G_TMPARTSIZE(cfg)]; if (mode == MC5_MODE_144_BIT) mc5->tcam_size /= 2; } Index: head/sys/dev/cxgb/common/cxgb_t3_cpl.h =================================================================== --- head/sys/dev/cxgb/common/cxgb_t3_cpl.h (revision 169977) +++ head/sys/dev/cxgb/common/cxgb_t3_cpl.h (revision 169978) @@ -1,1490 +1,1546 @@ /************************************************************************** Copyright (c) 2007, Chelsio Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the Chelsio Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. $FreeBSD$ ***************************************************************************/ #ifndef T3_CPL_H #define T3_CPL_H #if !defined(__LITTLE_ENDIAN_BITFIELD) && !defined(__BIG_ENDIAN_BITFIELD) # include #endif enum CPL_opcode { CPL_PASS_OPEN_REQ = 0x1, CPL_PASS_ACCEPT_RPL = 0x2, CPL_ACT_OPEN_REQ = 0x3, CPL_SET_TCB = 0x4, CPL_SET_TCB_FIELD = 0x5, CPL_GET_TCB = 0x6, CPL_PCMD = 0x7, CPL_CLOSE_CON_REQ = 0x8, CPL_CLOSE_LISTSRV_REQ = 0x9, CPL_ABORT_REQ = 0xA, CPL_ABORT_RPL = 0xB, CPL_TX_DATA = 0xC, CPL_RX_DATA_ACK = 0xD, CPL_TX_PKT = 0xE, CPL_RTE_DELETE_REQ = 0xF, CPL_RTE_WRITE_REQ = 0x10, CPL_RTE_READ_REQ = 0x11, CPL_L2T_WRITE_REQ = 0x12, CPL_L2T_READ_REQ = 0x13, CPL_SMT_WRITE_REQ = 0x14, CPL_SMT_READ_REQ = 0x15, CPL_TX_PKT_LSO = 0x16, CPL_PCMD_READ = 0x17, CPL_BARRIER = 0x18, CPL_TID_RELEASE = 0x1A, CPL_CLOSE_LISTSRV_RPL = 0x20, CPL_ERROR = 0x21, CPL_GET_TCB_RPL = 0x22, CPL_L2T_WRITE_RPL = 0x23, CPL_PCMD_READ_RPL = 0x24, CPL_PCMD_RPL = 0x25, CPL_PEER_CLOSE = 0x26, CPL_RTE_DELETE_RPL = 0x27, CPL_RTE_WRITE_RPL = 0x28, CPL_RX_DDP_COMPLETE = 0x29, CPL_RX_PHYS_ADDR = 0x2A, CPL_RX_PKT = 0x2B, CPL_RX_URG_NOTIFY = 0x2C, CPL_SET_TCB_RPL = 0x2D, CPL_SMT_WRITE_RPL = 0x2E, CPL_TX_DATA_ACK = 0x2F, CPL_ABORT_REQ_RSS = 0x30, CPL_ABORT_RPL_RSS = 0x31, CPL_CLOSE_CON_RPL = 0x32, CPL_ISCSI_HDR = 0x33, CPL_L2T_READ_RPL = 0x34, CPL_RDMA_CQE = 0x35, CPL_RDMA_CQE_READ_RSP = 0x36, CPL_RDMA_CQE_ERR = 0x37, CPL_RTE_READ_RPL = 0x38, CPL_RX_DATA = 0x39, CPL_ACT_OPEN_RPL = 0x40, CPL_PASS_OPEN_RPL = 0x41, CPL_RX_DATA_DDP = 0x42, CPL_SMT_READ_RPL = 0x43, CPL_ACT_ESTABLISH = 0x50, CPL_PASS_ESTABLISH = 0x51, CPL_PASS_ACCEPT_REQ = 0x70, CPL_ASYNC_NOTIF = 0x80, /* fake opcode for async notifications */ CPL_TX_DMA_ACK = 0xA0, CPL_RDMA_READ_REQ = 0xA1, CPL_RDMA_TERMINATE = 0xA2, CPL_TRACE_PKT = 0xA3, CPL_RDMA_EC_STATUS = 0xA5, NUM_CPL_CMDS /* must be last and previous entries must be sorted */ }; enum CPL_error { CPL_ERR_NONE = 0, CPL_ERR_TCAM_PARITY = 1, CPL_ERR_TCAM_FULL = 3, CPL_ERR_CONN_RESET = 20, CPL_ERR_CONN_EXIST = 22, CPL_ERR_ARP_MISS = 23, CPL_ERR_BAD_SYN = 24, CPL_ERR_CONN_TIMEDOUT = 30, CPL_ERR_XMIT_TIMEDOUT = 31, CPL_ERR_PERSIST_TIMEDOUT = 32, CPL_ERR_FINWAIT2_TIMEDOUT = 33, CPL_ERR_KEEPALIVE_TIMEDOUT = 34, CPL_ERR_RTX_NEG_ADVICE = 35, CPL_ERR_PERSIST_NEG_ADVICE = 36, CPL_ERR_ABORT_FAILED = 42, CPL_ERR_GENERAL = 99 }; enum { CPL_CONN_POLICY_AUTO = 0, CPL_CONN_POLICY_ASK = 1, CPL_CONN_POLICY_DENY = 3 }; enum { ULP_MODE_NONE = 0, ULP_MODE_TCP_DDP = 1, ULP_MODE_ISCSI = 2, ULP_MODE_RDMA = 4, ULP_MODE_TCPDDP = 5 }; enum { ULP_CRC_HEADER = 1 << 0, ULP_CRC_DATA = 1 << 1 }; enum { CPL_PASS_OPEN_ACCEPT, CPL_PASS_OPEN_REJECT }; enum { CPL_ABORT_SEND_RST = 0, CPL_ABORT_NO_RST, CPL_ABORT_POST_CLOSE_REQ = 2 }; enum { /* TX_PKT_LSO ethernet types */ CPL_ETH_II, CPL_ETH_II_VLAN, CPL_ETH_802_3, CPL_ETH_802_3_VLAN }; enum { /* TCP congestion control algorithms */ CONG_ALG_RENO, CONG_ALG_TAHOE, CONG_ALG_NEWRENO, CONG_ALG_HIGHSPEED }; enum { /* RSS hash type */ RSS_HASH_NONE = 0, RSS_HASH_2_TUPLE = 1 << 0, RSS_HASH_4_TUPLE = 1 << 1 }; union opcode_tid { __be32 opcode_tid; __u8 opcode; }; #define S_OPCODE 24 #define V_OPCODE(x) ((x) << S_OPCODE) #define G_OPCODE(x) (((x) >> S_OPCODE) & 0xFF) #define G_TID(x) ((x) & 0xFFFFFF) #define S_HASHTYPE 22 #define M_HASHTYPE 0x3 #define G_HASHTYPE(x) (((x) >> S_HASHTYPE) & M_HASHTYPE) #define S_QNUM 0 #define G_QNUM(x) (((x) >> S_QNUM) & 0xFFFF) /* tid is assumed to be 24-bits */ #define MK_OPCODE_TID(opcode, tid) (V_OPCODE(opcode) | (tid)) #define OPCODE_TID(cmd) ((cmd)->ot.opcode_tid) /* extract the TID from a CPL command */ #define GET_TID(cmd) (G_TID(ntohl(OPCODE_TID(cmd)))) struct tcp_options { __be16 mss; __u8 wsf; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 :5; __u8 ecn:1; __u8 sack:1; __u8 tstamp:1; #else __u8 tstamp:1; __u8 sack:1; __u8 ecn:1; __u8 :5; #endif }; struct rss_header { __u8 opcode; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 cpu_idx:6; __u8 hash_type:2; #else __u8 hash_type:2; __u8 cpu_idx:6; #endif __be16 cq_idx; __be32 rss_hash_val; }; #ifndef CHELSIO_FW struct work_request_hdr { __be32 wr_hi; __be32 wr_lo; }; /* wr_hi fields */ #define S_WR_SGE_CREDITS 0 #define M_WR_SGE_CREDITS 0xFF #define V_WR_SGE_CREDITS(x) ((x) << S_WR_SGE_CREDITS) #define G_WR_SGE_CREDITS(x) (((x) >> S_WR_SGE_CREDITS) & M_WR_SGE_CREDITS) #define S_WR_SGLSFLT 8 #define M_WR_SGLSFLT 0xFF #define V_WR_SGLSFLT(x) ((x) << S_WR_SGLSFLT) #define G_WR_SGLSFLT(x) (((x) >> S_WR_SGLSFLT) & M_WR_SGLSFLT) #define S_WR_BCNTLFLT 16 #define M_WR_BCNTLFLT 0xF #define V_WR_BCNTLFLT(x) ((x) << S_WR_BCNTLFLT) #define G_WR_BCNTLFLT(x) (((x) >> S_WR_BCNTLFLT) & M_WR_BCNTLFLT) +/* Applicable to BYPASS WRs only: the uP will added a CPL_BARRIER before + * and after the BYPASS WR if the ATOMIC bit is set. + */ +#define S_WR_ATOMIC 16 +#define V_WR_ATOMIC(x) ((x) << S_WR_ATOMIC) +#define F_WR_ATOMIC V_WR_ATOMIC(1U) + +/* Applicable to BYPASS WRs only: the uP will flush buffered non abort + * related WRs. + */ +#define S_WR_FLUSH 17 +#define V_WR_FLUSH(x) ((x) << S_WR_FLUSH) +#define F_WR_FLUSH V_WR_FLUSH(1U) + #define S_WR_DATATYPE 20 #define V_WR_DATATYPE(x) ((x) << S_WR_DATATYPE) #define F_WR_DATATYPE V_WR_DATATYPE(1U) #define S_WR_COMPL 21 #define V_WR_COMPL(x) ((x) << S_WR_COMPL) #define F_WR_COMPL V_WR_COMPL(1U) #define S_WR_EOP 22 #define V_WR_EOP(x) ((x) << S_WR_EOP) #define F_WR_EOP V_WR_EOP(1U) #define S_WR_SOP 23 #define V_WR_SOP(x) ((x) << S_WR_SOP) #define F_WR_SOP V_WR_SOP(1U) #define S_WR_OP 24 #define M_WR_OP 0xFF #define V_WR_OP(x) ((x) << S_WR_OP) #define G_WR_OP(x) (((x) >> S_WR_OP) & M_WR_OP) /* wr_lo fields */ #define S_WR_LEN 0 #define M_WR_LEN 0xFF #define V_WR_LEN(x) ((x) << S_WR_LEN) #define G_WR_LEN(x) (((x) >> S_WR_LEN) & M_WR_LEN) #define S_WR_TID 8 #define M_WR_TID 0xFFFFF #define V_WR_TID(x) ((x) << S_WR_TID) #define G_WR_TID(x) (((x) >> S_WR_TID) & M_WR_TID) #define S_WR_CR_FLUSH 30 #define V_WR_CR_FLUSH(x) ((x) << S_WR_CR_FLUSH) #define F_WR_CR_FLUSH V_WR_CR_FLUSH(1U) #define S_WR_GEN 31 #define V_WR_GEN(x) ((x) << S_WR_GEN) #define F_WR_GEN V_WR_GEN(1U) # define WR_HDR struct work_request_hdr wr # define RSS_HDR #else # define WR_HDR # define RSS_HDR struct rss_header rss_hdr; #endif /* option 0 lower-half fields */ #define S_CPL_STATUS 0 #define M_CPL_STATUS 0xFF #define V_CPL_STATUS(x) ((x) << S_CPL_STATUS) #define G_CPL_STATUS(x) (((x) >> S_CPL_STATUS) & M_CPL_STATUS) #define S_INJECT_TIMER 6 #define V_INJECT_TIMER(x) ((x) << S_INJECT_TIMER) #define F_INJECT_TIMER V_INJECT_TIMER(1U) #define S_NO_OFFLOAD 7 #define V_NO_OFFLOAD(x) ((x) << S_NO_OFFLOAD) #define F_NO_OFFLOAD V_NO_OFFLOAD(1U) #define S_ULP_MODE 8 #define M_ULP_MODE 0xF #define V_ULP_MODE(x) ((x) << S_ULP_MODE) #define G_ULP_MODE(x) (((x) >> S_ULP_MODE) & M_ULP_MODE) #define S_RCV_BUFSIZ 12 #define M_RCV_BUFSIZ 0x3FFF #define V_RCV_BUFSIZ(x) ((x) << S_RCV_BUFSIZ) #define G_RCV_BUFSIZ(x) (((x) >> S_RCV_BUFSIZ) & M_RCV_BUFSIZ) #define S_TOS 26 #define M_TOS 0x3F #define V_TOS(x) ((x) << S_TOS) #define G_TOS(x) (((x) >> S_TOS) & M_TOS) /* option 0 upper-half fields */ #define S_DELACK 0 #define V_DELACK(x) ((x) << S_DELACK) #define F_DELACK V_DELACK(1U) #define S_NO_CONG 1 #define V_NO_CONG(x) ((x) << S_NO_CONG) #define F_NO_CONG V_NO_CONG(1U) #define S_SRC_MAC_SEL 2 #define M_SRC_MAC_SEL 0x3 #define V_SRC_MAC_SEL(x) ((x) << S_SRC_MAC_SEL) #define G_SRC_MAC_SEL(x) (((x) >> S_SRC_MAC_SEL) & M_SRC_MAC_SEL) #define S_L2T_IDX 4 #define M_L2T_IDX 0x7FF #define V_L2T_IDX(x) ((x) << S_L2T_IDX) #define G_L2T_IDX(x) (((x) >> S_L2T_IDX) & M_L2T_IDX) #define S_TX_CHANNEL 15 #define V_TX_CHANNEL(x) ((x) << S_TX_CHANNEL) #define F_TX_CHANNEL V_TX_CHANNEL(1U) #define S_TCAM_BYPASS 16 #define V_TCAM_BYPASS(x) ((x) << S_TCAM_BYPASS) #define F_TCAM_BYPASS V_TCAM_BYPASS(1U) #define S_NAGLE 17 #define V_NAGLE(x) ((x) << S_NAGLE) #define F_NAGLE V_NAGLE(1U) #define S_WND_SCALE 18 #define M_WND_SCALE 0xF #define V_WND_SCALE(x) ((x) << S_WND_SCALE) #define G_WND_SCALE(x) (((x) >> S_WND_SCALE) & M_WND_SCALE) #define S_KEEP_ALIVE 22 #define V_KEEP_ALIVE(x) ((x) << S_KEEP_ALIVE) #define F_KEEP_ALIVE V_KEEP_ALIVE(1U) #define S_MAX_RETRANS 23 #define M_MAX_RETRANS 0xF #define V_MAX_RETRANS(x) ((x) << S_MAX_RETRANS) #define G_MAX_RETRANS(x) (((x) >> S_MAX_RETRANS) & M_MAX_RETRANS) #define S_MAX_RETRANS_OVERRIDE 27 #define V_MAX_RETRANS_OVERRIDE(x) ((x) << S_MAX_RETRANS_OVERRIDE) #define F_MAX_RETRANS_OVERRIDE V_MAX_RETRANS_OVERRIDE(1U) #define S_MSS_IDX 28 #define M_MSS_IDX 0xF #define V_MSS_IDX(x) ((x) << S_MSS_IDX) #define G_MSS_IDX(x) (((x) >> S_MSS_IDX) & M_MSS_IDX) /* option 1 fields */ #define S_RSS_ENABLE 0 #define V_RSS_ENABLE(x) ((x) << S_RSS_ENABLE) #define F_RSS_ENABLE V_RSS_ENABLE(1U) #define S_RSS_MASK_LEN 1 #define M_RSS_MASK_LEN 0x7 #define V_RSS_MASK_LEN(x) ((x) << S_RSS_MASK_LEN) #define G_RSS_MASK_LEN(x) (((x) >> S_RSS_MASK_LEN) & M_RSS_MASK_LEN) #define S_CPU_IDX 4 #define M_CPU_IDX 0x3F #define V_CPU_IDX(x) ((x) << S_CPU_IDX) #define G_CPU_IDX(x) (((x) >> S_CPU_IDX) & M_CPU_IDX) #define S_MAC_MATCH_VALID 18 #define V_MAC_MATCH_VALID(x) ((x) << S_MAC_MATCH_VALID) #define F_MAC_MATCH_VALID V_MAC_MATCH_VALID(1U) #define S_CONN_POLICY 19 #define M_CONN_POLICY 0x3 #define V_CONN_POLICY(x) ((x) << S_CONN_POLICY) #define G_CONN_POLICY(x) (((x) >> S_CONN_POLICY) & M_CONN_POLICY) #define S_SYN_DEFENSE 21 #define V_SYN_DEFENSE(x) ((x) << S_SYN_DEFENSE) #define F_SYN_DEFENSE V_SYN_DEFENSE(1U) #define S_VLAN_PRI 22 #define M_VLAN_PRI 0x3 #define V_VLAN_PRI(x) ((x) << S_VLAN_PRI) #define G_VLAN_PRI(x) (((x) >> S_VLAN_PRI) & M_VLAN_PRI) #define S_VLAN_PRI_VALID 24 #define V_VLAN_PRI_VALID(x) ((x) << S_VLAN_PRI_VALID) #define F_VLAN_PRI_VALID V_VLAN_PRI_VALID(1U) #define S_PKT_TYPE 25 #define M_PKT_TYPE 0x3 #define V_PKT_TYPE(x) ((x) << S_PKT_TYPE) #define G_PKT_TYPE(x) (((x) >> S_PKT_TYPE) & M_PKT_TYPE) #define S_MAC_MATCH 27 #define M_MAC_MATCH 0x1F #define V_MAC_MATCH(x) ((x) << S_MAC_MATCH) #define G_MAC_MATCH(x) (((x) >> S_MAC_MATCH) & M_MAC_MATCH) /* option 2 fields */ #define S_CPU_INDEX 0 #define M_CPU_INDEX 0x7F #define V_CPU_INDEX(x) ((x) << S_CPU_INDEX) #define G_CPU_INDEX(x) (((x) >> S_CPU_INDEX) & M_CPU_INDEX) #define S_CPU_INDEX_VALID 7 #define V_CPU_INDEX_VALID(x) ((x) << S_CPU_INDEX_VALID) #define F_CPU_INDEX_VALID V_CPU_INDEX_VALID(1U) #define S_RX_COALESCE 8 #define M_RX_COALESCE 0x3 #define V_RX_COALESCE(x) ((x) << S_RX_COALESCE) #define G_RX_COALESCE(x) (((x) >> S_RX_COALESCE) & M_RX_COALESCE) #define S_RX_COALESCE_VALID 10 #define V_RX_COALESCE_VALID(x) ((x) << S_RX_COALESCE_VALID) #define F_RX_COALESCE_VALID V_RX_COALESCE_VALID(1U) #define S_CONG_CONTROL_FLAVOR 11 #define M_CONG_CONTROL_FLAVOR 0x3 #define V_CONG_CONTROL_FLAVOR(x) ((x) << S_CONG_CONTROL_FLAVOR) #define G_CONG_CONTROL_FLAVOR(x) (((x) >> S_CONG_CONTROL_FLAVOR) & M_CONG_CONTROL_FLAVOR) #define S_PACING_FLAVOR 13 #define M_PACING_FLAVOR 0x3 #define V_PACING_FLAVOR(x) ((x) << S_PACING_FLAVOR) #define G_PACING_FLAVOR(x) (((x) >> S_PACING_FLAVOR) & M_PACING_FLAVOR) #define S_FLAVORS_VALID 15 #define V_FLAVORS_VALID(x) ((x) << S_FLAVORS_VALID) #define F_FLAVORS_VALID V_FLAVORS_VALID(1U) #define S_RX_FC_DISABLE 16 #define V_RX_FC_DISABLE(x) ((x) << S_RX_FC_DISABLE) #define F_RX_FC_DISABLE V_RX_FC_DISABLE(1U) #define S_RX_FC_VALID 17 #define V_RX_FC_VALID(x) ((x) << S_RX_FC_VALID) #define F_RX_FC_VALID V_RX_FC_VALID(1U) struct cpl_pass_open_req { WR_HDR; union opcode_tid ot; __be16 local_port; __be16 peer_port; __be32 local_ip; __be32 peer_ip; __be32 opt0h; __be32 opt0l; __be32 peer_netmask; __be32 opt1; }; struct cpl_pass_open_rpl { RSS_HDR union opcode_tid ot; __be16 local_port; __be16 peer_port; __be32 local_ip; __be32 peer_ip; __u8 resvd[7]; __u8 status; }; struct cpl_pass_establish { RSS_HDR union opcode_tid ot; __be16 local_port; __be16 peer_port; __be32 local_ip; __be32 peer_ip; __be32 tos_tid; __be16 l2t_idx; __be16 tcp_opt; __be32 snd_isn; __be32 rcv_isn; }; /* cpl_pass_establish.tos_tid fields */ #define S_PASS_OPEN_TID 0 #define M_PASS_OPEN_TID 0xFFFFFF #define V_PASS_OPEN_TID(x) ((x) << S_PASS_OPEN_TID) #define G_PASS_OPEN_TID(x) (((x) >> S_PASS_OPEN_TID) & M_PASS_OPEN_TID) #define S_PASS_OPEN_TOS 24 #define M_PASS_OPEN_TOS 0xFF #define V_PASS_OPEN_TOS(x) ((x) << S_PASS_OPEN_TOS) #define G_PASS_OPEN_TOS(x) (((x) >> S_PASS_OPEN_TOS) & M_PASS_OPEN_TOS) /* cpl_pass_establish.l2t_idx fields */ #define S_L2T_IDX16 5 #define M_L2T_IDX16 0x7FF #define V_L2T_IDX16(x) ((x) << S_L2T_IDX16) #define G_L2T_IDX16(x) (((x) >> S_L2T_IDX16) & M_L2T_IDX16) /* cpl_pass_establish.tcp_opt fields (also applies act_open_establish) */ #define G_TCPOPT_WSCALE_OK(x) (((x) >> 5) & 1) #define G_TCPOPT_SACK(x) (((x) >> 6) & 1) #define G_TCPOPT_TSTAMP(x) (((x) >> 7) & 1) #define G_TCPOPT_SND_WSCALE(x) (((x) >> 8) & 0xf) #define G_TCPOPT_MSS(x) (((x) >> 12) & 0xf) struct cpl_pass_accept_req { RSS_HDR union opcode_tid ot; __be16 local_port; __be16 peer_port; __be32 local_ip; __be32 peer_ip; __be32 tos_tid; struct tcp_options tcp_options; __u8 dst_mac[6]; __be16 vlan_tag; __u8 src_mac[6]; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 :3; __u8 addr_idx:3; __u8 port_idx:1; __u8 exact_match:1; #else __u8 exact_match:1; __u8 port_idx:1; __u8 addr_idx:3; __u8 :3; #endif __u8 rsvd; __be32 rcv_isn; __be32 rsvd2; }; struct cpl_pass_accept_rpl { WR_HDR; union opcode_tid ot; __be32 opt2; __be32 rsvd; __be32 peer_ip; __be32 opt0h; __be32 opt0l_status; }; struct cpl_act_open_req { WR_HDR; union opcode_tid ot; __be16 local_port; __be16 peer_port; __be32 local_ip; __be32 peer_ip; __be32 opt0h; __be32 opt0l; __be32 params; __be32 opt2; }; /* cpl_act_open_req.params fields */ #define S_AOPEN_VLAN_PRI 9 #define M_AOPEN_VLAN_PRI 0x3 #define V_AOPEN_VLAN_PRI(x) ((x) << S_AOPEN_VLAN_PRI) #define G_AOPEN_VLAN_PRI(x) (((x) >> S_AOPEN_VLAN_PRI) & M_AOPEN_VLAN_PRI) #define S_AOPEN_VLAN_PRI_VALID 11 #define V_AOPEN_VLAN_PRI_VALID(x) ((x) << S_AOPEN_VLAN_PRI_VALID) #define F_AOPEN_VLAN_PRI_VALID V_AOPEN_VLAN_PRI_VALID(1U) #define S_AOPEN_PKT_TYPE 12 #define M_AOPEN_PKT_TYPE 0x3 #define V_AOPEN_PKT_TYPE(x) ((x) << S_AOPEN_PKT_TYPE) #define G_AOPEN_PKT_TYPE(x) (((x) >> S_AOPEN_PKT_TYPE) & M_AOPEN_PKT_TYPE) #define S_AOPEN_MAC_MATCH 14 #define M_AOPEN_MAC_MATCH 0x1F #define V_AOPEN_MAC_MATCH(x) ((x) << S_AOPEN_MAC_MATCH) #define G_AOPEN_MAC_MATCH(x) (((x) >> S_AOPEN_MAC_MATCH) & M_AOPEN_MAC_MATCH) #define S_AOPEN_MAC_MATCH_VALID 19 #define V_AOPEN_MAC_MATCH_VALID(x) ((x) << S_AOPEN_MAC_MATCH_VALID) #define F_AOPEN_MAC_MATCH_VALID V_AOPEN_MAC_MATCH_VALID(1U) #define S_AOPEN_IFF_VLAN 20 #define M_AOPEN_IFF_VLAN 0xFFF #define V_AOPEN_IFF_VLAN(x) ((x) << S_AOPEN_IFF_VLAN) #define G_AOPEN_IFF_VLAN(x) (((x) >> S_AOPEN_IFF_VLAN) & M_AOPEN_IFF_VLAN) struct cpl_act_open_rpl { RSS_HDR union opcode_tid ot; __be16 local_port; __be16 peer_port; __be32 local_ip; __be32 peer_ip; __be32 atid; __u8 rsvd[3]; __u8 status; }; struct cpl_act_establish { RSS_HDR union opcode_tid ot; __be16 local_port; __be16 peer_port; __be32 local_ip; __be32 peer_ip; __be32 tos_tid; __be16 l2t_idx; __be16 tcp_opt; __be32 snd_isn; __be32 rcv_isn; }; struct cpl_get_tcb { WR_HDR; union opcode_tid ot; __be16 cpuno; __be16 rsvd; }; struct cpl_get_tcb_rpl { RSS_HDR union opcode_tid ot; __u8 rsvd; __u8 status; __be16 len; }; struct cpl_set_tcb { WR_HDR; union opcode_tid ot; __u8 reply; __u8 cpu_idx; __be16 len; }; /* cpl_set_tcb.reply fields */ #define S_NO_REPLY 7 #define V_NO_REPLY(x) ((x) << S_NO_REPLY) #define F_NO_REPLY V_NO_REPLY(1U) struct cpl_set_tcb_field { WR_HDR; union opcode_tid ot; __u8 reply; __u8 cpu_idx; __be16 word; __be64 mask; __be64 val; }; struct cpl_set_tcb_rpl { RSS_HDR union opcode_tid ot; __u8 rsvd[3]; __u8 status; }; struct cpl_pcmd { WR_HDR; union opcode_tid ot; __u8 rsvd[3]; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 src:1; __u8 bundle:1; __u8 channel:1; __u8 :5; #else __u8 :5; __u8 channel:1; __u8 bundle:1; __u8 src:1; #endif __be32 pcmd_parm[2]; }; struct cpl_pcmd_reply { RSS_HDR union opcode_tid ot; __u8 status; __u8 rsvd; __be16 len; }; struct cpl_close_con_req { WR_HDR; union opcode_tid ot; __be32 rsvd; }; struct cpl_close_con_rpl { RSS_HDR union opcode_tid ot; __u8 rsvd[3]; __u8 status; __be32 snd_nxt; __be32 rcv_nxt; }; struct cpl_close_listserv_req { WR_HDR; union opcode_tid ot; __u8 rsvd0; __u8 cpu_idx; __be16 rsvd1; }; struct cpl_close_listserv_rpl { RSS_HDR union opcode_tid ot; __u8 rsvd[3]; __u8 status; }; struct cpl_abort_req_rss { RSS_HDR union opcode_tid ot; __be32 rsvd0; __u8 rsvd1; __u8 status; __u8 rsvd2[6]; }; struct cpl_abort_req { WR_HDR; union opcode_tid ot; __be32 rsvd0; __u8 rsvd1; __u8 cmd; __u8 rsvd2[6]; }; struct cpl_abort_rpl_rss { RSS_HDR union opcode_tid ot; __be32 rsvd0; __u8 rsvd1; __u8 status; __u8 rsvd2[6]; }; struct cpl_abort_rpl { WR_HDR; union opcode_tid ot; __be32 rsvd0; __u8 rsvd1; __u8 cmd; __u8 rsvd2[6]; }; struct cpl_peer_close { RSS_HDR union opcode_tid ot; __be32 rcv_nxt; }; struct tx_data_wr { __be32 wr_hi; __be32 wr_lo; __be32 len; __be32 flags; __be32 sndseq; __be32 param; }; /* tx_data_wr.param fields */ #define S_TX_PORT 0 #define M_TX_PORT 0x7 #define V_TX_PORT(x) ((x) << S_TX_PORT) #define G_TX_PORT(x) (((x) >> S_TX_PORT) & M_TX_PORT) #define S_TX_MSS 4 #define M_TX_MSS 0xF #define V_TX_MSS(x) ((x) << S_TX_MSS) #define G_TX_MSS(x) (((x) >> S_TX_MSS) & M_TX_MSS) #define S_TX_QOS 8 #define M_TX_QOS 0xFF #define V_TX_QOS(x) ((x) << S_TX_QOS) #define G_TX_QOS(x) (((x) >> S_TX_QOS) & M_TX_QOS) #define S_TX_SNDBUF 16 #define M_TX_SNDBUF 0xFFFF #define V_TX_SNDBUF(x) ((x) << S_TX_SNDBUF) #define G_TX_SNDBUF(x) (((x) >> S_TX_SNDBUF) & M_TX_SNDBUF) struct cpl_tx_data { union opcode_tid ot; __be32 len; __be32 rsvd; __be16 urg; __be16 flags; }; /* cpl_tx_data.flags fields */ #define S_TX_ULP_SUBMODE 6 #define M_TX_ULP_SUBMODE 0xF #define V_TX_ULP_SUBMODE(x) ((x) << S_TX_ULP_SUBMODE) #define G_TX_ULP_SUBMODE(x) (((x) >> S_TX_ULP_SUBMODE) & M_TX_ULP_SUBMODE) #define S_TX_ULP_MODE 10 #define M_TX_ULP_MODE 0xF #define V_TX_ULP_MODE(x) ((x) << S_TX_ULP_MODE) #define G_TX_ULP_MODE(x) (((x) >> S_TX_ULP_MODE) & M_TX_ULP_MODE) #define S_TX_SHOVE 14 #define V_TX_SHOVE(x) ((x) << S_TX_SHOVE) #define F_TX_SHOVE V_TX_SHOVE(1U) #define S_TX_MORE 15 #define V_TX_MORE(x) ((x) << S_TX_MORE) #define F_TX_MORE V_TX_MORE(1U) /* additional tx_data_wr.flags fields */ #define S_TX_CPU_IDX 0 #define M_TX_CPU_IDX 0x3F #define V_TX_CPU_IDX(x) ((x) << S_TX_CPU_IDX) #define G_TX_CPU_IDX(x) (((x) >> S_TX_CPU_IDX) & M_TX_CPU_IDX) #define S_TX_URG 16 #define V_TX_URG(x) ((x) << S_TX_URG) #define F_TX_URG V_TX_URG(1U) #define S_TX_CLOSE 17 #define V_TX_CLOSE(x) ((x) << S_TX_CLOSE) #define F_TX_CLOSE V_TX_CLOSE(1U) #define S_TX_INIT 18 #define V_TX_INIT(x) ((x) << S_TX_INIT) #define F_TX_INIT V_TX_INIT(1U) #define S_TX_IMM_ACK 19 #define V_TX_IMM_ACK(x) ((x) << S_TX_IMM_ACK) #define F_TX_IMM_ACK V_TX_IMM_ACK(1U) #define S_TX_IMM_DMA 20 #define V_TX_IMM_DMA(x) ((x) << S_TX_IMM_DMA) #define F_TX_IMM_DMA V_TX_IMM_DMA(1U) struct cpl_tx_data_ack { RSS_HDR union opcode_tid ot; __be32 ack_seq; }; struct cpl_wr_ack { RSS_HDR union opcode_tid ot; __be16 credits; __be16 rsvd; __be32 snd_nxt; __be32 snd_una; }; struct cpl_rdma_ec_status { RSS_HDR union opcode_tid ot; __u8 rsvd[3]; __u8 status; }; struct mngt_pktsched_wr { __be32 wr_hi; __be32 wr_lo; __u8 mngt_opcode; __u8 rsvd[7]; __u8 sched; __u8 idx; __u8 min; __u8 max; __u8 binding; __u8 rsvd1[3]; }; struct cpl_iscsi_hdr { RSS_HDR union opcode_tid ot; __be16 pdu_len_ddp; __be16 len; __be32 seq; __be16 urg; __u8 rsvd; __u8 status; }; /* cpl_iscsi_hdr.pdu_len_ddp fields */ #define S_ISCSI_PDU_LEN 0 #define M_ISCSI_PDU_LEN 0x7FFF #define V_ISCSI_PDU_LEN(x) ((x) << S_ISCSI_PDU_LEN) #define G_ISCSI_PDU_LEN(x) (((x) >> S_ISCSI_PDU_LEN) & M_ISCSI_PDU_LEN) #define S_ISCSI_DDP 15 #define V_ISCSI_DDP(x) ((x) << S_ISCSI_DDP) #define F_ISCSI_DDP V_ISCSI_DDP(1U) struct cpl_rx_data { RSS_HDR union opcode_tid ot; __be16 rsvd; __be16 len; __be32 seq; __be16 urg; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 dack_mode:2; __u8 psh:1; __u8 heartbeat:1; __u8 :4; #else __u8 :4; __u8 heartbeat:1; __u8 psh:1; __u8 dack_mode:2; #endif __u8 status; }; struct cpl_rx_data_ack { WR_HDR; union opcode_tid ot; __be32 credit_dack; }; /* cpl_rx_data_ack.ack_seq fields */ #define S_RX_CREDITS 0 #define M_RX_CREDITS 0x7FFFFFF #define V_RX_CREDITS(x) ((x) << S_RX_CREDITS) #define G_RX_CREDITS(x) (((x) >> S_RX_CREDITS) & M_RX_CREDITS) #define S_RX_MODULATE 27 #define V_RX_MODULATE(x) ((x) << S_RX_MODULATE) #define F_RX_MODULATE V_RX_MODULATE(1U) #define S_RX_FORCE_ACK 28 #define V_RX_FORCE_ACK(x) ((x) << S_RX_FORCE_ACK) #define F_RX_FORCE_ACK V_RX_FORCE_ACK(1U) #define S_RX_DACK_MODE 29 #define M_RX_DACK_MODE 0x3 #define V_RX_DACK_MODE(x) ((x) << S_RX_DACK_MODE) #define G_RX_DACK_MODE(x) (((x) >> S_RX_DACK_MODE) & M_RX_DACK_MODE) #define S_RX_DACK_CHANGE 31 #define V_RX_DACK_CHANGE(x) ((x) << S_RX_DACK_CHANGE) #define F_RX_DACK_CHANGE V_RX_DACK_CHANGE(1U) struct cpl_rx_urg_notify { RSS_HDR union opcode_tid ot; __be32 seq; }; struct cpl_rx_ddp_complete { RSS_HDR union opcode_tid ot; __be32 ddp_report; }; struct cpl_rx_data_ddp { RSS_HDR union opcode_tid ot; __be16 urg; __be16 len; __be32 seq; union { __be32 nxt_seq; __be32 ddp_report; } __U; __be32 ulp_crc; __be32 ddpvld_status; }; /* cpl_rx_data_ddp.ddpvld_status fields */ #define S_DDP_STATUS 0 #define M_DDP_STATUS 0xFF #define V_DDP_STATUS(x) ((x) << S_DDP_STATUS) #define G_DDP_STATUS(x) (((x) >> S_DDP_STATUS) & M_DDP_STATUS) #define S_DDP_VALID 15 #define M_DDP_VALID 0x1FFFF #define V_DDP_VALID(x) ((x) << S_DDP_VALID) #define G_DDP_VALID(x) (((x) >> S_DDP_VALID) & M_DDP_VALID) #define S_DDP_PPOD_MISMATCH 15 #define V_DDP_PPOD_MISMATCH(x) ((x) << S_DDP_PPOD_MISMATCH) #define F_DDP_PPOD_MISMATCH V_DDP_PPOD_MISMATCH(1U) #define S_DDP_PDU 16 #define V_DDP_PDU(x) ((x) << S_DDP_PDU) #define F_DDP_PDU V_DDP_PDU(1U) #define S_DDP_LLIMIT_ERR 17 #define V_DDP_LLIMIT_ERR(x) ((x) << S_DDP_LLIMIT_ERR) #define F_DDP_LLIMIT_ERR V_DDP_LLIMIT_ERR(1U) #define S_DDP_PPOD_PARITY_ERR 18 #define V_DDP_PPOD_PARITY_ERR(x) ((x) << S_DDP_PPOD_PARITY_ERR) #define F_DDP_PPOD_PARITY_ERR V_DDP_PPOD_PARITY_ERR(1U) #define S_DDP_PADDING_ERR 19 #define V_DDP_PADDING_ERR(x) ((x) << S_DDP_PADDING_ERR) #define F_DDP_PADDING_ERR V_DDP_PADDING_ERR(1U) #define S_DDP_HDRCRC_ERR 20 #define V_DDP_HDRCRC_ERR(x) ((x) << S_DDP_HDRCRC_ERR) #define F_DDP_HDRCRC_ERR V_DDP_HDRCRC_ERR(1U) #define S_DDP_DATACRC_ERR 21 #define V_DDP_DATACRC_ERR(x) ((x) << S_DDP_DATACRC_ERR) #define F_DDP_DATACRC_ERR V_DDP_DATACRC_ERR(1U) #define S_DDP_INVALID_TAG 22 #define V_DDP_INVALID_TAG(x) ((x) << S_DDP_INVALID_TAG) #define F_DDP_INVALID_TAG V_DDP_INVALID_TAG(1U) #define S_DDP_ULIMIT_ERR 23 #define V_DDP_ULIMIT_ERR(x) ((x) << S_DDP_ULIMIT_ERR) #define F_DDP_ULIMIT_ERR V_DDP_ULIMIT_ERR(1U) #define S_DDP_OFFSET_ERR 24 #define V_DDP_OFFSET_ERR(x) ((x) << S_DDP_OFFSET_ERR) #define F_DDP_OFFSET_ERR V_DDP_OFFSET_ERR(1U) #define S_DDP_COLOR_ERR 25 #define V_DDP_COLOR_ERR(x) ((x) << S_DDP_COLOR_ERR) #define F_DDP_COLOR_ERR V_DDP_COLOR_ERR(1U) #define S_DDP_TID_MISMATCH 26 #define V_DDP_TID_MISMATCH(x) ((x) << S_DDP_TID_MISMATCH) #define F_DDP_TID_MISMATCH V_DDP_TID_MISMATCH(1U) #define S_DDP_INVALID_PPOD 27 #define V_DDP_INVALID_PPOD(x) ((x) << S_DDP_INVALID_PPOD) #define F_DDP_INVALID_PPOD V_DDP_INVALID_PPOD(1U) #define S_DDP_ULP_MODE 28 #define M_DDP_ULP_MODE 0xF #define V_DDP_ULP_MODE(x) ((x) << S_DDP_ULP_MODE) #define G_DDP_ULP_MODE(x) (((x) >> S_DDP_ULP_MODE) & M_DDP_ULP_MODE) /* cpl_rx_data_ddp.ddp_report fields */ #define S_DDP_OFFSET 0 #define M_DDP_OFFSET 0x3FFFFF #define V_DDP_OFFSET(x) ((x) << S_DDP_OFFSET) #define G_DDP_OFFSET(x) (((x) >> S_DDP_OFFSET) & M_DDP_OFFSET) #define S_DDP_URG 24 #define V_DDP_URG(x) ((x) << S_DDP_URG) #define F_DDP_URG V_DDP_URG(1U) #define S_DDP_PSH 25 #define V_DDP_PSH(x) ((x) << S_DDP_PSH) #define F_DDP_PSH V_DDP_PSH(1U) #define S_DDP_BUF_COMPLETE 26 #define V_DDP_BUF_COMPLETE(x) ((x) << S_DDP_BUF_COMPLETE) #define F_DDP_BUF_COMPLETE V_DDP_BUF_COMPLETE(1U) #define S_DDP_BUF_TIMED_OUT 27 #define V_DDP_BUF_TIMED_OUT(x) ((x) << S_DDP_BUF_TIMED_OUT) #define F_DDP_BUF_TIMED_OUT V_DDP_BUF_TIMED_OUT(1U) #define S_DDP_BUF_IDX 28 #define V_DDP_BUF_IDX(x) ((x) << S_DDP_BUF_IDX) #define F_DDP_BUF_IDX V_DDP_BUF_IDX(1U) struct cpl_tx_pkt { WR_HDR; __be32 cntrl; __be32 len; }; struct cpl_tx_pkt_lso { WR_HDR; __be32 cntrl; __be32 len; __be32 rsvd; __be32 lso_info; }; /* cpl_tx_pkt*.cntrl fields */ #define S_TXPKT_VLAN 0 #define M_TXPKT_VLAN 0xFFFF #define V_TXPKT_VLAN(x) ((x) << S_TXPKT_VLAN) #define G_TXPKT_VLAN(x) (((x) >> S_TXPKT_VLAN) & M_TXPKT_VLAN) #define S_TXPKT_INTF 16 #define M_TXPKT_INTF 0xF #define V_TXPKT_INTF(x) ((x) << S_TXPKT_INTF) #define G_TXPKT_INTF(x) (((x) >> S_TXPKT_INTF) & M_TXPKT_INTF) #define S_TXPKT_IPCSUM_DIS 20 #define V_TXPKT_IPCSUM_DIS(x) ((x) << S_TXPKT_IPCSUM_DIS) #define F_TXPKT_IPCSUM_DIS V_TXPKT_IPCSUM_DIS(1U) #define S_TXPKT_L4CSUM_DIS 21 #define V_TXPKT_L4CSUM_DIS(x) ((x) << S_TXPKT_L4CSUM_DIS) #define F_TXPKT_L4CSUM_DIS V_TXPKT_L4CSUM_DIS(1U) #define S_TXPKT_VLAN_VLD 22 #define V_TXPKT_VLAN_VLD(x) ((x) << S_TXPKT_VLAN_VLD) #define F_TXPKT_VLAN_VLD V_TXPKT_VLAN_VLD(1U) #define S_TXPKT_LOOPBACK 23 #define V_TXPKT_LOOPBACK(x) ((x) << S_TXPKT_LOOPBACK) #define F_TXPKT_LOOPBACK V_TXPKT_LOOPBACK(1U) #define S_TXPKT_OPCODE 24 #define M_TXPKT_OPCODE 0xFF #define V_TXPKT_OPCODE(x) ((x) << S_TXPKT_OPCODE) #define G_TXPKT_OPCODE(x) (((x) >> S_TXPKT_OPCODE) & M_TXPKT_OPCODE) /* cpl_tx_pkt_lso.lso_info fields */ #define S_LSO_MSS 0 #define M_LSO_MSS 0x3FFF #define V_LSO_MSS(x) ((x) << S_LSO_MSS) #define G_LSO_MSS(x) (((x) >> S_LSO_MSS) & M_LSO_MSS) #define S_LSO_ETH_TYPE 14 #define M_LSO_ETH_TYPE 0x3 #define V_LSO_ETH_TYPE(x) ((x) << S_LSO_ETH_TYPE) #define G_LSO_ETH_TYPE(x) (((x) >> S_LSO_ETH_TYPE) & M_LSO_ETH_TYPE) #define S_LSO_TCPHDR_WORDS 16 #define M_LSO_TCPHDR_WORDS 0xF #define V_LSO_TCPHDR_WORDS(x) ((x) << S_LSO_TCPHDR_WORDS) #define G_LSO_TCPHDR_WORDS(x) (((x) >> S_LSO_TCPHDR_WORDS) & M_LSO_TCPHDR_WORDS) #define S_LSO_IPHDR_WORDS 20 #define M_LSO_IPHDR_WORDS 0xF #define V_LSO_IPHDR_WORDS(x) ((x) << S_LSO_IPHDR_WORDS) #define G_LSO_IPHDR_WORDS(x) (((x) >> S_LSO_IPHDR_WORDS) & M_LSO_IPHDR_WORDS) #define S_LSO_IPV6 24 #define V_LSO_IPV6(x) ((x) << S_LSO_IPV6) #define F_LSO_IPV6 V_LSO_IPV6(1U) struct cpl_trace_pkt { #ifdef CHELSIO_FW __u8 rss_opcode; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 err:1; __u8 :7; #else __u8 :7; __u8 err:1; #endif __u8 rsvd0; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 qid:4; __u8 :4; #else __u8 :4; __u8 qid:4; #endif __be32 tstamp; #endif /* CHELSIO_FW */ __u8 opcode; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 iff:4; __u8 :4; #else __u8 :4; __u8 iff:4; #endif __u8 rsvd[4]; __be16 len; }; struct cpl_rx_pkt { RSS_HDR __u8 opcode; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 iff:4; __u8 csum_valid:1; __u8 ipmi_pkt:1; __u8 vlan_valid:1; __u8 fragment:1; #else __u8 fragment:1; __u8 vlan_valid:1; __u8 ipmi_pkt:1; __u8 csum_valid:1; __u8 iff:4; #endif __be16 csum; __be16 vlan; __be16 len; }; struct cpl_l2t_write_req { WR_HDR; union opcode_tid ot; __be32 params; __u8 rsvd[2]; __u8 dst_mac[6]; }; /* cpl_l2t_write_req.params fields */ #define S_L2T_W_IDX 0 #define M_L2T_W_IDX 0x7FF #define V_L2T_W_IDX(x) ((x) << S_L2T_W_IDX) #define G_L2T_W_IDX(x) (((x) >> S_L2T_W_IDX) & M_L2T_W_IDX) #define S_L2T_W_VLAN 11 #define M_L2T_W_VLAN 0xFFF #define V_L2T_W_VLAN(x) ((x) << S_L2T_W_VLAN) #define G_L2T_W_VLAN(x) (((x) >> S_L2T_W_VLAN) & M_L2T_W_VLAN) #define S_L2T_W_IFF 23 #define M_L2T_W_IFF 0xF #define V_L2T_W_IFF(x) ((x) << S_L2T_W_IFF) #define G_L2T_W_IFF(x) (((x) >> S_L2T_W_IFF) & M_L2T_W_IFF) #define S_L2T_W_PRIO 27 #define M_L2T_W_PRIO 0x7 #define V_L2T_W_PRIO(x) ((x) << S_L2T_W_PRIO) #define G_L2T_W_PRIO(x) (((x) >> S_L2T_W_PRIO) & M_L2T_W_PRIO) struct cpl_l2t_write_rpl { RSS_HDR union opcode_tid ot; __u8 status; __u8 rsvd[3]; }; struct cpl_l2t_read_req { WR_HDR; union opcode_tid ot; __be16 rsvd; __be16 l2t_idx; }; struct cpl_l2t_read_rpl { RSS_HDR union opcode_tid ot; __be32 params; __u8 rsvd[2]; __u8 dst_mac[6]; }; /* cpl_l2t_read_rpl.params fields */ #define S_L2T_R_PRIO 0 #define M_L2T_R_PRIO 0x7 #define V_L2T_R_PRIO(x) ((x) << S_L2T_R_PRIO) #define G_L2T_R_PRIO(x) (((x) >> S_L2T_R_PRIO) & M_L2T_R_PRIO) #define S_L2T_R_VLAN 8 #define M_L2T_R_VLAN 0xFFF #define V_L2T_R_VLAN(x) ((x) << S_L2T_R_VLAN) #define G_L2T_R_VLAN(x) (((x) >> S_L2T_R_VLAN) & M_L2T_R_VLAN) #define S_L2T_R_IFF 20 #define M_L2T_R_IFF 0xF #define V_L2T_R_IFF(x) ((x) << S_L2T_R_IFF) #define G_L2T_R_IFF(x) (((x) >> S_L2T_R_IFF) & M_L2T_R_IFF) #define S_L2T_STATUS 24 #define M_L2T_STATUS 0xFF #define V_L2T_STATUS(x) ((x) << S_L2T_STATUS) #define G_L2T_STATUS(x) (((x) >> S_L2T_STATUS) & M_L2T_STATUS) struct cpl_smt_write_req { WR_HDR; union opcode_tid ot; __u8 rsvd0; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 mtu_idx:4; __u8 iff:4; #else __u8 iff:4; __u8 mtu_idx:4; #endif __be16 rsvd2; __be16 rsvd3; __u8 src_mac1[6]; __be16 rsvd4; __u8 src_mac0[6]; }; struct cpl_smt_write_rpl { RSS_HDR union opcode_tid ot; __u8 status; __u8 rsvd[3]; }; struct cpl_smt_read_req { WR_HDR; union opcode_tid ot; __u8 rsvd0; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 :4; __u8 iff:4; #else __u8 iff:4; __u8 :4; #endif __be16 rsvd2; }; struct cpl_smt_read_rpl { RSS_HDR union opcode_tid ot; __u8 status; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 mtu_idx:4; __u8 :4; #else __u8 :4; __u8 mtu_idx:4; #endif __be16 rsvd2; __be16 rsvd3; __u8 src_mac1[6]; __be16 rsvd4; __u8 src_mac0[6]; }; struct cpl_rte_delete_req { WR_HDR; union opcode_tid ot; __be32 params; }; /* { cpl_rte_delete_req, cpl_rte_read_req }.params fields */ #define S_RTE_REQ_LUT_IX 8 #define M_RTE_REQ_LUT_IX 0x7FF #define V_RTE_REQ_LUT_IX(x) ((x) << S_RTE_REQ_LUT_IX) #define G_RTE_REQ_LUT_IX(x) (((x) >> S_RTE_REQ_LUT_IX) & M_RTE_REQ_LUT_IX) #define S_RTE_REQ_LUT_BASE 19 #define M_RTE_REQ_LUT_BASE 0x7FF #define V_RTE_REQ_LUT_BASE(x) ((x) << S_RTE_REQ_LUT_BASE) #define G_RTE_REQ_LUT_BASE(x) (((x) >> S_RTE_REQ_LUT_BASE) & M_RTE_REQ_LUT_BASE) #define S_RTE_READ_REQ_SELECT 31 #define V_RTE_READ_REQ_SELECT(x) ((x) << S_RTE_READ_REQ_SELECT) #define F_RTE_READ_REQ_SELECT V_RTE_READ_REQ_SELECT(1U) struct cpl_rte_delete_rpl { RSS_HDR union opcode_tid ot; __u8 status; __u8 rsvd[3]; }; struct cpl_rte_write_req { WR_HDR; union opcode_tid ot; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 :6; __u8 write_tcam:1; __u8 write_l2t_lut:1; #else __u8 write_l2t_lut:1; __u8 write_tcam:1; __u8 :6; #endif __u8 rsvd[3]; __be32 lut_params; __be16 rsvd2; __be16 l2t_idx; __be32 netmask; __be32 faddr; }; /* cpl_rte_write_req.lut_params fields */ #define S_RTE_WRITE_REQ_LUT_IX 10 #define M_RTE_WRITE_REQ_LUT_IX 0x7FF #define V_RTE_WRITE_REQ_LUT_IX(x) ((x) << S_RTE_WRITE_REQ_LUT_IX) #define G_RTE_WRITE_REQ_LUT_IX(x) (((x) >> S_RTE_WRITE_REQ_LUT_IX) & M_RTE_WRITE_REQ_LUT_IX) #define S_RTE_WRITE_REQ_LUT_BASE 21 #define M_RTE_WRITE_REQ_LUT_BASE 0x7FF #define V_RTE_WRITE_REQ_LUT_BASE(x) ((x) << S_RTE_WRITE_REQ_LUT_BASE) #define G_RTE_WRITE_REQ_LUT_BASE(x) (((x) >> S_RTE_WRITE_REQ_LUT_BASE) & M_RTE_WRITE_REQ_LUT_BASE) struct cpl_rte_write_rpl { RSS_HDR union opcode_tid ot; __u8 status; __u8 rsvd[3]; }; struct cpl_rte_read_req { WR_HDR; union opcode_tid ot; __be32 params; }; struct cpl_rte_read_rpl { RSS_HDR union opcode_tid ot; __u8 status; __u8 rsvd0; __be16 l2t_idx; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 :7; __u8 select:1; #else __u8 select:1; __u8 :7; #endif __u8 rsvd2[3]; __be32 addr; }; struct cpl_tid_release { WR_HDR; union opcode_tid ot; __be32 rsvd; }; struct cpl_barrier { WR_HDR; __u8 opcode; __u8 rsvd[7]; }; struct cpl_rdma_read_req { __u8 opcode; __u8 rsvd[15]; }; struct cpl_rdma_terminate { #ifdef CHELSIO_FW __u8 opcode; __u8 rsvd[2]; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 rspq:3; __u8 :5; #else __u8 :5; __u8 rspq:3; #endif __be32 tid_len; #endif __be32 msn; __be32 mo; __u8 data[0]; }; /* cpl_rdma_terminate.tid_len fields */ #define S_FLIT_CNT 0 #define M_FLIT_CNT 0xFF #define V_FLIT_CNT(x) ((x) << S_FLIT_CNT) #define G_FLIT_CNT(x) (((x) >> S_FLIT_CNT) & M_FLIT_CNT) #define S_TERM_TID 8 #define M_TERM_TID 0xFFFFF #define V_TERM_TID(x) ((x) << S_TERM_TID) #define G_TERM_TID(x) (((x) >> S_TERM_TID) & M_TERM_TID) + +/* ULP_TX opcodes */ +enum { ULP_MEM_READ = 2, ULP_MEM_WRITE = 3, ULP_TXPKT = 4 }; + +#define S_ULPTX_CMD 28 +#define M_ULPTX_CMD 0xF +#define V_ULPTX_CMD(x) ((x) << S_ULPTX_CMD) + +#define S_ULPTX_NFLITS 0 +#define M_ULPTX_NFLITS 0xFF +#define V_ULPTX_NFLITS(x) ((x) << S_ULPTX_NFLITS) + +struct ulp_mem_io { + WR_HDR; + __be32 cmd_lock_addr; + __be32 len; +}; + + /* ulp_mem_io.cmd_lock_addr fields */ +#define S_ULP_MEMIO_ADDR 0 +#define M_ULP_MEMIO_ADDR 0x7FFFFFF +#define V_ULP_MEMIO_ADDR(x) ((x) << S_ULP_MEMIO_ADDR) + +#define S_ULP_MEMIO_LOCK 27 +#define V_ULP_MEMIO_LOCK(x) ((x) << S_ULP_MEMIO_LOCK) +#define F_ULP_MEMIO_LOCK V_ULP_MEMIO_LOCK(1U) + + /* ulp_mem_io.len fields */ +#define S_ULP_MEMIO_DATA_LEN 28 +#define M_ULP_MEMIO_DATA_LEN 0xF +#define V_ULP_MEMIO_DATA_LEN(x) ((x) << S_ULP_MEMIO_DATA_LEN) + +struct ulp_txpkt { + __be32 cmd_dest; + __be32 len; +}; + + /* ulp_txpkt.cmd_dest fields */ +#define S_ULP_TXPKT_DEST 24 +#define M_ULP_TXPKT_DEST 0xF +#define V_ULP_TXPKT_DEST(x) ((x) << S_ULP_TXPKT_DEST) + #endif /* T3_CPL_H */ Index: head/sys/dev/cxgb/common/cxgb_t3_hw.c =================================================================== --- head/sys/dev/cxgb/common/cxgb_t3_hw.c (revision 169977) +++ head/sys/dev/cxgb/common/cxgb_t3_hw.c (revision 169978) @@ -1,3509 +1,3516 @@ /************************************************************************** Copyright (c) 2007, Chelsio Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the Chelsio Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ***************************************************************************/ #include __FBSDID("$FreeBSD$"); #include #include #include #include /** * t3_wait_op_done_val - wait until an operation is completed * @adapter: the adapter performing the operation * @reg: the register to check for completion * @mask: a single-bit field within @reg that indicates completion * @polarity: the value of the field when the operation is completed * @attempts: number of check iterations * @delay: delay in usecs between iterations * @valp: where to store the value of the register at completion time * * Wait until an operation is completed by checking a bit in a register * up to @attempts times. If @valp is not NULL the value of the register * at the time it indicated completion is stored there. Returns 0 if the * operation completes and -EAGAIN otherwise. */ int t3_wait_op_done_val(adapter_t *adapter, int reg, u32 mask, int polarity, int attempts, int delay, u32 *valp) { while (1) { u32 val = t3_read_reg(adapter, reg); if (!!(val & mask) == polarity) { if (valp) *valp = val; return 0; } if (--attempts == 0) return -EAGAIN; if (delay) udelay(delay); } } /** * t3_write_regs - write a bunch of registers * @adapter: the adapter to program * @p: an array of register address/register value pairs * @n: the number of address/value pairs * @offset: register address offset * * Takes an array of register address/register value pairs and writes each * value to the corresponding register. Register addresses are adjusted * by the supplied offset. */ void t3_write_regs(adapter_t *adapter, const struct addr_val_pair *p, int n, unsigned int offset) { while (n--) { t3_write_reg(adapter, p->reg_addr + offset, p->val); p++; } } /** * t3_set_reg_field - set a register field to a value * @adapter: the adapter to program * @addr: the register address * @mask: specifies the portion of the register to modify * @val: the new value for the register field * * Sets a register field specified by the supplied mask to the * given value. */ void t3_set_reg_field(adapter_t *adapter, unsigned int addr, u32 mask, u32 val) { u32 v = t3_read_reg(adapter, addr) & ~mask; t3_write_reg(adapter, addr, v | val); (void) t3_read_reg(adapter, addr); /* flush */ } /** * t3_read_indirect - read indirectly addressed registers * @adap: the adapter * @addr_reg: register holding the indirect address * @data_reg: register holding the value of the indirect register * @vals: where the read register values are stored * @start_idx: index of first indirect register to read * @nregs: how many indirect registers to read * * Reads registers that are accessed indirectly through an address/data * register pair. */ void t3_read_indirect(adapter_t *adap, unsigned int addr_reg, unsigned int data_reg, u32 *vals, unsigned int nregs, unsigned int start_idx) { while (nregs--) { t3_write_reg(adap, addr_reg, start_idx); *vals++ = t3_read_reg(adap, data_reg); start_idx++; } } /** * t3_mc7_bd_read - read from MC7 through backdoor accesses * @mc7: identifies MC7 to read from * @start: index of first 64-bit word to read * @n: number of 64-bit words to read * @buf: where to store the read result * * Read n 64-bit words from MC7 starting at word start, using backdoor * accesses. */ int t3_mc7_bd_read(struct mc7 *mc7, unsigned int start, unsigned int n, u64 *buf) { static int shift[] = { 0, 0, 16, 24 }; static int step[] = { 0, 32, 16, 8 }; unsigned int size64 = mc7->size / 8; /* # of 64-bit words */ adapter_t *adap = mc7->adapter; if (start >= size64 || start + n > size64) return -EINVAL; start *= (8 << mc7->width); while (n--) { int i; u64 val64 = 0; for (i = (1 << mc7->width) - 1; i >= 0; --i) { int attempts = 10; u32 val; t3_write_reg(adap, mc7->offset + A_MC7_BD_ADDR, start); t3_write_reg(adap, mc7->offset + A_MC7_BD_OP, 0); val = t3_read_reg(adap, mc7->offset + A_MC7_BD_OP); while ((val & F_BUSY) && attempts--) val = t3_read_reg(adap, mc7->offset + A_MC7_BD_OP); if (val & F_BUSY) return -EIO; val = t3_read_reg(adap, mc7->offset + A_MC7_BD_DATA1); if (mc7->width == 0) { val64 = t3_read_reg(adap, mc7->offset + A_MC7_BD_DATA0); val64 |= (u64)val << 32; } else { if (mc7->width > 1) val >>= shift[mc7->width]; val64 |= (u64)val << (step[mc7->width] * i); } start += 8; } *buf++ = val64; } return 0; } /* * Initialize MI1. */ static void mi1_init(adapter_t *adap, const struct adapter_info *ai) { u32 clkdiv = adap->params.vpd.cclk / (2 * adap->params.vpd.mdc) - 1; u32 val = F_PREEN | V_MDIINV(ai->mdiinv) | V_MDIEN(ai->mdien) | V_CLKDIV(clkdiv); if (!(ai->caps & SUPPORTED_10000baseT_Full)) val |= V_ST(1); t3_write_reg(adap, A_MI1_CFG, val); } #define MDIO_ATTEMPTS 10 /* * MI1 read/write operations for direct-addressed PHYs. */ static int mi1_read(adapter_t *adapter, int phy_addr, int mmd_addr, int reg_addr, unsigned int *valp) { int ret; u32 addr = V_REGADDR(reg_addr) | V_PHYADDR(phy_addr); if (mmd_addr) return -EINVAL; MDIO_LOCK(adapter); t3_write_reg(adapter, A_MI1_ADDR, addr); t3_write_reg(adapter, A_MI1_OP, V_MDI_OP(2)); ret = t3_wait_op_done(adapter, A_MI1_OP, F_BUSY, 0, MDIO_ATTEMPTS, 20); if (!ret) *valp = t3_read_reg(adapter, A_MI1_DATA); MDIO_UNLOCK(adapter); return ret; } static int mi1_write(adapter_t *adapter, int phy_addr, int mmd_addr, int reg_addr, unsigned int val) { int ret; u32 addr = V_REGADDR(reg_addr) | V_PHYADDR(phy_addr); if (mmd_addr) return -EINVAL; MDIO_LOCK(adapter); t3_write_reg(adapter, A_MI1_ADDR, addr); t3_write_reg(adapter, A_MI1_DATA, val); t3_write_reg(adapter, A_MI1_OP, V_MDI_OP(1)); ret = t3_wait_op_done(adapter, A_MI1_OP, F_BUSY, 0, MDIO_ATTEMPTS, 20); MDIO_UNLOCK(adapter); return ret; } static struct mdio_ops mi1_mdio_ops = { mi1_read, mi1_write }; /* * MI1 read/write operations for indirect-addressed PHYs. */ static int mi1_ext_read(adapter_t *adapter, int phy_addr, int mmd_addr, int reg_addr, unsigned int *valp) { int ret; u32 addr = V_REGADDR(mmd_addr) | V_PHYADDR(phy_addr); MDIO_LOCK(adapter); t3_write_reg(adapter, A_MI1_ADDR, addr); t3_write_reg(adapter, A_MI1_DATA, reg_addr); t3_write_reg(adapter, A_MI1_OP, V_MDI_OP(0)); ret = t3_wait_op_done(adapter, A_MI1_OP, F_BUSY, 0, MDIO_ATTEMPTS, 20); if (!ret) { t3_write_reg(adapter, A_MI1_OP, V_MDI_OP(3)); ret = t3_wait_op_done(adapter, A_MI1_OP, F_BUSY, 0, MDIO_ATTEMPTS, 20); if (!ret) *valp = t3_read_reg(adapter, A_MI1_DATA); } MDIO_UNLOCK(adapter); return ret; } static int mi1_ext_write(adapter_t *adapter, int phy_addr, int mmd_addr, int reg_addr, unsigned int val) { int ret; u32 addr = V_REGADDR(mmd_addr) | V_PHYADDR(phy_addr); MDIO_LOCK(adapter); t3_write_reg(adapter, A_MI1_ADDR, addr); t3_write_reg(adapter, A_MI1_DATA, reg_addr); t3_write_reg(adapter, A_MI1_OP, V_MDI_OP(0)); ret = t3_wait_op_done(adapter, A_MI1_OP, F_BUSY, 0, MDIO_ATTEMPTS, 20); if (!ret) { t3_write_reg(adapter, A_MI1_DATA, val); t3_write_reg(adapter, A_MI1_OP, V_MDI_OP(1)); ret = t3_wait_op_done(adapter, A_MI1_OP, F_BUSY, 0, MDIO_ATTEMPTS, 20); } MDIO_UNLOCK(adapter); return ret; } static struct mdio_ops mi1_mdio_ext_ops = { mi1_ext_read, mi1_ext_write }; /** * t3_mdio_change_bits - modify the value of a PHY register * @phy: the PHY to operate on * @mmd: the device address * @reg: the register address * @clear: what part of the register value to mask off * @set: what part of the register value to set * * Changes the value of a PHY register by applying a mask to its current * value and ORing the result with a new value. */ int t3_mdio_change_bits(struct cphy *phy, int mmd, int reg, unsigned int clear, unsigned int set) { int ret; unsigned int val; ret = mdio_read(phy, mmd, reg, &val); if (!ret) { val &= ~clear; ret = mdio_write(phy, mmd, reg, val | set); } return ret; } /** * t3_phy_reset - reset a PHY block * @phy: the PHY to operate on * @mmd: the device address of the PHY block to reset * @wait: how long to wait for the reset to complete in 1ms increments * * Resets a PHY block and optionally waits for the reset to complete. * @mmd should be 0 for 10/100/1000 PHYs and the device address to reset * for 10G PHYs. */ int t3_phy_reset(struct cphy *phy, int mmd, int wait) { int err; unsigned int ctl; err = t3_mdio_change_bits(phy, mmd, MII_BMCR, BMCR_PDOWN, BMCR_RESET); if (err || !wait) return err; do { err = mdio_read(phy, mmd, MII_BMCR, &ctl); if (err) return err; ctl &= BMCR_RESET; if (ctl) t3_os_sleep(1); } while (ctl && --wait); return ctl ? -1 : 0; } /** * t3_phy_advertise - set the PHY advertisement registers for autoneg * @phy: the PHY to operate on * @advert: bitmap of capabilities the PHY should advertise * * Sets a 10/100/1000 PHY's advertisement registers to advertise the * requested capabilities. */ int t3_phy_advertise(struct cphy *phy, unsigned int advert) { int err; unsigned int val = 0; err = mdio_read(phy, 0, MII_CTRL1000, &val); if (err) return err; val &= ~(ADVERTISE_1000HALF | ADVERTISE_1000FULL); if (advert & ADVERTISED_1000baseT_Half) val |= ADVERTISE_1000HALF; if (advert & ADVERTISED_1000baseT_Full) val |= ADVERTISE_1000FULL; err = mdio_write(phy, 0, MII_CTRL1000, val); if (err) return err; val = 1; if (advert & ADVERTISED_10baseT_Half) val |= ADVERTISE_10HALF; if (advert & ADVERTISED_10baseT_Full) val |= ADVERTISE_10FULL; if (advert & ADVERTISED_100baseT_Half) val |= ADVERTISE_100HALF; if (advert & ADVERTISED_100baseT_Full) val |= ADVERTISE_100FULL; if (advert & ADVERTISED_Pause) val |= ADVERTISE_PAUSE_CAP; if (advert & ADVERTISED_Asym_Pause) val |= ADVERTISE_PAUSE_ASYM; return mdio_write(phy, 0, MII_ADVERTISE, val); } /** * t3_set_phy_speed_duplex - force PHY speed and duplex * @phy: the PHY to operate on * @speed: requested PHY speed * @duplex: requested PHY duplex * * Force a 10/100/1000 PHY's speed and duplex. This also disables * auto-negotiation except for GigE, where auto-negotiation is mandatory. */ int t3_set_phy_speed_duplex(struct cphy *phy, int speed, int duplex) { int err; unsigned int ctl; err = mdio_read(phy, 0, MII_BMCR, &ctl); if (err) return err; if (speed >= 0) { ctl &= ~(BMCR_SPEED100 | BMCR_SPEED1000 | BMCR_ANENABLE); if (speed == SPEED_100) ctl |= BMCR_SPEED100; else if (speed == SPEED_1000) ctl |= BMCR_SPEED1000; } if (duplex >= 0) { ctl &= ~(BMCR_FULLDPLX | BMCR_ANENABLE); if (duplex == DUPLEX_FULL) ctl |= BMCR_FULLDPLX; } if (ctl & BMCR_SPEED1000) /* auto-negotiation required for GigE */ ctl |= BMCR_ANENABLE; return mdio_write(phy, 0, MII_BMCR, ctl); } static struct adapter_info t3_adap_info[] = { { 2, 0, 0, 0, F_GPIO2_OEN | F_GPIO4_OEN | F_GPIO2_OUT_VAL | F_GPIO4_OUT_VAL, F_GPIO3 | F_GPIO5, - SUPPORTED_OFFLOAD, + 0, &mi1_mdio_ops, "Chelsio PE9000" }, { 2, 0, 0, 0, F_GPIO2_OEN | F_GPIO4_OEN | F_GPIO2_OUT_VAL | F_GPIO4_OUT_VAL, F_GPIO3 | F_GPIO5, - SUPPORTED_OFFLOAD, + 0, &mi1_mdio_ops, "Chelsio T302" }, { 1, 0, 0, 0, F_GPIO1_OEN | F_GPIO6_OEN | F_GPIO7_OEN | F_GPIO10_OEN | F_GPIO1_OUT_VAL | F_GPIO6_OUT_VAL | F_GPIO10_OUT_VAL, 0, - SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_OFFLOAD, + SUPPORTED_10000baseT_Full | SUPPORTED_AUI, &mi1_mdio_ext_ops, "Chelsio T310" }, { 2, 0, 0, 0, F_GPIO1_OEN | F_GPIO2_OEN | F_GPIO4_OEN | F_GPIO5_OEN | F_GPIO6_OEN | F_GPIO7_OEN | F_GPIO10_OEN | F_GPIO11_OEN | F_GPIO1_OUT_VAL | F_GPIO5_OUT_VAL | F_GPIO6_OUT_VAL | F_GPIO10_OUT_VAL, 0, - SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_OFFLOAD, + SUPPORTED_10000baseT_Full | SUPPORTED_AUI, &mi1_mdio_ext_ops, "Chelsio T320" }, }; /* * Return the adapter_info structure with a given index. Out-of-range indices * return NULL. */ const struct adapter_info *t3_get_adapter_info(unsigned int id) { return id < ARRAY_SIZE(t3_adap_info) ? &t3_adap_info[id] : NULL; } #define CAPS_1G (SUPPORTED_10baseT_Full | SUPPORTED_100baseT_Full | \ SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg | SUPPORTED_MII) #define CAPS_10G (SUPPORTED_10000baseT_Full | SUPPORTED_AUI) static struct port_type_info port_types[] = { { NULL }, { t3_ael1002_phy_prep, CAPS_10G | SUPPORTED_FIBRE, "10GBASE-XR" }, { t3_vsc8211_phy_prep, CAPS_1G | SUPPORTED_TP | SUPPORTED_IRQ, "10/100/1000BASE-T" }, { t3_mv88e1xxx_phy_prep, CAPS_1G | SUPPORTED_TP | SUPPORTED_IRQ, "10/100/1000BASE-T" }, { t3_xaui_direct_phy_prep, CAPS_10G | SUPPORTED_TP, "10GBASE-CX4" }, { NULL, CAPS_10G, "10GBASE-KX4" }, { t3_qt2045_phy_prep, CAPS_10G | SUPPORTED_TP, "10GBASE-CX4" }, { t3_ael1006_phy_prep, CAPS_10G | SUPPORTED_FIBRE, "10GBASE-SR" }, { NULL, CAPS_10G | SUPPORTED_TP, "10GBASE-CX4" }, }; #undef CAPS_1G #undef CAPS_10G #define VPD_ENTRY(name, len) \ u8 name##_kword[2]; u8 name##_len; u8 name##_data[len] /* * Partial EEPROM Vital Product Data structure. Includes only the ID and * VPD-R sections. */ struct t3_vpd { u8 id_tag; u8 id_len[2]; u8 id_data[16]; u8 vpdr_tag; u8 vpdr_len[2]; VPD_ENTRY(pn, 16); /* part number */ VPD_ENTRY(ec, 16); /* EC level */ VPD_ENTRY(sn, 16); /* serial number */ VPD_ENTRY(na, 12); /* MAC address base */ VPD_ENTRY(cclk, 6); /* core clock */ VPD_ENTRY(mclk, 6); /* mem clock */ VPD_ENTRY(uclk, 6); /* uP clk */ VPD_ENTRY(mdc, 6); /* MDIO clk */ VPD_ENTRY(mt, 2); /* mem timing */ VPD_ENTRY(xaui0cfg, 6); /* XAUI0 config */ VPD_ENTRY(xaui1cfg, 6); /* XAUI1 config */ VPD_ENTRY(port0, 2); /* PHY0 complex */ VPD_ENTRY(port1, 2); /* PHY1 complex */ VPD_ENTRY(port2, 2); /* PHY2 complex */ VPD_ENTRY(port3, 2); /* PHY3 complex */ VPD_ENTRY(rv, 1); /* csum */ u32 pad; /* for multiple-of-4 sizing and alignment */ }; #define EEPROM_MAX_POLL 4 #define EEPROM_STAT_ADDR 0x4000 #define VPD_BASE 0xc00 /** * t3_seeprom_read - read a VPD EEPROM location * @adapter: adapter to read * @addr: EEPROM address * @data: where to store the read data * * Read a 32-bit word from a location in VPD EEPROM using the card's PCI * VPD ROM capability. A zero is written to the flag bit when the * addres is written to the control register. The hardware device will * set the flag to 1 when 4 bytes have been read into the data register. */ int t3_seeprom_read(adapter_t *adapter, u32 addr, u32 *data) { u16 val; int attempts = EEPROM_MAX_POLL; unsigned int base = adapter->params.pci.vpd_cap_addr; if ((addr >= EEPROMSIZE && addr != EEPROM_STAT_ADDR) || (addr & 3)) return -EINVAL; t3_os_pci_write_config_2(adapter, base + PCI_VPD_ADDR, (u16)addr); do { udelay(10); t3_os_pci_read_config_2(adapter, base + PCI_VPD_ADDR, &val); } while (!(val & PCI_VPD_ADDR_F) && --attempts); if (!(val & PCI_VPD_ADDR_F)) { CH_ERR(adapter, "reading EEPROM address 0x%x failed\n", addr); return -EIO; } t3_os_pci_read_config_4(adapter, base + PCI_VPD_DATA, data); *data = le32_to_cpu(*data); return 0; } /** * t3_seeprom_write - write a VPD EEPROM location * @adapter: adapter to write * @addr: EEPROM address * @data: value to write * * Write a 32-bit word to a location in VPD EEPROM using the card's PCI * VPD ROM capability. */ int t3_seeprom_write(adapter_t *adapter, u32 addr, u32 data) { u16 val; int attempts = EEPROM_MAX_POLL; unsigned int base = adapter->params.pci.vpd_cap_addr; if ((addr >= EEPROMSIZE && addr != EEPROM_STAT_ADDR) || (addr & 3)) return -EINVAL; t3_os_pci_write_config_4(adapter, base + PCI_VPD_DATA, cpu_to_le32(data)); t3_os_pci_write_config_2(adapter, base + PCI_VPD_ADDR, (u16)addr | PCI_VPD_ADDR_F); do { t3_os_sleep(1); t3_os_pci_read_config_2(adapter, base + PCI_VPD_ADDR, &val); } while ((val & PCI_VPD_ADDR_F) && --attempts); if (val & PCI_VPD_ADDR_F) { CH_ERR(adapter, "write to EEPROM address 0x%x failed\n", addr); return -EIO; } return 0; } /** * t3_seeprom_wp - enable/disable EEPROM write protection * @adapter: the adapter * @enable: 1 to enable write protection, 0 to disable it * * Enables or disables write protection on the serial EEPROM. */ int t3_seeprom_wp(adapter_t *adapter, int enable) { return t3_seeprom_write(adapter, EEPROM_STAT_ADDR, enable ? 0xc : 0); } /* * Convert a character holding a hex digit to a number. */ static unsigned int hex2int(unsigned char c) { return isdigit(c) ? c - '0' : toupper(c) - 'A' + 10; } /** * get_vpd_params - read VPD parameters from VPD EEPROM * @adapter: adapter to read * @p: where to store the parameters * * Reads card parameters stored in VPD EEPROM. */ static int get_vpd_params(adapter_t *adapter, struct vpd_params *p) { int i, addr, ret; struct t3_vpd vpd; /* * Card information is normally at VPD_BASE but some early cards had * it at 0. */ ret = t3_seeprom_read(adapter, VPD_BASE, (u32 *)&vpd); if (ret) return ret; addr = vpd.id_tag == 0x82 ? VPD_BASE : 0; for (i = 0; i < sizeof(vpd); i += 4) { ret = t3_seeprom_read(adapter, addr + i, (u32 *)((u8 *)&vpd + i)); if (ret) return ret; } p->cclk = simple_strtoul(vpd.cclk_data, NULL, 10); p->mclk = simple_strtoul(vpd.mclk_data, NULL, 10); p->uclk = simple_strtoul(vpd.uclk_data, NULL, 10); p->mdc = simple_strtoul(vpd.mdc_data, NULL, 10); p->mem_timing = simple_strtoul(vpd.mt_data, NULL, 10); /* Old eeproms didn't have port information */ if (adapter->params.rev == 0 && !vpd.port0_data[0]) { p->port_type[0] = uses_xaui(adapter) ? 1 : 2; p->port_type[1] = uses_xaui(adapter) ? 6 : 2; } else { p->port_type[0] = (u8)hex2int(vpd.port0_data[0]); p->port_type[1] = (u8)hex2int(vpd.port1_data[0]); p->xauicfg[0] = simple_strtoul(vpd.xaui0cfg_data, NULL, 16); p->xauicfg[1] = simple_strtoul(vpd.xaui1cfg_data, NULL, 16); } for (i = 0; i < 6; i++) p->eth_base[i] = hex2int(vpd.na_data[2 * i]) * 16 + hex2int(vpd.na_data[2 * i + 1]); return 0; } /* serial flash and firmware constants */ enum { SF_ATTEMPTS = 5, /* max retries for SF1 operations */ SF_SEC_SIZE = 64 * 1024, /* serial flash sector size */ SF_SIZE = SF_SEC_SIZE * 8, /* serial flash size */ /* flash command opcodes */ SF_PROG_PAGE = 2, /* program page */ SF_WR_DISABLE = 4, /* disable writes */ SF_RD_STATUS = 5, /* read status register */ SF_WR_ENABLE = 6, /* enable writes */ SF_RD_DATA_FAST = 0xb, /* read flash */ SF_ERASE_SECTOR = 0xd8, /* erase sector */ FW_FLASH_BOOT_ADDR = 0x70000, /* start address of FW in flash */ FW_VERS_ADDR = 0x77ffc, /* flash address holding FW version */ FW_MIN_SIZE = 8 /* at least version and csum */ }; /** * sf1_read - read data from the serial flash * @adapter: the adapter * @byte_cnt: number of bytes to read * @cont: whether another operation will be chained * @valp: where to store the read data * * Reads up to 4 bytes of data from the serial flash. The location of * the read needs to be specified prior to calling this by issuing the * appropriate commands to the serial flash. */ static int sf1_read(adapter_t *adapter, unsigned int byte_cnt, int cont, u32 *valp) { int ret; if (!byte_cnt || byte_cnt > 4) return -EINVAL; if (t3_read_reg(adapter, A_SF_OP) & F_BUSY) return -EBUSY; t3_write_reg(adapter, A_SF_OP, V_CONT(cont) | V_BYTECNT(byte_cnt - 1)); ret = t3_wait_op_done(adapter, A_SF_OP, F_BUSY, 0, SF_ATTEMPTS, 10); if (!ret) *valp = t3_read_reg(adapter, A_SF_DATA); return ret; } /** * sf1_write - write data to the serial flash * @adapter: the adapter * @byte_cnt: number of bytes to write * @cont: whether another operation will be chained * @val: value to write * * Writes up to 4 bytes of data to the serial flash. The location of * the write needs to be specified prior to calling this by issuing the * appropriate commands to the serial flash. */ static int sf1_write(adapter_t *adapter, unsigned int byte_cnt, int cont, u32 val) { if (!byte_cnt || byte_cnt > 4) return -EINVAL; if (t3_read_reg(adapter, A_SF_OP) & F_BUSY) return -EBUSY; t3_write_reg(adapter, A_SF_DATA, val); t3_write_reg(adapter, A_SF_OP, V_CONT(cont) | V_BYTECNT(byte_cnt - 1) | V_OP(1)); return t3_wait_op_done(adapter, A_SF_OP, F_BUSY, 0, SF_ATTEMPTS, 10); } /** * flash_wait_op - wait for a flash operation to complete * @adapter: the adapter * @attempts: max number of polls of the status register * @delay: delay between polls in ms * * Wait for a flash operation to complete by polling the status register. */ static int flash_wait_op(adapter_t *adapter, int attempts, int delay) { int ret; u32 status; while (1) { if ((ret = sf1_write(adapter, 1, 1, SF_RD_STATUS)) != 0 || (ret = sf1_read(adapter, 1, 0, &status)) != 0) return ret; if (!(status & 1)) return 0; if (--attempts == 0) return -EAGAIN; if (delay) t3_os_sleep(delay); } } /** * t3_read_flash - read words from serial flash * @adapter: the adapter * @addr: the start address for the read * @nwords: how many 32-bit words to read * @data: where to store the read data * @byte_oriented: whether to store data as bytes or as words * * Read the specified number of 32-bit words from the serial flash. * If @byte_oriented is set the read data is stored as a byte array * (i.e., big-endian), otherwise as 32-bit words in the platform's * natural endianess. */ int t3_read_flash(adapter_t *adapter, unsigned int addr, unsigned int nwords, u32 *data, int byte_oriented) { int ret; if (addr + nwords * sizeof(u32) > SF_SIZE || (addr & 3)) return -EINVAL; addr = swab32(addr) | SF_RD_DATA_FAST; if ((ret = sf1_write(adapter, 4, 1, addr)) != 0 || (ret = sf1_read(adapter, 1, 1, data)) != 0) return ret; for ( ; nwords; nwords--, data++) { ret = sf1_read(adapter, 4, nwords > 1, data); if (ret) return ret; if (byte_oriented) *data = htonl(*data); } return 0; } /** * t3_write_flash - write up to a page of data to the serial flash * @adapter: the adapter * @addr: the start address to write * @n: length of data to write * @data: the data to write * * Writes up to a page of data (256 bytes) to the serial flash starting * at the given address. */ static int t3_write_flash(adapter_t *adapter, unsigned int addr, unsigned int n, const u8 *data) { int ret; u32 buf[64]; unsigned int i, c, left, val, offset = addr & 0xff; if (addr + n > SF_SIZE || offset + n > 256) return -EINVAL; val = swab32(addr) | SF_PROG_PAGE; if ((ret = sf1_write(adapter, 1, 0, SF_WR_ENABLE)) != 0 || (ret = sf1_write(adapter, 4, 1, val)) != 0) return ret; for (left = n; left; left -= c) { c = min(left, 4U); for (val = 0, i = 0; i < c; ++i) val = (val << 8) + *data++; ret = sf1_write(adapter, c, c != left, val); if (ret) return ret; } if ((ret = flash_wait_op(adapter, 5, 1)) != 0) return ret; /* Read the page to verify the write succeeded */ ret = t3_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf, 1); if (ret) return ret; if (memcmp(data - n, (u8 *)buf + offset, n)) return -EIO; return 0; } enum fw_version_type { FW_VERSION_N3, FW_VERSION_T3 }; /** * t3_get_fw_version - read the firmware version * @adapter: the adapter * @vers: where to place the version * * Reads the FW version from flash. */ int t3_get_fw_version(adapter_t *adapter, u32 *vers) { return t3_read_flash(adapter, FW_VERS_ADDR, 1, vers, 0); } /** * t3_check_fw_version - check if the FW is compatible with this driver * @adapter: the adapter * * Checks if an adapter's FW is compatible with the driver. Returns 0 * if the versions are compatible, a negative error otherwise. */ int t3_check_fw_version(adapter_t *adapter) { int ret; u32 vers; unsigned int type, major, minor; ret = t3_get_fw_version(adapter, &vers); if (ret) return ret; type = G_FW_VERSION_TYPE(vers); major = G_FW_VERSION_MAJOR(vers); minor = G_FW_VERSION_MINOR(vers); if (type == FW_VERSION_T3 && major == FW_VERSION_MAJOR && minor == FW_VERSION_MINOR) return 0; CH_ERR(adapter, "found wrong FW version(%u.%u), " "driver needs version %d.%d\n", major, minor, FW_VERSION_MAJOR, FW_VERSION_MINOR); return -EINVAL; } /** * t3_flash_erase_sectors - erase a range of flash sectors * @adapter: the adapter * @start: the first sector to erase * @end: the last sector to erase * * Erases the sectors in the given range. */ static int t3_flash_erase_sectors(adapter_t *adapter, int start, int end) { while (start <= end) { int ret; if ((ret = sf1_write(adapter, 1, 0, SF_WR_ENABLE)) != 0 || (ret = sf1_write(adapter, 4, 0, SF_ERASE_SECTOR | (start << 8))) != 0 || (ret = flash_wait_op(adapter, 5, 500)) != 0) return ret; start++; } return 0; } /* * t3_load_fw - download firmware * @adapter: the adapter * @fw_data: the firrware image to write * @size: image size * * Write the supplied firmware image to the card's serial flash. * The FW image has the following sections: @size - 8 bytes of code and * data, followed by 4 bytes of FW version, followed by the 32-bit * 1's complement checksum of the whole image. */ int t3_load_fw(adapter_t *adapter, const u8 *fw_data, unsigned int size) { u32 csum; unsigned int i; const u32 *p = (const u32 *)fw_data; int ret, addr, fw_sector = FW_FLASH_BOOT_ADDR >> 16; if ((size & 3) || (size < FW_MIN_SIZE)) return -EINVAL; if (size > FW_VERS_ADDR + 8 - FW_FLASH_BOOT_ADDR) return -EFBIG; for (csum = 0, i = 0; i < size / sizeof(csum); i++) csum += ntohl(p[i]); if (csum != 0xffffffff) { CH_ERR(adapter, "corrupted firmware image, checksum %u\n", csum); return -EINVAL; } ret = t3_flash_erase_sectors(adapter, fw_sector, fw_sector); if (ret) goto out; size -= 8; /* trim off version and checksum */ for (addr = FW_FLASH_BOOT_ADDR; size; ) { unsigned int chunk_size = min(size, 256U); ret = t3_write_flash(adapter, addr, chunk_size, fw_data); if (ret) goto out; addr += chunk_size; fw_data += chunk_size; size -= chunk_size; } ret = t3_write_flash(adapter, FW_VERS_ADDR, 4, fw_data); out: if (ret) CH_ERR(adapter, "firmware download failed, error %d\n", ret); return ret; } #define CIM_CTL_BASE 0x2000 /** * t3_cim_ctl_blk_read - read a block from CIM control region * * @adap: the adapter * @addr: the start address within the CIM control region * @n: number of words to read * @valp: where to store the result * * Reads a block of 4-byte words from the CIM control region. */ int t3_cim_ctl_blk_read(adapter_t *adap, unsigned int addr, unsigned int n, unsigned int *valp) { int ret = 0; if (t3_read_reg(adap, A_CIM_HOST_ACC_CTRL) & F_HOSTBUSY) return -EBUSY; for ( ; !ret && n--; addr += 4) { t3_write_reg(adap, A_CIM_HOST_ACC_CTRL, CIM_CTL_BASE + addr); ret = t3_wait_op_done(adap, A_CIM_HOST_ACC_CTRL, F_HOSTBUSY, 0, 5, 2); if (!ret) *valp++ = t3_read_reg(adap, A_CIM_HOST_ACC_DATA); } return ret; } /** * t3_link_changed - handle interface link changes * @adapter: the adapter * @port_id: the port index that changed link state * * Called when a port's link settings change to propagate the new values * to the associated PHY and MAC. After performing the common tasks it * invokes an OS-specific handler. */ void t3_link_changed(adapter_t *adapter, int port_id) { int link_ok, speed, duplex, fc; struct cphy *phy = &adapter->port[port_id].phy; struct cmac *mac = &adapter->port[port_id].mac; struct link_config *lc = &adapter->port[port_id].link_config; phy->ops->get_link_status(phy, &link_ok, &speed, &duplex, &fc); if (link_ok != lc->link_ok && adapter->params.rev > 0 && uses_xaui(adapter)) { if (link_ok) t3b_pcs_reset(mac); t3_write_reg(adapter, A_XGM_XAUI_ACT_CTRL + mac->offset, link_ok ? F_TXACTENABLE | F_RXEN : 0); } lc->link_ok = (unsigned char)link_ok; lc->speed = speed < 0 ? SPEED_INVALID : speed; lc->duplex = duplex < 0 ? DUPLEX_INVALID : duplex; if (lc->requested_fc & PAUSE_AUTONEG) fc &= lc->requested_fc; else fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX); if (link_ok && speed >= 0 && lc->autoneg == AUTONEG_ENABLE) { /* Set MAC speed, duplex, and flow control to match PHY. */ t3_mac_set_speed_duplex_fc(mac, speed, duplex, fc); lc->fc = (unsigned char)fc; } t3_os_link_changed(adapter, port_id, link_ok, speed, duplex, fc); } /** * t3_link_start - apply link configuration to MAC/PHY * @phy: the PHY to setup * @mac: the MAC to setup * @lc: the requested link configuration * * Set up a port's MAC and PHY according to a desired link configuration. * - If the PHY can auto-negotiate first decide what to advertise, then * enable/disable auto-negotiation as desired, and reset. * - If the PHY does not auto-negotiate just reset it. * - If auto-negotiation is off set the MAC to the proper speed/duplex/FC, * otherwise do it later based on the outcome of auto-negotiation. */ int t3_link_start(struct cphy *phy, struct cmac *mac, struct link_config *lc) { unsigned int fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX); lc->link_ok = 0; if (lc->supported & SUPPORTED_Autoneg) { lc->advertising &= ~(ADVERTISED_Asym_Pause | ADVERTISED_Pause); if (fc) { lc->advertising |= ADVERTISED_Asym_Pause; if (fc & PAUSE_RX) lc->advertising |= ADVERTISED_Pause; } phy->ops->advertise(phy, lc->advertising); if (lc->autoneg == AUTONEG_DISABLE) { lc->speed = lc->requested_speed; lc->duplex = lc->requested_duplex; lc->fc = (unsigned char)fc; t3_mac_set_speed_duplex_fc(mac, lc->speed, lc->duplex, fc); /* Also disables autoneg */ phy->ops->set_speed_duplex(phy, lc->speed, lc->duplex); phy->ops->reset(phy, 0); } else phy->ops->autoneg_enable(phy); } else { t3_mac_set_speed_duplex_fc(mac, -1, -1, fc); lc->fc = (unsigned char)fc; phy->ops->reset(phy, 0); } return 0; } /** * t3_set_vlan_accel - control HW VLAN extraction * @adapter: the adapter * @ports: bitmap of adapter ports to operate on * @on: enable (1) or disable (0) HW VLAN extraction * * Enables or disables HW extraction of VLAN tags for the given port. */ void t3_set_vlan_accel(adapter_t *adapter, unsigned int ports, int on) { t3_set_reg_field(adapter, A_TP_OUT_CONFIG, ports << S_VLANEXTRACTIONENABLE, on ? (ports << S_VLANEXTRACTIONENABLE) : 0); } struct intr_info { unsigned int mask; /* bits to check in interrupt status */ const char *msg; /* message to print or NULL */ short stat_idx; /* stat counter to increment or -1 */ unsigned short fatal:1; /* whether the condition reported is fatal */ }; /** * t3_handle_intr_status - table driven interrupt handler * @adapter: the adapter that generated the interrupt * @reg: the interrupt status register to process * @mask: a mask to apply to the interrupt status * @acts: table of interrupt actions * @stats: statistics counters tracking interrupt occurences * * A table driven interrupt handler that applies a set of masks to an * interrupt status word and performs the corresponding actions if the * interrupts described by the mask have occured. The actions include * optionally printing a warning or alert message, and optionally * incrementing a stat counter. The table is terminated by an entry * specifying mask 0. Returns the number of fatal interrupt conditions. */ static int t3_handle_intr_status(adapter_t *adapter, unsigned int reg, unsigned int mask, const struct intr_info *acts, unsigned long *stats) { int fatal = 0; unsigned int status = t3_read_reg(adapter, reg) & mask; for ( ; acts->mask; ++acts) { if (!(status & acts->mask)) continue; if (acts->fatal) { fatal++; CH_ALERT(adapter, "%s (0x%x)\n", acts->msg, status & acts->mask); } else if (acts->msg) CH_WARN(adapter, "%s (0x%x)\n", acts->msg, status & acts->mask); if (acts->stat_idx >= 0) stats[acts->stat_idx]++; } if (status) /* clear processed interrupts */ t3_write_reg(adapter, reg, status); return fatal; } #define SGE_INTR_MASK (F_RSPQDISABLED) #define MC5_INTR_MASK (F_PARITYERR | F_ACTRGNFULL | F_UNKNOWNCMD | \ F_REQQPARERR | F_DISPQPARERR | F_DELACTEMPTY | \ F_NFASRCHFAIL) #define MC7_INTR_MASK (F_AE | F_UE | F_CE | V_PE(M_PE)) #define XGM_INTR_MASK (V_TXFIFO_PRTY_ERR(M_TXFIFO_PRTY_ERR) | \ V_RXFIFO_PRTY_ERR(M_RXFIFO_PRTY_ERR) | \ F_TXFIFO_UNDERRUN | F_RXFIFO_OVERFLOW) #define PCIX_INTR_MASK (F_MSTDETPARERR | F_SIGTARABT | F_RCVTARABT | \ F_RCVMSTABT | F_SIGSYSERR | F_DETPARERR | \ F_SPLCMPDIS | F_UNXSPLCMP | F_RCVSPLCMPERR | \ F_DETCORECCERR | F_DETUNCECCERR | F_PIOPARERR | \ V_WFPARERR(M_WFPARERR) | V_RFPARERR(M_RFPARERR) | \ V_CFPARERR(M_CFPARERR) /* | V_MSIXPARERR(M_MSIXPARERR) */) #define PCIE_INTR_MASK (F_UNXSPLCPLERRR | F_UNXSPLCPLERRC | F_PCIE_PIOPARERR |\ F_PCIE_WFPARERR | F_PCIE_RFPARERR | F_PCIE_CFPARERR | \ /* V_PCIE_MSIXPARERR(M_PCIE_MSIXPARERR) | */ \ V_BISTERR(M_BISTERR) | F_PEXERR) #define ULPRX_INTR_MASK F_PARERR #define ULPTX_INTR_MASK 0 #define CPLSW_INTR_MASK (F_TP_FRAMING_ERROR | \ F_SGE_FRAMING_ERROR | F_CIM_FRAMING_ERROR | \ F_ZERO_SWITCH_ERROR) #define CIM_INTR_MASK (F_BLKWRPLINT | F_BLKRDPLINT | F_BLKWRCTLINT | \ F_BLKRDCTLINT | F_BLKWRFLASHINT | F_BLKRDFLASHINT | \ F_SGLWRFLASHINT | F_WRBLKFLASHINT | F_BLKWRBOOTINT | \ F_FLASHRANGEINT | F_SDRAMRANGEINT | F_RSVDSPACEINT) #define PMTX_INTR_MASK (F_ZERO_C_CMD_ERROR | ICSPI_FRM_ERR | OESPI_FRM_ERR | \ V_ICSPI_PAR_ERROR(M_ICSPI_PAR_ERROR) | \ V_OESPI_PAR_ERROR(M_OESPI_PAR_ERROR)) #define PMRX_INTR_MASK (F_ZERO_E_CMD_ERROR | IESPI_FRM_ERR | OCSPI_FRM_ERR | \ V_IESPI_PAR_ERROR(M_IESPI_PAR_ERROR) | \ V_OCSPI_PAR_ERROR(M_OCSPI_PAR_ERROR)) #define MPS_INTR_MASK (V_TX0TPPARERRENB(M_TX0TPPARERRENB) | \ V_TX1TPPARERRENB(M_TX1TPPARERRENB) | \ V_RXTPPARERRENB(M_RXTPPARERRENB) | \ V_MCAPARERRENB(M_MCAPARERRENB)) #define PL_INTR_MASK (F_T3DBG | F_XGMAC0_0 | F_XGMAC0_1 | F_MC5A | F_PM1_TX | \ F_PM1_RX | F_ULP2_TX | F_ULP2_RX | F_TP1 | F_CIM | \ F_MC7_CM | F_MC7_PMTX | F_MC7_PMRX | F_SGE3 | F_PCIM0 | \ F_MPS0 | F_CPL_SWITCH) /* * Interrupt handler for the PCIX1 module. */ static void pci_intr_handler(adapter_t *adapter) { static struct intr_info pcix1_intr_info[] = { { F_MSTDETPARERR, "PCI master detected parity error", -1, 1 }, { F_SIGTARABT, "PCI signaled target abort", -1, 1 }, { F_RCVTARABT, "PCI received target abort", -1, 1 }, { F_RCVMSTABT, "PCI received master abort", -1, 1 }, { F_SIGSYSERR, "PCI signaled system error", -1, 1 }, { F_DETPARERR, "PCI detected parity error", -1, 1 }, { F_SPLCMPDIS, "PCI split completion discarded", -1, 1 }, { F_UNXSPLCMP, "PCI unexpected split completion error", -1, 1 }, { F_RCVSPLCMPERR, "PCI received split completion error", -1, 1 }, { F_DETCORECCERR, "PCI correctable ECC error", STAT_PCI_CORR_ECC, 0 }, { F_DETUNCECCERR, "PCI uncorrectable ECC error", -1, 1 }, { F_PIOPARERR, "PCI PIO FIFO parity error", -1, 1 }, { V_WFPARERR(M_WFPARERR), "PCI write FIFO parity error", -1, 1 }, { V_RFPARERR(M_RFPARERR), "PCI read FIFO parity error", -1, 1 }, { V_CFPARERR(M_CFPARERR), "PCI command FIFO parity error", -1, 1 }, { V_MSIXPARERR(M_MSIXPARERR), "PCI MSI-X table/PBA parity " "error", -1, 1 }, { 0 } }; if (t3_handle_intr_status(adapter, A_PCIX_INT_CAUSE, PCIX_INTR_MASK, pcix1_intr_info, adapter->irq_stats)) t3_fatal_err(adapter); } /* * Interrupt handler for the PCIE module. */ static void pcie_intr_handler(adapter_t *adapter) { static struct intr_info pcie_intr_info[] = { { F_PEXERR, "PCI PEX error", -1, 1 }, { F_UNXSPLCPLERRR, "PCI unexpected split completion DMA read error", -1, 1 }, { F_UNXSPLCPLERRC, "PCI unexpected split completion DMA command error", -1, 1 }, { F_PCIE_PIOPARERR, "PCI PIO FIFO parity error", -1, 1 }, { F_PCIE_WFPARERR, "PCI write FIFO parity error", -1, 1 }, { F_PCIE_RFPARERR, "PCI read FIFO parity error", -1, 1 }, { F_PCIE_CFPARERR, "PCI command FIFO parity error", -1, 1 }, { V_PCIE_MSIXPARERR(M_PCIE_MSIXPARERR), "PCI MSI-X table/PBA parity error", -1, 1 }, { V_BISTERR(M_BISTERR), "PCI BIST error", -1, 1 }, { 0 } }; if (t3_handle_intr_status(adapter, A_PCIE_INT_CAUSE, PCIE_INTR_MASK, pcie_intr_info, adapter->irq_stats)) t3_fatal_err(adapter); } /* * TP interrupt handler. */ static void tp_intr_handler(adapter_t *adapter) { static struct intr_info tp_intr_info[] = { { 0xffffff, "TP parity error", -1, 1 }, { 0x1000000, "TP out of Rx pages", -1, 1 }, { 0x2000000, "TP out of Tx pages", -1, 1 }, { 0 } }; if (t3_handle_intr_status(adapter, A_TP_INT_CAUSE, 0xffffffff, tp_intr_info, NULL)) t3_fatal_err(adapter); } /* * CIM interrupt handler. */ static void cim_intr_handler(adapter_t *adapter) { static struct intr_info cim_intr_info[] = { { F_RSVDSPACEINT, "CIM reserved space write", -1, 1 }, { F_SDRAMRANGEINT, "CIM SDRAM address out of range", -1, 1 }, { F_FLASHRANGEINT, "CIM flash address out of range", -1, 1 }, { F_BLKWRBOOTINT, "CIM block write to boot space", -1, 1 }, { F_WRBLKFLASHINT, "CIM write to cached flash space", -1, 1 }, { F_SGLWRFLASHINT, "CIM single write to flash space", -1, 1 }, { F_BLKRDFLASHINT, "CIM block read from flash space", -1, 1 }, { F_BLKWRFLASHINT, "CIM block write to flash space", -1, 1 }, { F_BLKRDCTLINT, "CIM block read from CTL space", -1, 1 }, { F_BLKWRCTLINT, "CIM block write to CTL space", -1, 1 }, { F_BLKRDPLINT, "CIM block read from PL space", -1, 1 }, { F_BLKWRPLINT, "CIM block write to PL space", -1, 1 }, { 0 } }; if (t3_handle_intr_status(adapter, A_CIM_HOST_INT_CAUSE, 0xffffffff, cim_intr_info, NULL)) t3_fatal_err(adapter); } /* * ULP RX interrupt handler. */ static void ulprx_intr_handler(adapter_t *adapter) { static struct intr_info ulprx_intr_info[] = { { F_PARERR, "ULP RX parity error", -1, 1 }, { 0 } }; if (t3_handle_intr_status(adapter, A_ULPRX_INT_CAUSE, 0xffffffff, ulprx_intr_info, NULL)) t3_fatal_err(adapter); } /* * ULP TX interrupt handler. */ static void ulptx_intr_handler(adapter_t *adapter) { static struct intr_info ulptx_intr_info[] = { { F_PBL_BOUND_ERR_CH0, "ULP TX channel 0 PBL out of bounds", STAT_ULP_CH0_PBL_OOB, 0 }, { F_PBL_BOUND_ERR_CH1, "ULP TX channel 1 PBL out of bounds", STAT_ULP_CH1_PBL_OOB, 0 }, { 0 } }; if (t3_handle_intr_status(adapter, A_ULPTX_INT_CAUSE, 0xffffffff, ulptx_intr_info, adapter->irq_stats)) t3_fatal_err(adapter); } #define ICSPI_FRM_ERR (F_ICSPI0_FIFO2X_RX_FRAMING_ERROR | \ F_ICSPI1_FIFO2X_RX_FRAMING_ERROR | F_ICSPI0_RX_FRAMING_ERROR | \ F_ICSPI1_RX_FRAMING_ERROR | F_ICSPI0_TX_FRAMING_ERROR | \ F_ICSPI1_TX_FRAMING_ERROR) #define OESPI_FRM_ERR (F_OESPI0_RX_FRAMING_ERROR | \ F_OESPI1_RX_FRAMING_ERROR | F_OESPI0_TX_FRAMING_ERROR | \ F_OESPI1_TX_FRAMING_ERROR | F_OESPI0_OFIFO2X_TX_FRAMING_ERROR | \ F_OESPI1_OFIFO2X_TX_FRAMING_ERROR) /* * PM TX interrupt handler. */ static void pmtx_intr_handler(adapter_t *adapter) { static struct intr_info pmtx_intr_info[] = { { F_ZERO_C_CMD_ERROR, "PMTX 0-length pcmd", -1, 1 }, { ICSPI_FRM_ERR, "PMTX ispi framing error", -1, 1 }, { OESPI_FRM_ERR, "PMTX ospi framing error", -1, 1 }, { V_ICSPI_PAR_ERROR(M_ICSPI_PAR_ERROR), "PMTX ispi parity error", -1, 1 }, { V_OESPI_PAR_ERROR(M_OESPI_PAR_ERROR), "PMTX ospi parity error", -1, 1 }, { 0 } }; if (t3_handle_intr_status(adapter, A_PM1_TX_INT_CAUSE, 0xffffffff, pmtx_intr_info, NULL)) t3_fatal_err(adapter); } #define IESPI_FRM_ERR (F_IESPI0_FIFO2X_RX_FRAMING_ERROR | \ F_IESPI1_FIFO2X_RX_FRAMING_ERROR | F_IESPI0_RX_FRAMING_ERROR | \ F_IESPI1_RX_FRAMING_ERROR | F_IESPI0_TX_FRAMING_ERROR | \ F_IESPI1_TX_FRAMING_ERROR) #define OCSPI_FRM_ERR (F_OCSPI0_RX_FRAMING_ERROR | \ F_OCSPI1_RX_FRAMING_ERROR | F_OCSPI0_TX_FRAMING_ERROR | \ F_OCSPI1_TX_FRAMING_ERROR | F_OCSPI0_OFIFO2X_TX_FRAMING_ERROR | \ F_OCSPI1_OFIFO2X_TX_FRAMING_ERROR) /* * PM RX interrupt handler. */ static void pmrx_intr_handler(adapter_t *adapter) { static struct intr_info pmrx_intr_info[] = { { F_ZERO_E_CMD_ERROR, "PMRX 0-length pcmd", -1, 1 }, { IESPI_FRM_ERR, "PMRX ispi framing error", -1, 1 }, { OCSPI_FRM_ERR, "PMRX ospi framing error", -1, 1 }, { V_IESPI_PAR_ERROR(M_IESPI_PAR_ERROR), "PMRX ispi parity error", -1, 1 }, { V_OCSPI_PAR_ERROR(M_OCSPI_PAR_ERROR), "PMRX ospi parity error", -1, 1 }, { 0 } }; if (t3_handle_intr_status(adapter, A_PM1_RX_INT_CAUSE, 0xffffffff, pmrx_intr_info, NULL)) t3_fatal_err(adapter); } /* * CPL switch interrupt handler. */ static void cplsw_intr_handler(adapter_t *adapter) { static struct intr_info cplsw_intr_info[] = { // { F_CIM_OVFL_ERROR, "CPL switch CIM overflow", -1, 1 }, { F_TP_FRAMING_ERROR, "CPL switch TP framing error", -1, 1 }, { F_SGE_FRAMING_ERROR, "CPL switch SGE framing error", -1, 1 }, { F_CIM_FRAMING_ERROR, "CPL switch CIM framing error", -1, 1 }, { F_ZERO_SWITCH_ERROR, "CPL switch no-switch error", -1, 1 }, { 0 } }; if (t3_handle_intr_status(adapter, A_CPL_INTR_CAUSE, 0xffffffff, cplsw_intr_info, NULL)) t3_fatal_err(adapter); } /* * MPS interrupt handler. */ static void mps_intr_handler(adapter_t *adapter) { static struct intr_info mps_intr_info[] = { { 0x1ff, "MPS parity error", -1, 1 }, { 0 } }; if (t3_handle_intr_status(adapter, A_MPS_INT_CAUSE, 0xffffffff, mps_intr_info, NULL)) t3_fatal_err(adapter); } #define MC7_INTR_FATAL (F_UE | V_PE(M_PE) | F_AE) /* * MC7 interrupt handler. */ static void mc7_intr_handler(struct mc7 *mc7) { adapter_t *adapter = mc7->adapter; u32 cause = t3_read_reg(adapter, mc7->offset + A_MC7_INT_CAUSE); if (cause & F_CE) { mc7->stats.corr_err++; CH_WARN(adapter, "%s MC7 correctable error at addr 0x%x, " "data 0x%x 0x%x 0x%x\n", mc7->name, t3_read_reg(adapter, mc7->offset + A_MC7_CE_ADDR), t3_read_reg(adapter, mc7->offset + A_MC7_CE_DATA0), t3_read_reg(adapter, mc7->offset + A_MC7_CE_DATA1), t3_read_reg(adapter, mc7->offset + A_MC7_CE_DATA2)); } if (cause & F_UE) { mc7->stats.uncorr_err++; CH_ALERT(adapter, "%s MC7 uncorrectable error at addr 0x%x, " "data 0x%x 0x%x 0x%x\n", mc7->name, t3_read_reg(adapter, mc7->offset + A_MC7_UE_ADDR), t3_read_reg(adapter, mc7->offset + A_MC7_UE_DATA0), t3_read_reg(adapter, mc7->offset + A_MC7_UE_DATA1), t3_read_reg(adapter, mc7->offset + A_MC7_UE_DATA2)); } if (G_PE(cause)) { mc7->stats.parity_err++; CH_ALERT(adapter, "%s MC7 parity error 0x%x\n", mc7->name, G_PE(cause)); } if (cause & F_AE) { u32 addr = 0; if (adapter->params.rev > 0) addr = t3_read_reg(adapter, mc7->offset + A_MC7_ERR_ADDR); mc7->stats.addr_err++; CH_ALERT(adapter, "%s MC7 address error: 0x%x\n", mc7->name, addr); } if (cause & MC7_INTR_FATAL) t3_fatal_err(adapter); t3_write_reg(adapter, mc7->offset + A_MC7_INT_CAUSE, cause); } #define XGM_INTR_FATAL (V_TXFIFO_PRTY_ERR(M_TXFIFO_PRTY_ERR) | \ V_RXFIFO_PRTY_ERR(M_RXFIFO_PRTY_ERR)) /* * XGMAC interrupt handler. */ static int mac_intr_handler(adapter_t *adap, unsigned int idx) { struct cmac *mac = &adap->port[idx].mac; u32 cause = t3_read_reg(adap, A_XGM_INT_CAUSE + mac->offset); if (cause & V_TXFIFO_PRTY_ERR(M_TXFIFO_PRTY_ERR)) { mac->stats.tx_fifo_parity_err++; CH_ALERT(adap, "port%d: MAC TX FIFO parity error\n", idx); } if (cause & V_RXFIFO_PRTY_ERR(M_RXFIFO_PRTY_ERR)) { mac->stats.rx_fifo_parity_err++; CH_ALERT(adap, "port%d: MAC RX FIFO parity error\n", idx); } if (cause & F_TXFIFO_UNDERRUN) mac->stats.tx_fifo_urun++; if (cause & F_RXFIFO_OVERFLOW) mac->stats.rx_fifo_ovfl++; if (cause & V_SERDES_LOS(M_SERDES_LOS)) mac->stats.serdes_signal_loss++; if (cause & F_XAUIPCSCTCERR) mac->stats.xaui_pcs_ctc_err++; if (cause & F_XAUIPCSALIGNCHANGE) mac->stats.xaui_pcs_align_change++; t3_write_reg(adap, A_XGM_INT_CAUSE + mac->offset, cause); if (cause & XGM_INTR_FATAL) t3_fatal_err(adap); return cause != 0; } /* * Interrupt handler for PHY events. */ int t3_phy_intr_handler(adapter_t *adapter) { u32 mask, gpi = adapter_info(adapter)->gpio_intr; u32 i, cause = t3_read_reg(adapter, A_T3DBG_INT_CAUSE); for_each_port(adapter, i) { struct port_info *p = &adapter->port[i]; mask = gpi - (gpi & (gpi - 1)); gpi -= mask; if (!(p->port_type->caps & SUPPORTED_IRQ)) continue; if (cause & mask) { int phy_cause = p->phy.ops->intr_handler(&p->phy); if (phy_cause & cphy_cause_link_change) t3_link_changed(adapter, i); if (phy_cause & cphy_cause_fifo_error) p->phy.fifo_errors++; } } t3_write_reg(adapter, A_T3DBG_INT_CAUSE, cause); return 0; } /* * T3 slow path (non-data) interrupt handler. */ int t3_slow_intr_handler(adapter_t *adapter) { u32 cause = t3_read_reg(adapter, A_PL_INT_CAUSE0); cause &= adapter->slow_intr_mask; if (!cause) return 0; if (cause & F_PCIM0) { if (is_pcie(adapter)) pcie_intr_handler(adapter); else pci_intr_handler(adapter); } if (cause & F_SGE3) t3_sge_err_intr_handler(adapter); if (cause & F_MC7_PMRX) mc7_intr_handler(&adapter->pmrx); if (cause & F_MC7_PMTX) mc7_intr_handler(&adapter->pmtx); if (cause & F_MC7_CM) mc7_intr_handler(&adapter->cm); if (cause & F_CIM) cim_intr_handler(adapter); if (cause & F_TP1) tp_intr_handler(adapter); if (cause & F_ULP2_RX) ulprx_intr_handler(adapter); if (cause & F_ULP2_TX) ulptx_intr_handler(adapter); if (cause & F_PM1_RX) pmrx_intr_handler(adapter); if (cause & F_PM1_TX) pmtx_intr_handler(adapter); if (cause & F_CPL_SWITCH) cplsw_intr_handler(adapter); if (cause & F_MPS0) mps_intr_handler(adapter); if (cause & F_MC5A) t3_mc5_intr_handler(&adapter->mc5); if (cause & F_XGMAC0_0) mac_intr_handler(adapter, 0); if (cause & F_XGMAC0_1) mac_intr_handler(adapter, 1); if (cause & F_T3DBG) t3_os_ext_intr_handler(adapter); /* Clear the interrupts just processed. */ t3_write_reg(adapter, A_PL_INT_CAUSE0, cause); (void) t3_read_reg(adapter, A_PL_INT_CAUSE0); /* flush */ return 1; } /** * t3_intr_enable - enable interrupts * @adapter: the adapter whose interrupts should be enabled * * Enable interrupts by setting the interrupt enable registers of the * various HW modules and then enabling the top-level interrupt * concentrator. */ void t3_intr_enable(adapter_t *adapter) { static struct addr_val_pair intr_en_avp[] = { { A_SG_INT_ENABLE, SGE_INTR_MASK }, { A_MC7_INT_ENABLE, MC7_INTR_MASK }, { A_MC7_INT_ENABLE - MC7_PMRX_BASE_ADDR + MC7_PMTX_BASE_ADDR, MC7_INTR_MASK }, { A_MC7_INT_ENABLE - MC7_PMRX_BASE_ADDR + MC7_CM_BASE_ADDR, MC7_INTR_MASK }, { A_MC5_DB_INT_ENABLE, MC5_INTR_MASK }, { A_ULPRX_INT_ENABLE, ULPRX_INTR_MASK }, { A_TP_INT_ENABLE, 0x3bfffff }, { A_PM1_TX_INT_ENABLE, PMTX_INTR_MASK }, { A_PM1_RX_INT_ENABLE, PMRX_INTR_MASK }, { A_CIM_HOST_INT_ENABLE, CIM_INTR_MASK }, { A_MPS_INT_ENABLE, MPS_INTR_MASK }, }; adapter->slow_intr_mask = PL_INTR_MASK; t3_write_regs(adapter, intr_en_avp, ARRAY_SIZE(intr_en_avp), 0); if (adapter->params.rev > 0) { t3_write_reg(adapter, A_CPL_INTR_ENABLE, CPLSW_INTR_MASK | F_CIM_OVFL_ERROR); t3_write_reg(adapter, A_ULPTX_INT_ENABLE, ULPTX_INTR_MASK | F_PBL_BOUND_ERR_CH0 | F_PBL_BOUND_ERR_CH1); } else { t3_write_reg(adapter, A_CPL_INTR_ENABLE, CPLSW_INTR_MASK); t3_write_reg(adapter, A_ULPTX_INT_ENABLE, ULPTX_INTR_MASK); } t3_write_reg(adapter, A_T3DBG_GPIO_ACT_LOW, adapter_info(adapter)->gpio_intr); t3_write_reg(adapter, A_T3DBG_INT_ENABLE, adapter_info(adapter)->gpio_intr); if (is_pcie(adapter)) { t3_write_reg(adapter, A_PCIE_INT_ENABLE, PCIE_INTR_MASK); } else { t3_write_reg(adapter, A_PCIX_INT_ENABLE, PCIX_INTR_MASK); } t3_write_reg(adapter, A_PL_INT_ENABLE0, adapter->slow_intr_mask); (void) t3_read_reg(adapter, A_PL_INT_ENABLE0); /* flush */ } /** * t3_intr_disable - disable a card's interrupts * @adapter: the adapter whose interrupts should be disabled * * Disable interrupts. We only disable the top-level interrupt * concentrator and the SGE data interrupts. */ void t3_intr_disable(adapter_t *adapter) { t3_write_reg(adapter, A_PL_INT_ENABLE0, 0); (void) t3_read_reg(adapter, A_PL_INT_ENABLE0); /* flush */ adapter->slow_intr_mask = 0; } /** * t3_intr_clear - clear all interrupts * @adapter: the adapter whose interrupts should be cleared * * Clears all interrupts. */ void t3_intr_clear(adapter_t *adapter) { static const unsigned int cause_reg_addr[] = { A_SG_INT_CAUSE, A_SG_RSPQ_FL_STATUS, A_PCIX_INT_CAUSE, A_MC7_INT_CAUSE, A_MC7_INT_CAUSE - MC7_PMRX_BASE_ADDR + MC7_PMTX_BASE_ADDR, A_MC7_INT_CAUSE - MC7_PMRX_BASE_ADDR + MC7_CM_BASE_ADDR, A_CIM_HOST_INT_CAUSE, A_TP_INT_CAUSE, A_MC5_DB_INT_CAUSE, A_ULPRX_INT_CAUSE, A_ULPTX_INT_CAUSE, A_CPL_INTR_CAUSE, A_PM1_TX_INT_CAUSE, A_PM1_RX_INT_CAUSE, A_MPS_INT_CAUSE, A_T3DBG_INT_CAUSE, }; unsigned int i; /* Clear PHY and MAC interrupts for each port. */ for_each_port(adapter, i) t3_port_intr_clear(adapter, i); for (i = 0; i < ARRAY_SIZE(cause_reg_addr); ++i) t3_write_reg(adapter, cause_reg_addr[i], 0xffffffff); t3_write_reg(adapter, A_PL_INT_CAUSE0, 0xffffffff); (void) t3_read_reg(adapter, A_PL_INT_CAUSE0); /* flush */ } /** * t3_port_intr_enable - enable port-specific interrupts * @adapter: associated adapter * @idx: index of port whose interrupts should be enabled * * Enable port-specific (i.e., MAC and PHY) interrupts for the given * adapter port. */ void t3_port_intr_enable(adapter_t *adapter, int idx) { t3_write_reg(adapter, XGM_REG(A_XGM_INT_ENABLE, idx), XGM_INTR_MASK); adapter->port[idx].phy.ops->intr_enable(&adapter->port[idx].phy); } /** * t3_port_intr_disable - disable port-specific interrupts * @adapter: associated adapter * @idx: index of port whose interrupts should be disabled * * Disable port-specific (i.e., MAC and PHY) interrupts for the given * adapter port. */ void t3_port_intr_disable(adapter_t *adapter, int idx) { t3_write_reg(adapter, XGM_REG(A_XGM_INT_ENABLE, idx), 0); adapter->port[idx].phy.ops->intr_disable(&adapter->port[idx].phy); } /** * t3_port_intr_clear - clear port-specific interrupts * @adapter: associated adapter * @idx: index of port whose interrupts to clear * * Clear port-specific (i.e., MAC and PHY) interrupts for the given * adapter port. */ void t3_port_intr_clear(adapter_t *adapter, int idx) { t3_write_reg(adapter, XGM_REG(A_XGM_INT_CAUSE, idx), 0xffffffff); adapter->port[idx].phy.ops->intr_clear(&adapter->port[idx].phy); } /** * t3_sge_write_context - write an SGE context * @adapter: the adapter * @id: the context id * @type: the context type * * Program an SGE context with the values already loaded in the * CONTEXT_DATA? registers. */ static int t3_sge_write_context(adapter_t *adapter, unsigned int id, unsigned int type) { t3_write_reg(adapter, A_SG_CONTEXT_MASK0, 0xffffffff); t3_write_reg(adapter, A_SG_CONTEXT_MASK1, 0xffffffff); t3_write_reg(adapter, A_SG_CONTEXT_MASK2, 0xffffffff); t3_write_reg(adapter, A_SG_CONTEXT_MASK3, 0xffffffff); t3_write_reg(adapter, A_SG_CONTEXT_CMD, V_CONTEXT_CMD_OPCODE(1) | type | V_CONTEXT(id)); return t3_wait_op_done(adapter, A_SG_CONTEXT_CMD, F_CONTEXT_CMD_BUSY, 0, 5, 1); } /** * t3_sge_init_ecntxt - initialize an SGE egress context * @adapter: the adapter to configure * @id: the context id * @gts_enable: whether to enable GTS for the context * @type: the egress context type * @respq: associated response queue * @base_addr: base address of queue * @size: number of queue entries * @token: uP token * @gen: initial generation value for the context * @cidx: consumer pointer * * Initialize an SGE egress context and make it ready for use. If the * platform allows concurrent context operations, the caller is * responsible for appropriate locking. */ int t3_sge_init_ecntxt(adapter_t *adapter, unsigned int id, int gts_enable, enum sge_context_type type, int respq, u64 base_addr, unsigned int size, unsigned int token, int gen, unsigned int cidx) { unsigned int credits = type == SGE_CNTXT_OFLD ? 0 : FW_WR_NUM; if (base_addr & 0xfff) /* must be 4K aligned */ return -EINVAL; if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) return -EBUSY; base_addr >>= 12; t3_write_reg(adapter, A_SG_CONTEXT_DATA0, V_EC_INDEX(cidx) | V_EC_CREDITS(credits) | V_EC_GTS(gts_enable)); t3_write_reg(adapter, A_SG_CONTEXT_DATA1, V_EC_SIZE(size) | V_EC_BASE_LO((u32)base_addr & 0xffff)); base_addr >>= 16; t3_write_reg(adapter, A_SG_CONTEXT_DATA2, (u32)base_addr); base_addr >>= 32; t3_write_reg(adapter, A_SG_CONTEXT_DATA3, V_EC_BASE_HI((u32)base_addr & 0xf) | V_EC_RESPQ(respq) | V_EC_TYPE(type) | V_EC_GEN(gen) | V_EC_UP_TOKEN(token) | F_EC_VALID); return t3_sge_write_context(adapter, id, F_EGRESS); } /** * t3_sge_init_flcntxt - initialize an SGE free-buffer list context * @adapter: the adapter to configure * @id: the context id * @gts_enable: whether to enable GTS for the context * @base_addr: base address of queue * @size: number of queue entries * @bsize: size of each buffer for this queue * @cong_thres: threshold to signal congestion to upstream producers * @gen: initial generation value for the context * @cidx: consumer pointer * * Initialize an SGE free list context and make it ready for use. The * caller is responsible for ensuring only one context operation occurs * at a time. */ int t3_sge_init_flcntxt(adapter_t *adapter, unsigned int id, int gts_enable, u64 base_addr, unsigned int size, unsigned int bsize, unsigned int cong_thres, int gen, unsigned int cidx) { if (base_addr & 0xfff) /* must be 4K aligned */ return -EINVAL; if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) return -EBUSY; base_addr >>= 12; t3_write_reg(adapter, A_SG_CONTEXT_DATA0, (u32)base_addr); base_addr >>= 32; t3_write_reg(adapter, A_SG_CONTEXT_DATA1, V_FL_BASE_HI((u32)base_addr) | V_FL_INDEX_LO(cidx & M_FL_INDEX_LO)); t3_write_reg(adapter, A_SG_CONTEXT_DATA2, V_FL_SIZE(size) | V_FL_GEN(gen) | V_FL_INDEX_HI(cidx >> 12) | V_FL_ENTRY_SIZE_LO(bsize & M_FL_ENTRY_SIZE_LO)); t3_write_reg(adapter, A_SG_CONTEXT_DATA3, V_FL_ENTRY_SIZE_HI(bsize >> (32 - S_FL_ENTRY_SIZE_LO)) | V_FL_CONG_THRES(cong_thres) | V_FL_GTS(gts_enable)); return t3_sge_write_context(adapter, id, F_FREELIST); } /** * t3_sge_init_rspcntxt - initialize an SGE response queue context * @adapter: the adapter to configure * @id: the context id * @irq_vec_idx: MSI-X interrupt vector index, 0 if no MSI-X, -1 if no IRQ * @base_addr: base address of queue * @size: number of queue entries * @fl_thres: threshold for selecting the normal or jumbo free list * @gen: initial generation value for the context * @cidx: consumer pointer * * Initialize an SGE response queue context and make it ready for use. * The caller is responsible for ensuring only one context operation * occurs at a time. */ int t3_sge_init_rspcntxt(adapter_t *adapter, unsigned int id, int irq_vec_idx, u64 base_addr, unsigned int size, unsigned int fl_thres, int gen, unsigned int cidx) { unsigned int intr = 0; if (base_addr & 0xfff) /* must be 4K aligned */ return -EINVAL; if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) return -EBUSY; base_addr >>= 12; t3_write_reg(adapter, A_SG_CONTEXT_DATA0, V_CQ_SIZE(size) | V_CQ_INDEX(cidx)); t3_write_reg(adapter, A_SG_CONTEXT_DATA1, (u32)base_addr); base_addr >>= 32; if (irq_vec_idx >= 0) intr = V_RQ_MSI_VEC(irq_vec_idx) | F_RQ_INTR_EN; t3_write_reg(adapter, A_SG_CONTEXT_DATA2, V_CQ_BASE_HI((u32)base_addr) | intr | V_RQ_GEN(gen)); t3_write_reg(adapter, A_SG_CONTEXT_DATA3, fl_thres); return t3_sge_write_context(adapter, id, F_RESPONSEQ); } /** * t3_sge_init_cqcntxt - initialize an SGE completion queue context * @adapter: the adapter to configure * @id: the context id * @base_addr: base address of queue * @size: number of queue entries * @rspq: response queue for async notifications * @ovfl_mode: CQ overflow mode * @credits: completion queue credits * @credit_thres: the credit threshold * * Initialize an SGE completion queue context and make it ready for use. * The caller is responsible for ensuring only one context operation * occurs at a time. */ int t3_sge_init_cqcntxt(adapter_t *adapter, unsigned int id, u64 base_addr, unsigned int size, int rspq, int ovfl_mode, unsigned int credits, unsigned int credit_thres) { if (base_addr & 0xfff) /* must be 4K aligned */ return -EINVAL; if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) return -EBUSY; base_addr >>= 12; t3_write_reg(adapter, A_SG_CONTEXT_DATA0, V_CQ_SIZE(size)); t3_write_reg(adapter, A_SG_CONTEXT_DATA1, (u32)base_addr); base_addr >>= 32; t3_write_reg(adapter, A_SG_CONTEXT_DATA2, V_CQ_BASE_HI((u32)base_addr) | V_CQ_RSPQ(rspq) | V_CQ_GEN(1) | V_CQ_OVERFLOW_MODE(ovfl_mode)); t3_write_reg(adapter, A_SG_CONTEXT_DATA3, V_CQ_CREDITS(credits) | V_CQ_CREDIT_THRES(credit_thres)); return t3_sge_write_context(adapter, id, F_CQ); } /** * t3_sge_enable_ecntxt - enable/disable an SGE egress context * @adapter: the adapter * @id: the egress context id * @enable: enable (1) or disable (0) the context * * Enable or disable an SGE egress context. The caller is responsible for * ensuring only one context operation occurs at a time. */ int t3_sge_enable_ecntxt(adapter_t *adapter, unsigned int id, int enable) { if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) return -EBUSY; t3_write_reg(adapter, A_SG_CONTEXT_MASK0, 0); t3_write_reg(adapter, A_SG_CONTEXT_MASK1, 0); t3_write_reg(adapter, A_SG_CONTEXT_MASK2, 0); t3_write_reg(adapter, A_SG_CONTEXT_MASK3, F_EC_VALID); t3_write_reg(adapter, A_SG_CONTEXT_DATA3, V_EC_VALID(enable)); t3_write_reg(adapter, A_SG_CONTEXT_CMD, V_CONTEXT_CMD_OPCODE(1) | F_EGRESS | V_CONTEXT(id)); return t3_wait_op_done(adapter, A_SG_CONTEXT_CMD, F_CONTEXT_CMD_BUSY, 0, 5, 1); } /** * t3_sge_disable_fl - disable an SGE free-buffer list * @adapter: the adapter * @id: the free list context id * * Disable an SGE free-buffer list. The caller is responsible for * ensuring only one context operation occurs at a time. */ int t3_sge_disable_fl(adapter_t *adapter, unsigned int id) { if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) return -EBUSY; t3_write_reg(adapter, A_SG_CONTEXT_MASK0, 0); t3_write_reg(adapter, A_SG_CONTEXT_MASK1, 0); t3_write_reg(adapter, A_SG_CONTEXT_MASK2, V_FL_SIZE(M_FL_SIZE)); t3_write_reg(adapter, A_SG_CONTEXT_MASK3, 0); t3_write_reg(adapter, A_SG_CONTEXT_DATA2, 0); t3_write_reg(adapter, A_SG_CONTEXT_CMD, V_CONTEXT_CMD_OPCODE(1) | F_FREELIST | V_CONTEXT(id)); return t3_wait_op_done(adapter, A_SG_CONTEXT_CMD, F_CONTEXT_CMD_BUSY, 0, 5, 1); } /** * t3_sge_disable_rspcntxt - disable an SGE response queue * @adapter: the adapter * @id: the response queue context id * * Disable an SGE response queue. The caller is responsible for * ensuring only one context operation occurs at a time. */ int t3_sge_disable_rspcntxt(adapter_t *adapter, unsigned int id) { if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) return -EBUSY; t3_write_reg(adapter, A_SG_CONTEXT_MASK0, V_CQ_SIZE(M_CQ_SIZE)); t3_write_reg(adapter, A_SG_CONTEXT_MASK1, 0); t3_write_reg(adapter, A_SG_CONTEXT_MASK2, 0); t3_write_reg(adapter, A_SG_CONTEXT_MASK3, 0); t3_write_reg(adapter, A_SG_CONTEXT_DATA0, 0); t3_write_reg(adapter, A_SG_CONTEXT_CMD, V_CONTEXT_CMD_OPCODE(1) | F_RESPONSEQ | V_CONTEXT(id)); return t3_wait_op_done(adapter, A_SG_CONTEXT_CMD, F_CONTEXT_CMD_BUSY, 0, 5, 1); } /** * t3_sge_disable_cqcntxt - disable an SGE completion queue * @adapter: the adapter * @id: the completion queue context id * * Disable an SGE completion queue. The caller is responsible for * ensuring only one context operation occurs at a time. */ int t3_sge_disable_cqcntxt(adapter_t *adapter, unsigned int id) { if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) return -EBUSY; t3_write_reg(adapter, A_SG_CONTEXT_MASK0, V_CQ_SIZE(M_CQ_SIZE)); t3_write_reg(adapter, A_SG_CONTEXT_MASK1, 0); t3_write_reg(adapter, A_SG_CONTEXT_MASK2, 0); t3_write_reg(adapter, A_SG_CONTEXT_MASK3, 0); t3_write_reg(adapter, A_SG_CONTEXT_DATA0, 0); t3_write_reg(adapter, A_SG_CONTEXT_CMD, V_CONTEXT_CMD_OPCODE(1) | F_CQ | V_CONTEXT(id)); return t3_wait_op_done(adapter, A_SG_CONTEXT_CMD, F_CONTEXT_CMD_BUSY, 0, 5, 1); } /** * t3_sge_cqcntxt_op - perform an operation on a completion queue context * @adapter: the adapter * @id: the context id * @op: the operation to perform * * Perform the selected operation on an SGE completion queue context. * The caller is responsible for ensuring only one context operation * occurs at a time. */ int t3_sge_cqcntxt_op(adapter_t *adapter, unsigned int id, unsigned int op, unsigned int credits) { u32 val; if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) return -EBUSY; t3_write_reg(adapter, A_SG_CONTEXT_DATA0, credits << 16); t3_write_reg(adapter, A_SG_CONTEXT_CMD, V_CONTEXT_CMD_OPCODE(op) | V_CONTEXT(id) | F_CQ); if (t3_wait_op_done_val(adapter, A_SG_CONTEXT_CMD, F_CONTEXT_CMD_BUSY, 0, 5, 1, &val)) return -EIO; if (op >= 2 && op < 7) { if (adapter->params.rev > 0) return G_CQ_INDEX(val); t3_write_reg(adapter, A_SG_CONTEXT_CMD, V_CONTEXT_CMD_OPCODE(0) | F_CQ | V_CONTEXT(id)); if (t3_wait_op_done(adapter, A_SG_CONTEXT_CMD, F_CONTEXT_CMD_BUSY, 0, 5, 1)) return -EIO; return G_CQ_INDEX(t3_read_reg(adapter, A_SG_CONTEXT_DATA0)); } return 0; } /** * t3_sge_read_context - read an SGE context * @type: the context type * @adapter: the adapter * @id: the context id * @data: holds the retrieved context * * Read an SGE egress context. The caller is responsible for ensuring * only one context operation occurs at a time. */ static int t3_sge_read_context(unsigned int type, adapter_t *adapter, unsigned int id, u32 data[4]) { if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) return -EBUSY; t3_write_reg(adapter, A_SG_CONTEXT_CMD, V_CONTEXT_CMD_OPCODE(0) | type | V_CONTEXT(id)); if (t3_wait_op_done(adapter, A_SG_CONTEXT_CMD, F_CONTEXT_CMD_BUSY, 0, 5, 1)) return -EIO; data[0] = t3_read_reg(adapter, A_SG_CONTEXT_DATA0); data[1] = t3_read_reg(adapter, A_SG_CONTEXT_DATA1); data[2] = t3_read_reg(adapter, A_SG_CONTEXT_DATA2); data[3] = t3_read_reg(adapter, A_SG_CONTEXT_DATA3); return 0; } /** * t3_sge_read_ecntxt - read an SGE egress context * @adapter: the adapter * @id: the context id * @data: holds the retrieved context * * Read an SGE egress context. The caller is responsible for ensuring * only one context operation occurs at a time. */ int t3_sge_read_ecntxt(adapter_t *adapter, unsigned int id, u32 data[4]) { if (id >= 65536) return -EINVAL; return t3_sge_read_context(F_EGRESS, adapter, id, data); } /** * t3_sge_read_cq - read an SGE CQ context * @adapter: the adapter * @id: the context id * @data: holds the retrieved context * * Read an SGE CQ context. The caller is responsible for ensuring * only one context operation occurs at a time. */ int t3_sge_read_cq(adapter_t *adapter, unsigned int id, u32 data[4]) { if (id >= 65536) return -EINVAL; return t3_sge_read_context(F_CQ, adapter, id, data); } /** * t3_sge_read_fl - read an SGE free-list context * @adapter: the adapter * @id: the context id * @data: holds the retrieved context * * Read an SGE free-list context. The caller is responsible for ensuring * only one context operation occurs at a time. */ int t3_sge_read_fl(adapter_t *adapter, unsigned int id, u32 data[4]) { if (id >= SGE_QSETS * 2) return -EINVAL; return t3_sge_read_context(F_FREELIST, adapter, id, data); } /** * t3_sge_read_rspq - read an SGE response queue context * @adapter: the adapter * @id: the context id * @data: holds the retrieved context * * Read an SGE response queue context. The caller is responsible for * ensuring only one context operation occurs at a time. */ int t3_sge_read_rspq(adapter_t *adapter, unsigned int id, u32 data[4]) { if (id >= SGE_QSETS) return -EINVAL; return t3_sge_read_context(F_RESPONSEQ, adapter, id, data); } /** * t3_config_rss - configure Rx packet steering * @adapter: the adapter * @rss_config: RSS settings (written to TP_RSS_CONFIG) * @cpus: values for the CPU lookup table (0xff terminated) * @rspq: values for the response queue lookup table (0xffff terminated) * * Programs the receive packet steering logic. @cpus and @rspq provide * the values for the CPU and response queue lookup tables. If they * provide fewer values than the size of the tables the supplied values * are used repeatedly until the tables are fully populated. */ void t3_config_rss(adapter_t *adapter, unsigned int rss_config, const u8 *cpus, const u16 *rspq) { int i, j, cpu_idx = 0, q_idx = 0; if (cpus) for (i = 0; i < RSS_TABLE_SIZE; ++i) { u32 val = i << 16; for (j = 0; j < 2; ++j) { val |= (cpus[cpu_idx++] & 0x3f) << (8 * j); if (cpus[cpu_idx] == 0xff) cpu_idx = 0; } t3_write_reg(adapter, A_TP_RSS_LKP_TABLE, val); } if (rspq) for (i = 0; i < RSS_TABLE_SIZE; ++i) { t3_write_reg(adapter, A_TP_RSS_MAP_TABLE, (i << 16) | rspq[q_idx++]); if (rspq[q_idx] == 0xffff) q_idx = 0; } t3_write_reg(adapter, A_TP_RSS_CONFIG, rss_config); } /** * t3_read_rss - read the contents of the RSS tables * @adapter: the adapter * @lkup: holds the contents of the RSS lookup table * @map: holds the contents of the RSS map table * * Reads the contents of the receive packet steering tables. */ int t3_read_rss(adapter_t *adapter, u8 *lkup, u16 *map) { int i; u32 val; if (lkup) for (i = 0; i < RSS_TABLE_SIZE; ++i) { t3_write_reg(adapter, A_TP_RSS_LKP_TABLE, 0xffff0000 | i); val = t3_read_reg(adapter, A_TP_RSS_LKP_TABLE); if (!(val & 0x80000000)) return -EAGAIN; *lkup++ = (u8)val; *lkup++ = (u8)(val >> 8); } if (map) for (i = 0; i < RSS_TABLE_SIZE; ++i) { t3_write_reg(adapter, A_TP_RSS_MAP_TABLE, 0xffff0000 | i); val = t3_read_reg(adapter, A_TP_RSS_MAP_TABLE); if (!(val & 0x80000000)) return -EAGAIN; *map++ = (u16)val; } return 0; } /** * t3_tp_set_offload_mode - put TP in NIC/offload mode * @adap: the adapter * @enable: 1 to select offload mode, 0 for regular NIC * * Switches TP to NIC/offload mode. */ void t3_tp_set_offload_mode(adapter_t *adap, int enable) { if (is_offload(adap) || !enable) t3_set_reg_field(adap, A_TP_IN_CONFIG, F_NICMODE, V_NICMODE(!enable)); } /** * pm_num_pages - calculate the number of pages of the payload memory * @mem_size: the size of the payload memory * @pg_size: the size of each payload memory page * * Calculate the number of pages, each of the given size, that fit in a * memory of the specified size, respecting the HW requirement that the * number of pages must be a multiple of 24. */ static inline unsigned int pm_num_pages(unsigned int mem_size, unsigned int pg_size) { unsigned int n = mem_size / pg_size; return n - n % 24; } #define mem_region(adap, start, size, reg) \ t3_write_reg((adap), A_ ## reg, (start)); \ start += size /* * partition_mem - partition memory and configure TP memory settings * @adap: the adapter * @p: the TP parameters * * Partitions context and payload memory and configures TP's memory * registers. */ static void partition_mem(adapter_t *adap, const struct tp_params *p) { unsigned int m, pstructs, tids = t3_mc5_size(&adap->mc5); unsigned int timers = 0, timers_shift = 22; if (adap->params.rev > 0) { if (tids <= 16 * 1024) { timers = 1; timers_shift = 16; } else if (tids <= 64 * 1024) { timers = 2; timers_shift = 18; } else if (tids <= 256 * 1024) { timers = 3; timers_shift = 20; } } t3_write_reg(adap, A_TP_PMM_SIZE, p->chan_rx_size | (p->chan_tx_size >> 16)); t3_write_reg(adap, A_TP_PMM_TX_BASE, 0); t3_write_reg(adap, A_TP_PMM_TX_PAGE_SIZE, p->tx_pg_size); t3_write_reg(adap, A_TP_PMM_TX_MAX_PAGE, p->tx_num_pgs); t3_set_reg_field(adap, A_TP_PARA_REG3, V_TXDATAACKIDX(M_TXDATAACKIDX), V_TXDATAACKIDX(fls(p->tx_pg_size) - 12)); t3_write_reg(adap, A_TP_PMM_RX_BASE, 0); t3_write_reg(adap, A_TP_PMM_RX_PAGE_SIZE, p->rx_pg_size); t3_write_reg(adap, A_TP_PMM_RX_MAX_PAGE, p->rx_num_pgs); pstructs = p->rx_num_pgs + p->tx_num_pgs; /* Add a bit of headroom and make multiple of 24 */ pstructs += 48; pstructs -= pstructs % 24; t3_write_reg(adap, A_TP_CMM_MM_MAX_PSTRUCT, pstructs); m = tids * TCB_SIZE; mem_region(adap, m, (64 << 10) * 64, SG_EGR_CNTX_BADDR); mem_region(adap, m, (64 << 10) * 64, SG_CQ_CONTEXT_BADDR); t3_write_reg(adap, A_TP_CMM_TIMER_BASE, V_CMTIMERMAXNUM(timers) | m); m += ((p->ntimer_qs - 1) << timers_shift) + (1 << 22); mem_region(adap, m, pstructs * 64, TP_CMM_MM_BASE); mem_region(adap, m, 64 * (pstructs / 24), TP_CMM_MM_PS_FLST_BASE); mem_region(adap, m, 64 * (p->rx_num_pgs / 24), TP_CMM_MM_RX_FLST_BASE); mem_region(adap, m, 64 * (p->tx_num_pgs / 24), TP_CMM_MM_TX_FLST_BASE); m = (m + 4095) & ~0xfff; t3_write_reg(adap, A_CIM_SDRAM_BASE_ADDR, m); t3_write_reg(adap, A_CIM_SDRAM_ADDR_SIZE, p->cm_size - m); tids = (p->cm_size - m - (3 << 20)) / 3072 - 32; m = t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers - adap->params.mc5.nfilters - adap->params.mc5.nroutes; if (tids < m) adap->params.mc5.nservers += m - tids; } static inline void tp_wr_indirect(adapter_t *adap, unsigned int addr, u32 val) { t3_write_reg(adap, A_TP_PIO_ADDR, addr); t3_write_reg(adap, A_TP_PIO_DATA, val); } static void tp_config(adapter_t *adap, const struct tp_params *p) { t3_write_reg(adap, A_TP_GLOBAL_CONFIG, F_TXPACINGENABLE | F_PATHMTU | F_IPCHECKSUMOFFLOAD | F_UDPCHECKSUMOFFLOAD | F_TCPCHECKSUMOFFLOAD | V_IPTTL(64)); t3_write_reg(adap, A_TP_TCP_OPTIONS, V_MTUDEFAULT(576) | F_MTUENABLE | V_WINDOWSCALEMODE(1) | V_TIMESTAMPSMODE(0) | V_SACKMODE(1) | V_SACKRX(1)); t3_write_reg(adap, A_TP_DACK_CONFIG, V_AUTOSTATE3(1) | V_AUTOSTATE2(1) | V_AUTOSTATE1(0) | V_BYTETHRESHOLD(16384) | V_MSSTHRESHOLD(2) | F_AUTOCAREFUL | F_AUTOENABLE | V_DACK_MODE(1)); t3_set_reg_field(adap, A_TP_IN_CONFIG, F_IPV6ENABLE | F_NICMODE, F_IPV6ENABLE | F_NICMODE); t3_write_reg(adap, A_TP_TX_RESOURCE_LIMIT, 0x18141814); t3_write_reg(adap, A_TP_PARA_REG4, 0x5050105); t3_set_reg_field(adap, A_TP_PARA_REG6, adap->params.rev > 0 ? F_ENABLEESND : F_T3A_ENABLEESND, 0); t3_set_reg_field(adap, A_TP_PC_CONFIG, F_ENABLEEPCMDAFULL | F_ENABLEOCSPIFULL, F_TXDEFERENABLE | F_HEARBEATDACK | F_TXCONGESTIONMODE | F_RXCONGESTIONMODE); t3_set_reg_field(adap, A_TP_PC_CONFIG2, F_CHDRAFULL, 0); if (adap->params.rev > 0) { tp_wr_indirect(adap, A_TP_EGRESS_CONFIG, F_REWRITEFORCETOSIZE); t3_set_reg_field(adap, A_TP_PARA_REG3, F_TXPACEAUTO, F_TXPACEAUTO); t3_set_reg_field(adap, A_TP_PC_CONFIG, F_LOCKTID, F_LOCKTID); t3_set_reg_field(adap, A_TP_PARA_REG3, 0, F_TXPACEAUTOSTRICT); } else t3_set_reg_field(adap, A_TP_PARA_REG3, 0, F_TXPACEFIXED); t3_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT1, 0); t3_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT0, 0); t3_write_reg(adap, A_TP_MOD_CHANNEL_WEIGHT, 0); t3_write_reg(adap, A_TP_MOD_RATE_LIMIT, 0); } -/* Desired TP timer resolution in usec */ -#define TP_TMR_RES 200 - /* TCP timer values in ms */ #define TP_DACK_TIMER 50 #define TP_RTO_MIN 250 /** * tp_set_timers - set TP timing parameters * @adap: the adapter to set * @core_clk: the core clock frequency in Hz * * Set TP's timing parameters, such as the various timer resolutions and * the TCP timer values. */ static void tp_set_timers(adapter_t *adap, unsigned int core_clk) { unsigned int tre = fls(core_clk / (1000000 / TP_TMR_RES)) - 1; unsigned int dack_re = adap->params.tp.dack_re; unsigned int tstamp_re = fls(core_clk / 1000); /* 1ms, at least */ unsigned int tps = core_clk >> tre; t3_write_reg(adap, A_TP_TIMER_RESOLUTION, V_TIMERRESOLUTION(tre) | V_DELAYEDACKRESOLUTION(dack_re) | V_TIMESTAMPRESOLUTION(tstamp_re)); t3_write_reg(adap, A_TP_DACK_TIMER, (core_clk >> dack_re) / (1000 / TP_DACK_TIMER)); t3_write_reg(adap, A_TP_TCP_BACKOFF_REG0, 0x3020100); t3_write_reg(adap, A_TP_TCP_BACKOFF_REG1, 0x7060504); t3_write_reg(adap, A_TP_TCP_BACKOFF_REG2, 0xb0a0908); t3_write_reg(adap, A_TP_TCP_BACKOFF_REG3, 0xf0e0d0c); t3_write_reg(adap, A_TP_SHIFT_CNT, V_SYNSHIFTMAX(6) | V_RXTSHIFTMAXR1(4) | V_RXTSHIFTMAXR2(15) | V_PERSHIFTBACKOFFMAX(8) | V_PERSHIFTMAX(8) | V_KEEPALIVEMAX(9)); #define SECONDS * tps t3_write_reg(adap, A_TP_MSL, adap->params.rev > 0 ? 0 : 2 SECONDS); t3_write_reg(adap, A_TP_RXT_MIN, tps / (1000 / TP_RTO_MIN)); t3_write_reg(adap, A_TP_RXT_MAX, 64 SECONDS); t3_write_reg(adap, A_TP_PERS_MIN, 5 SECONDS); t3_write_reg(adap, A_TP_PERS_MAX, 64 SECONDS); t3_write_reg(adap, A_TP_KEEP_IDLE, 7200 SECONDS); t3_write_reg(adap, A_TP_KEEP_INTVL, 75 SECONDS); t3_write_reg(adap, A_TP_INIT_SRTT, 3 SECONDS); t3_write_reg(adap, A_TP_FINWAIT2_TIMER, 600 SECONDS); #undef SECONDS } #ifdef CONFIG_CHELSIO_T3_CORE /** * t3_tp_set_coalescing_size - set receive coalescing size * @adap: the adapter * @size: the receive coalescing size * @psh: whether a set PSH bit should deliver coalesced data * * Set the receive coalescing size and PSH bit handling. */ int t3_tp_set_coalescing_size(adapter_t *adap, unsigned int size, int psh) { u32 val; if (size > MAX_RX_COALESCING_LEN) return -EINVAL; val = t3_read_reg(adap, A_TP_PARA_REG3); val &= ~(F_RXCOALESCEENABLE | F_RXCOALESCEPSHEN); if (size) { val |= F_RXCOALESCEENABLE; if (psh) val |= F_RXCOALESCEPSHEN; t3_write_reg(adap, A_TP_PARA_REG2, V_RXCOALESCESIZE(size) | V_MAXRXDATA(MAX_RX_COALESCING_LEN)); } t3_write_reg(adap, A_TP_PARA_REG3, val); return 0; } /** * t3_tp_set_max_rxsize - set the max receive size * @adap: the adapter * @size: the max receive size * * Set TP's max receive size. This is the limit that applies when * receive coalescing is disabled. */ void t3_tp_set_max_rxsize(adapter_t *adap, unsigned int size) { t3_write_reg(adap, A_TP_PARA_REG7, V_PMMAXXFERLEN0(size) | V_PMMAXXFERLEN1(size)); } static void __devinit init_mtus(unsigned short mtus[]) { /* * See draft-mathis-plpmtud-00.txt for the values. The min is 88 so * it can accomodate max size TCP/IP headers when SACK and timestamps * are enabled and still have at least 8 bytes of payload. */ mtus[0] = 88; mtus[1] = 88; /* workaround for silicon starting at 1 */ mtus[2] = 256; mtus[3] = 512; mtus[4] = 576; /* mtus[4] = 808; */ mtus[5] = 1024; mtus[6] = 1280; mtus[7] = 1492; mtus[8] = 1500; mtus[9] = 2002; mtus[10] = 2048; mtus[11] = 4096; mtus[12] = 4352; mtus[13] = 8192; mtus[14] = 9000; mtus[15] = 9600; } /* * Initial congestion control parameters. */ static void __devinit init_cong_ctrl(unsigned short *a, unsigned short *b) { a[0] = a[1] = a[2] = a[3] = a[4] = a[5] = a[6] = a[7] = a[8] = 1; a[9] = 2; a[10] = 3; a[11] = 4; a[12] = 5; a[13] = 6; a[14] = 7; a[15] = 8; a[16] = 9; a[17] = 10; a[18] = 14; a[19] = 17; a[20] = 21; a[21] = 25; a[22] = 30; a[23] = 35; a[24] = 45; a[25] = 60; a[26] = 80; a[27] = 100; a[28] = 200; a[29] = 300; a[30] = 400; a[31] = 500; b[0] = b[1] = b[2] = b[3] = b[4] = b[5] = b[6] = b[7] = b[8] = 0; b[9] = b[10] = 1; b[11] = b[12] = 2; b[13] = b[14] = b[15] = b[16] = 3; b[17] = b[18] = b[19] = b[20] = b[21] = 4; b[22] = b[23] = b[24] = b[25] = b[26] = b[27] = 5; b[28] = b[29] = 6; b[30] = b[31] = 7; } /* The minimum additive increment value for the congestion control table */ #define CC_MIN_INCR 2U /** * t3_load_mtus - write the MTU and congestion control HW tables * @adap: the adapter * @mtus: the unrestricted values for the MTU table * @alphs: the values for the congestion control alpha parameter * @beta: the values for the congestion control beta parameter * @mtu_cap: the maximum permitted effective MTU * * Write the MTU table with the supplied MTUs capping each at &mtu_cap. * Update the high-speed congestion control table with the supplied alpha, * beta, and MTUs. */ void t3_load_mtus(adapter_t *adap, unsigned short mtus[NMTUS], unsigned short alpha[NCCTRL_WIN], unsigned short beta[NCCTRL_WIN], unsigned short mtu_cap) { static const unsigned int avg_pkts[NCCTRL_WIN] = { 2, 6, 10, 14, 20, 28, 40, 56, 80, 112, 160, 224, 320, 448, 640, 896, 1281, 1792, 2560, 3584, 5120, 7168, 10240, 14336, 20480, 28672, 40960, 57344, 81920, 114688, 163840, 229376 }; unsigned int i, w; for (i = 0; i < NMTUS; ++i) { unsigned int mtu = min(mtus[i], mtu_cap); unsigned int log2 = fls(mtu); if (!(mtu & ((1 << log2) >> 2))) /* round */ log2--; t3_write_reg(adap, A_TP_MTU_TABLE, (i << 24) | (log2 << 16) | mtu); for (w = 0; w < NCCTRL_WIN; ++w) { unsigned int inc; inc = max(((mtu - 40) * alpha[w]) / avg_pkts[w], CC_MIN_INCR); t3_write_reg(adap, A_TP_CCTRL_TABLE, (i << 21) | (w << 16) | (beta[w] << 13) | inc); } } } /** * t3_read_hw_mtus - returns the values in the HW MTU table * @adap: the adapter * @mtus: where to store the HW MTU values * * Reads the HW MTU table. */ void t3_read_hw_mtus(adapter_t *adap, unsigned short mtus[NMTUS]) { int i; for (i = 0; i < NMTUS; ++i) { unsigned int val; t3_write_reg(adap, A_TP_MTU_TABLE, 0xff000000 | i); val = t3_read_reg(adap, A_TP_MTU_TABLE); mtus[i] = val & 0x3fff; } } /** * t3_get_cong_cntl_tab - reads the congestion control table * @adap: the adapter * @incr: where to store the alpha values * * Reads the additive increments programmed into the HW congestion * control table. */ void t3_get_cong_cntl_tab(adapter_t *adap, unsigned short incr[NMTUS][NCCTRL_WIN]) { unsigned int mtu, w; for (mtu = 0; mtu < NMTUS; ++mtu) for (w = 0; w < NCCTRL_WIN; ++w) { t3_write_reg(adap, A_TP_CCTRL_TABLE, 0xffff0000 | (mtu << 5) | w); incr[mtu][w] = (unsigned short)t3_read_reg(adap, A_TP_CCTRL_TABLE) & 0x1fff; } } /** * t3_tp_get_mib_stats - read TP's MIB counters * @adap: the adapter * @tps: holds the returned counter values * * Returns the values of TP's MIB counters. */ void t3_tp_get_mib_stats(adapter_t *adap, struct tp_mib_stats *tps) { t3_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_RDATA, (u32 *)tps, sizeof(*tps) / sizeof(u32), 0); } /** * t3_read_pace_tbl - read the pace table * @adap: the adapter * @pace_vals: holds the returned values * * Returns the values of TP's pace table in nanoseconds. */ void t3_read_pace_tbl(adapter_t *adap, unsigned int pace_vals[NTX_SCHED]) { unsigned int i, tick_ns = dack_ticks_to_usec(adap, 1000); for (i = 0; i < NTX_SCHED; i++) { t3_write_reg(adap, A_TP_PACE_TABLE, 0xffff0000 + i); pace_vals[i] = t3_read_reg(adap, A_TP_PACE_TABLE) * tick_ns; } } /** * t3_set_pace_tbl - set the pace table * @adap: the adapter * @pace_vals: the pace values in nanoseconds * @start: index of the first entry in the HW pace table to set * @n: how many entries to set * * Sets (a subset of the) HW pace table. */ void t3_set_pace_tbl(adapter_t *adap, unsigned int *pace_vals, unsigned int start, unsigned int n) { unsigned int tick_ns = dack_ticks_to_usec(adap, 1000); for ( ; n; n--, start++, pace_vals++) t3_write_reg(adap, A_TP_PACE_TABLE, (start << 16) | ((*pace_vals + tick_ns / 2) / tick_ns)); } #define ulp_region(adap, name, start, len) \ t3_write_reg((adap), A_ULPRX_ ## name ## _LLIMIT, (start)); \ t3_write_reg((adap), A_ULPRX_ ## name ## _ULIMIT, \ (start) + (len) - 1); \ start += len #define ulptx_region(adap, name, start, len) \ t3_write_reg((adap), A_ULPTX_ ## name ## _LLIMIT, (start)); \ t3_write_reg((adap), A_ULPTX_ ## name ## _ULIMIT, \ (start) + (len) - 1) static void ulp_config(adapter_t *adap, const struct tp_params *p) { unsigned int m = p->chan_rx_size; ulp_region(adap, ISCSI, m, p->chan_rx_size / 8); ulp_region(adap, TDDP, m, p->chan_rx_size / 8); ulptx_region(adap, TPT, m, p->chan_rx_size / 4); ulp_region(adap, STAG, m, p->chan_rx_size / 4); ulp_region(adap, RQ, m, p->chan_rx_size / 4); ulptx_region(adap, PBL, m, p->chan_rx_size / 4); ulp_region(adap, PBL, m, p->chan_rx_size / 4); t3_write_reg(adap, A_ULPRX_TDDP_TAGMASK, 0xffffffff); } #endif void t3_config_trace_filter(adapter_t *adapter, const struct trace_params *tp, int filter_index, int invert, int enable) { u32 addr, key[4], mask[4]; key[0] = tp->sport | (tp->sip << 16); key[1] = (tp->sip >> 16) | (tp->dport << 16); key[2] = tp->dip; key[3] = tp->proto | (tp->vlan << 8) | (tp->intf << 20); mask[0] = tp->sport_mask | (tp->sip_mask << 16); mask[1] = (tp->sip_mask >> 16) | (tp->dport_mask << 16); mask[2] = tp->dip_mask; mask[3] = tp->proto_mask | (tp->vlan_mask << 8) | (tp->intf_mask << 20); if (invert) key[3] |= (1 << 29); if (enable) key[3] |= (1 << 28); addr = filter_index ? A_TP_RX_TRC_KEY0 : A_TP_TX_TRC_KEY0; tp_wr_indirect(adapter, addr++, key[0]); tp_wr_indirect(adapter, addr++, mask[0]); tp_wr_indirect(adapter, addr++, key[1]); tp_wr_indirect(adapter, addr++, mask[1]); tp_wr_indirect(adapter, addr++, key[2]); tp_wr_indirect(adapter, addr++, mask[2]); tp_wr_indirect(adapter, addr++, key[3]); tp_wr_indirect(adapter, addr, mask[3]); (void) t3_read_reg(adapter, A_TP_PIO_DATA); } /** * t3_config_sched - configure a HW traffic scheduler * @adap: the adapter * @kbps: target rate in Kbps * @sched: the scheduler index * * Configure a Tx HW scheduler for the target rate. */ int t3_config_sched(adapter_t *adap, unsigned int kbps, int sched) { unsigned int v, tps, cpt, bpt, delta, mindelta = ~0; unsigned int clk = adap->params.vpd.cclk * 1000; unsigned int selected_cpt = 0, selected_bpt = 0; if (kbps > 0) { kbps *= 125; /* -> bytes */ for (cpt = 1; cpt <= 255; cpt++) { tps = clk / cpt; bpt = (kbps + tps / 2) / tps; if (bpt > 0 && bpt <= 255) { v = bpt * tps; delta = v >= kbps ? v - kbps : kbps - v; if (delta <= mindelta) { mindelta = delta; selected_cpt = cpt; selected_bpt = bpt; } } else if (selected_cpt) break; } if (!selected_cpt) return -EINVAL; } t3_write_reg(adap, A_TP_TM_PIO_ADDR, A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2); v = t3_read_reg(adap, A_TP_TM_PIO_DATA); if (sched & 1) v = (v & 0xffff) | (selected_cpt << 16) | (selected_bpt << 24); else v = (v & 0xffff0000) | selected_cpt | (selected_bpt << 8); t3_write_reg(adap, A_TP_TM_PIO_DATA, v); return 0; } /** * t3_set_sched_ipg - set the IPG for a Tx HW packet rate scheduler * @adap: the adapter * @sched: the scheduler index * @ipg: the interpacket delay in tenths of nanoseconds * * Set the interpacket delay for a HW packet rate scheduler. */ int t3_set_sched_ipg(adapter_t *adap, int sched, unsigned int ipg) { unsigned int v, addr = A_TP_TX_MOD_Q1_Q0_TIMER_SEPARATOR - sched / 2; /* convert ipg to nearest number of core clocks */ ipg *= core_ticks_per_usec(adap); ipg = (ipg + 5000) / 10000; if (ipg > 0xffff) return -EINVAL; t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr); v = t3_read_reg(adap, A_TP_TM_PIO_DATA); if (sched & 1) v = (v & 0xffff) | (ipg << 16); else v = (v & 0xffff0000) | ipg; t3_write_reg(adap, A_TP_TM_PIO_DATA, v); t3_read_reg(adap, A_TP_TM_PIO_DATA); return 0; } /** * t3_get_tx_sched - get the configuration of a Tx HW traffic scheduler * @adap: the adapter * @sched: the scheduler index * @kbps: the byte rate in Kbps * @ipg: the interpacket delay in tenths of nanoseconds * * Return the current configuration of a HW Tx scheduler. */ void t3_get_tx_sched(adapter_t *adap, unsigned int sched, unsigned int *kbps, unsigned int *ipg) { unsigned int v, addr, bpt, cpt; if (kbps) { addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2; t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr); v = t3_read_reg(adap, A_TP_TM_PIO_DATA); if (sched & 1) v >>= 16; bpt = (v >> 8) & 0xff; cpt = v & 0xff; if (!cpt) *kbps = 0; /* scheduler disabled */ else { v = (adap->params.vpd.cclk * 1000) / cpt; *kbps = (v * bpt) / 125; } } if (ipg) { addr = A_TP_TX_MOD_Q1_Q0_TIMER_SEPARATOR - sched / 2; t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr); v = t3_read_reg(adap, A_TP_TM_PIO_DATA); if (sched & 1) v >>= 16; v &= 0xffff; *ipg = (10000 * v) / core_ticks_per_usec(adap); } } static int tp_init(adapter_t *adap, const struct tp_params *p) { int busy = 0; tp_config(adap, p); t3_set_vlan_accel(adap, 3, 0); if (is_offload(adap)) { tp_set_timers(adap, adap->params.vpd.cclk * 1000); t3_write_reg(adap, A_TP_RESET, F_FLSTINITENABLE); busy = t3_wait_op_done(adap, A_TP_RESET, F_FLSTINITENABLE, 0, 1000, 5); if (busy) CH_ERR(adap, "TP initialization timed out\n"); } if (!busy) t3_write_reg(adap, A_TP_RESET, F_TPRESET); return busy; } int t3_mps_set_active_ports(adapter_t *adap, unsigned int port_mask) { if (port_mask & ~((1 << adap->params.nports) - 1)) return -EINVAL; t3_set_reg_field(adap, A_MPS_CFG, F_PORT1ACTIVE | F_PORT0ACTIVE, port_mask << S_PORT0ACTIVE); return 0; } /* * Perform the bits of HW initialization that are dependent on the number * of available ports. */ static void init_hw_for_avail_ports(adapter_t *adap, int nports) { int i; if (nports == 1) { t3_set_reg_field(adap, A_ULPRX_CTL, F_ROUND_ROBIN, 0); t3_set_reg_field(adap, A_ULPTX_CONFIG, F_CFG_RR_ARB, 0); t3_write_reg(adap, A_MPS_CFG, F_TPRXPORTEN | F_TPTXPORT0EN | F_PORT0ACTIVE | F_ENFORCEPKT); t3_write_reg(adap, A_PM1_TX_CFG, 0xc000c000); } else { t3_set_reg_field(adap, A_ULPRX_CTL, 0, F_ROUND_ROBIN); t3_set_reg_field(adap, A_ULPTX_CONFIG, 0, F_CFG_RR_ARB); t3_write_reg(adap, A_ULPTX_DMA_WEIGHT, V_D1_WEIGHT(16) | V_D0_WEIGHT(16)); t3_write_reg(adap, A_MPS_CFG, F_TPTXPORT0EN | F_TPTXPORT1EN | F_TPRXPORTEN | F_PORT0ACTIVE | F_PORT1ACTIVE | F_ENFORCEPKT); t3_write_reg(adap, A_PM1_TX_CFG, 0x80008000); t3_set_reg_field(adap, A_TP_PC_CONFIG, 0, F_TXTOSQUEUEMAPMODE); t3_write_reg(adap, A_TP_TX_MOD_QUEUE_REQ_MAP, V_TX_MOD_QUEUE_REQ_MAP(0xaa)); for (i = 0; i < 16; i++) t3_write_reg(adap, A_TP_TX_MOD_QUE_TABLE, (i << 16) | 0x1010); } } static int calibrate_xgm(adapter_t *adapter) { if (uses_xaui(adapter)) { unsigned int v, i; for (i = 0; i < 5; ++i) { t3_write_reg(adapter, A_XGM_XAUI_IMP, 0); (void) t3_read_reg(adapter, A_XGM_XAUI_IMP); t3_os_sleep(1); v = t3_read_reg(adapter, A_XGM_XAUI_IMP); if (!(v & (F_XGM_CALFAULT | F_CALBUSY))) { t3_write_reg(adapter, A_XGM_XAUI_IMP, V_XAUIIMP(G_CALIMP(v) >> 2)); return 0; } } CH_ERR(adapter, "MAC calibration failed\n"); return -1; } else { t3_write_reg(adapter, A_XGM_RGMII_IMP, V_RGMIIIMPPD(2) | V_RGMIIIMPPU(3)); t3_set_reg_field(adapter, A_XGM_RGMII_IMP, F_XGM_IMPSETUPDATE, F_XGM_IMPSETUPDATE); } return 0; } static void calibrate_xgm_t3b(adapter_t *adapter) { if (!uses_xaui(adapter)) { t3_write_reg(adapter, A_XGM_RGMII_IMP, F_CALRESET | F_CALUPDATE | V_RGMIIIMPPD(2) | V_RGMIIIMPPU(3)); t3_set_reg_field(adapter, A_XGM_RGMII_IMP, F_CALRESET, 0); t3_set_reg_field(adapter, A_XGM_RGMII_IMP, 0, F_XGM_IMPSETUPDATE); t3_set_reg_field(adapter, A_XGM_RGMII_IMP, F_XGM_IMPSETUPDATE, 0); t3_set_reg_field(adapter, A_XGM_RGMII_IMP, F_CALUPDATE, 0); t3_set_reg_field(adapter, A_XGM_RGMII_IMP, 0, F_CALUPDATE); } } struct mc7_timing_params { unsigned char ActToPreDly; unsigned char ActToRdWrDly; unsigned char PreCyc; unsigned char RefCyc[5]; unsigned char BkCyc; unsigned char WrToRdDly; unsigned char RdToWrDly; }; /* * Write a value to a register and check that the write completed. These * writes normally complete in a cycle or two, so one read should suffice. * The very first read exists to flush the posted write to the device. */ static int wrreg_wait(adapter_t *adapter, unsigned int addr, u32 val) { t3_write_reg(adapter, addr, val); (void) t3_read_reg(adapter, addr); /* flush */ if (!(t3_read_reg(adapter, addr) & F_BUSY)) return 0; CH_ERR(adapter, "write to MC7 register 0x%x timed out\n", addr); return -EIO; } static int mc7_init(struct mc7 *mc7, unsigned int mc7_clock, int mem_type) { static const unsigned int mc7_mode[] = { 0x632, 0x642, 0x652, 0x432, 0x442 }; static const struct mc7_timing_params mc7_timings[] = { { 12, 3, 4, { 20, 28, 34, 52, 0 }, 15, 6, 4 }, { 12, 4, 5, { 20, 28, 34, 52, 0 }, 16, 7, 4 }, { 12, 5, 6, { 20, 28, 34, 52, 0 }, 17, 8, 4 }, { 9, 3, 4, { 15, 21, 26, 39, 0 }, 12, 6, 4 }, { 9, 4, 5, { 15, 21, 26, 39, 0 }, 13, 7, 4 } }; u32 val; unsigned int width, density, slow, attempts; adapter_t *adapter = mc7->adapter; const struct mc7_timing_params *p = &mc7_timings[mem_type]; + if (mc7->size == 0) + return 0; + val = t3_read_reg(adapter, mc7->offset + A_MC7_CFG); slow = val & F_SLOW; width = G_WIDTH(val); density = G_DEN(val); t3_write_reg(adapter, mc7->offset + A_MC7_CFG, val | F_IFEN); val = t3_read_reg(adapter, mc7->offset + A_MC7_CFG); /* flush */ t3_os_sleep(1); if (!slow) { t3_write_reg(adapter, mc7->offset + A_MC7_CAL, F_SGL_CAL_EN); (void) t3_read_reg(adapter, mc7->offset + A_MC7_CAL); t3_os_sleep(1); if (t3_read_reg(adapter, mc7->offset + A_MC7_CAL) & (F_BUSY | F_SGL_CAL_EN | F_CAL_FAULT)) { CH_ERR(adapter, "%s MC7 calibration timed out\n", mc7->name); goto out_fail; } } t3_write_reg(adapter, mc7->offset + A_MC7_PARM, V_ACTTOPREDLY(p->ActToPreDly) | V_ACTTORDWRDLY(p->ActToRdWrDly) | V_PRECYC(p->PreCyc) | V_REFCYC(p->RefCyc[density]) | V_BKCYC(p->BkCyc) | V_WRTORDDLY(p->WrToRdDly) | V_RDTOWRDLY(p->RdToWrDly)); t3_write_reg(adapter, mc7->offset + A_MC7_CFG, val | F_CLKEN | F_TERM150); (void) t3_read_reg(adapter, mc7->offset + A_MC7_CFG); /* flush */ if (!slow) t3_set_reg_field(adapter, mc7->offset + A_MC7_DLL, F_DLLENB, F_DLLENB); udelay(1); val = slow ? 3 : 6; if (wrreg_wait(adapter, mc7->offset + A_MC7_PRE, 0) || wrreg_wait(adapter, mc7->offset + A_MC7_EXT_MODE2, 0) || wrreg_wait(adapter, mc7->offset + A_MC7_EXT_MODE3, 0) || wrreg_wait(adapter, mc7->offset + A_MC7_EXT_MODE1, val)) goto out_fail; if (!slow) { t3_write_reg(adapter, mc7->offset + A_MC7_MODE, 0x100); t3_set_reg_field(adapter, mc7->offset + A_MC7_DLL, F_DLLRST, 0); udelay(5); } if (wrreg_wait(adapter, mc7->offset + A_MC7_PRE, 0) || wrreg_wait(adapter, mc7->offset + A_MC7_REF, 0) || wrreg_wait(adapter, mc7->offset + A_MC7_REF, 0) || wrreg_wait(adapter, mc7->offset + A_MC7_MODE, mc7_mode[mem_type]) || wrreg_wait(adapter, mc7->offset + A_MC7_EXT_MODE1, val | 0x380) || wrreg_wait(adapter, mc7->offset + A_MC7_EXT_MODE1, val)) goto out_fail; /* clock value is in KHz */ mc7_clock = mc7_clock * 7812 + mc7_clock / 2; /* ns */ mc7_clock /= 1000000; /* KHz->MHz, ns->us */ t3_write_reg(adapter, mc7->offset + A_MC7_REF, F_PERREFEN | V_PREREFDIV(mc7_clock)); (void) t3_read_reg(adapter, mc7->offset + A_MC7_REF); /* flush */ t3_write_reg(adapter, mc7->offset + A_MC7_ECC, F_ECCGENEN | F_ECCCHKEN); t3_write_reg(adapter, mc7->offset + A_MC7_BIST_DATA, 0); t3_write_reg(adapter, mc7->offset + A_MC7_BIST_ADDR_BEG, 0); t3_write_reg(adapter, mc7->offset + A_MC7_BIST_ADDR_END, (mc7->size << width) - 1); t3_write_reg(adapter, mc7->offset + A_MC7_BIST_OP, V_OP(1)); (void) t3_read_reg(adapter, mc7->offset + A_MC7_BIST_OP); /* flush */ attempts = 50; do { t3_os_sleep(250); val = t3_read_reg(adapter, mc7->offset + A_MC7_BIST_OP); } while ((val & F_BUSY) && --attempts); if (val & F_BUSY) { CH_ERR(adapter, "%s MC7 BIST timed out\n", mc7->name); goto out_fail; } /* Enable normal memory accesses. */ t3_set_reg_field(adapter, mc7->offset + A_MC7_CFG, 0, F_RDY); return 0; out_fail: return -1; } static void config_pcie(adapter_t *adap) { static const u16 ack_lat[4][6] = { { 237, 416, 559, 1071, 2095, 4143 }, { 128, 217, 289, 545, 1057, 2081 }, { 73, 118, 154, 282, 538, 1050 }, { 67, 107, 86, 150, 278, 534 } }; static const u16 rpl_tmr[4][6] = { { 711, 1248, 1677, 3213, 6285, 12429 }, { 384, 651, 867, 1635, 3171, 6243 }, { 219, 354, 462, 846, 1614, 3150 }, { 201, 321, 258, 450, 834, 1602 } }; u16 val; unsigned int log2_width, pldsize; unsigned int fst_trn_rx, fst_trn_tx, acklat, rpllmt; t3_os_pci_read_config_2(adap, adap->params.pci.pcie_cap_addr + PCI_EXP_DEVCTL, &val); pldsize = (val & PCI_EXP_DEVCTL_PAYLOAD) >> 5; t3_os_pci_read_config_2(adap, adap->params.pci.pcie_cap_addr + PCI_EXP_LNKCTL, &val); fst_trn_tx = G_NUMFSTTRNSEQ(t3_read_reg(adap, A_PCIE_PEX_CTRL0)); fst_trn_rx = adap->params.rev == 0 ? fst_trn_tx : G_NUMFSTTRNSEQRX(t3_read_reg(adap, A_PCIE_MODE)); log2_width = fls(adap->params.pci.width) - 1; acklat = ack_lat[log2_width][pldsize]; if (val & 1) /* check LOsEnable */ acklat += fst_trn_tx * 4; rpllmt = rpl_tmr[log2_width][pldsize] + fst_trn_rx * 4; if (adap->params.rev == 0) t3_set_reg_field(adap, A_PCIE_PEX_CTRL1, V_T3A_ACKLAT(M_T3A_ACKLAT), V_T3A_ACKLAT(acklat)); else t3_set_reg_field(adap, A_PCIE_PEX_CTRL1, V_ACKLAT(M_ACKLAT), V_ACKLAT(acklat)); t3_set_reg_field(adap, A_PCIE_PEX_CTRL0, V_REPLAYLMT(M_REPLAYLMT), V_REPLAYLMT(rpllmt)); t3_write_reg(adap, A_PCIE_PEX_ERR, 0xffffffff); t3_set_reg_field(adap, A_PCIE_CFG, F_PCIE_CLIDECEN, F_PCIE_CLIDECEN); } /* * Initialize and configure T3 HW modules. This performs the * initialization steps that need to be done once after a card is reset. * MAC and PHY initialization is handled separarely whenever a port is enabled. * * fw_params are passed to FW and their value is platform dependent. Only the * top 8 bits are available for use, the rest must be 0. */ int t3_init_hw(adapter_t *adapter, u32 fw_params) { int err = -EIO, attempts = 100; const struct vpd_params *vpd = &adapter->params.vpd; if (adapter->params.rev > 0) calibrate_xgm_t3b(adapter); else if (calibrate_xgm(adapter)) goto out_err; if (vpd->mclk) { partition_mem(adapter, &adapter->params.tp); if (mc7_init(&adapter->pmrx, vpd->mclk, vpd->mem_timing) || mc7_init(&adapter->pmtx, vpd->mclk, vpd->mem_timing) || mc7_init(&adapter->cm, vpd->mclk, vpd->mem_timing) || t3_mc5_init(&adapter->mc5, adapter->params.mc5.nservers, adapter->params.mc5.nfilters, adapter->params.mc5.nroutes)) goto out_err; } if (tp_init(adapter, &adapter->params.tp)) goto out_err; #ifdef CONFIG_CHELSIO_T3_CORE t3_tp_set_coalescing_size(adapter, min(adapter->params.sge.max_pkt_size, MAX_RX_COALESCING_LEN), 1); t3_tp_set_max_rxsize(adapter, min(adapter->params.sge.max_pkt_size, 16384U)); ulp_config(adapter, &adapter->params.tp); #endif if (is_pcie(adapter)) config_pcie(adapter); else t3_set_reg_field(adapter, A_PCIX_CFG, 0, F_CLIDECEN); t3_write_reg(adapter, A_PM1_RX_CFG, 0xf000f000); init_hw_for_avail_ports(adapter, adapter->params.nports); t3_sge_init(adapter, &adapter->params.sge); t3_write_reg(adapter, A_CIM_HOST_ACC_DATA, vpd->uclk | fw_params); t3_write_reg(adapter, A_CIM_BOOT_CFG, V_BOOTADDR(FW_FLASH_BOOT_ADDR >> 2)); (void) t3_read_reg(adapter, A_CIM_BOOT_CFG); /* flush */ do { /* wait for uP to initialize */ t3_os_sleep(20); } while (t3_read_reg(adapter, A_CIM_HOST_ACC_DATA) && --attempts); - if (!attempts) + if (!attempts) { + CH_ERR(adapter, "uP initialization timed out\n"); goto out_err; - + } err = 0; out_err: return err; } /** * get_pci_mode - determine a card's PCI mode * @adapter: the adapter * @p: where to store the PCI settings * * Determines a card's PCI mode and associated parameters, such as speed * and width. */ static void __devinit get_pci_mode(adapter_t *adapter, struct pci_params *p) { static unsigned short speed_map[] = { 33, 66, 100, 133 }; u32 pci_mode, pcie_cap; pcie_cap = t3_os_find_pci_capability(adapter, PCI_CAP_ID_EXP); if (pcie_cap) { u16 val; p->variant = PCI_VARIANT_PCIE; p->pcie_cap_addr = pcie_cap; t3_os_pci_read_config_2(adapter, pcie_cap + PCI_EXP_LNKSTA, &val); p->width = (val >> 4) & 0x3f; return; } pci_mode = t3_read_reg(adapter, A_PCIX_MODE); p->speed = speed_map[G_PCLKRANGE(pci_mode)]; p->width = (pci_mode & F_64BIT) ? 64 : 32; pci_mode = G_PCIXINITPAT(pci_mode); if (pci_mode == 0) p->variant = PCI_VARIANT_PCI; else if (pci_mode < 4) p->variant = PCI_VARIANT_PCIX_MODE1_PARITY; else if (pci_mode < 8) p->variant = PCI_VARIANT_PCIX_MODE1_ECC; else p->variant = PCI_VARIANT_PCIX_266_MODE2; } /** * init_link_config - initialize a link's SW state * @lc: structure holding the link state * @ai: information about the current card * * Initializes the SW state maintained for each link, including the link's * capabilities and default speed/duplex/flow-control/autonegotiation * settings. */ static void __devinit init_link_config(struct link_config *lc, unsigned int caps) { lc->supported = caps; lc->requested_speed = lc->speed = SPEED_INVALID; lc->requested_duplex = lc->duplex = DUPLEX_INVALID; lc->requested_fc = lc->fc = PAUSE_RX | PAUSE_TX; if (lc->supported & SUPPORTED_Autoneg) { lc->advertising = lc->supported; lc->autoneg = AUTONEG_ENABLE; lc->requested_fc |= PAUSE_AUTONEG; } else { lc->advertising = 0; lc->autoneg = AUTONEG_DISABLE; } } /** * mc7_calc_size - calculate MC7 memory size * @cfg: the MC7 configuration * * Calculates the size of an MC7 memory in bytes from the value of its * configuration register. */ static unsigned int __devinit mc7_calc_size(u32 cfg) { unsigned int width = G_WIDTH(cfg); unsigned int banks = !!(cfg & F_BKS) + 1; unsigned int org = !!(cfg & F_ORG) + 1; unsigned int density = G_DEN(cfg); unsigned int MBs = ((256 << density) * banks) / (org << width); return MBs << 20; } static void __devinit mc7_prep(adapter_t *adapter, struct mc7 *mc7, unsigned int base_addr, const char *name) { u32 cfg; mc7->adapter = adapter; mc7->name = name; mc7->offset = base_addr - MC7_PMRX_BASE_ADDR; cfg = t3_read_reg(adapter, mc7->offset + A_MC7_CFG); - mc7->size = mc7_calc_size(cfg); + mc7->size = G_DEN(cfg) == M_DEN ? 0 : mc7_calc_size(cfg); mc7->width = G_WIDTH(cfg); } void mac_prep(struct cmac *mac, adapter_t *adapter, int index) { mac->adapter = adapter; mac->offset = (XGMAC0_1_BASE_ADDR - XGMAC0_0_BASE_ADDR) * index; mac->nucast = 1; if (adapter->params.rev == 0 && uses_xaui(adapter)) { t3_write_reg(adapter, A_XGM_SERDES_CTRL + mac->offset, is_10G(adapter) ? 0x2901c04 : 0x2301c04); t3_set_reg_field(adapter, A_XGM_PORT_CFG + mac->offset, F_ENRGMII, 0); } } void early_hw_init(adapter_t *adapter, const struct adapter_info *ai) { u32 val = V_PORTSPEED(is_10G(adapter) ? 3 : 2); mi1_init(adapter, ai); t3_write_reg(adapter, A_I2C_CFG, /* set for 80KHz */ V_I2C_CLKDIV(adapter->params.vpd.cclk / 80 - 1)); t3_write_reg(adapter, A_T3DBG_GPIO_EN, ai->gpio_out | F_GPIO0_OEN | F_GPIO0_OUT_VAL); - + t3_write_reg(adapter, A_MC5_DB_SERVER_INDEX, 0); + if (adapter->params.rev == 0 || !uses_xaui(adapter)) val |= F_ENRGMII; /* Enable MAC clocks so we can access the registers */ t3_write_reg(adapter, A_XGM_PORT_CFG, val); (void) t3_read_reg(adapter, A_XGM_PORT_CFG); val |= F_CLKDIVRESET_; t3_write_reg(adapter, A_XGM_PORT_CFG, val); (void) t3_read_reg(adapter, A_XGM_PORT_CFG); t3_write_reg(adapter, XGM_REG(A_XGM_PORT_CFG, 1), val); (void) t3_read_reg(adapter, A_XGM_PORT_CFG); } /* * Reset the adapter. PCIe cards lose their config space during reset, PCI-X * ones don't. */ int t3_reset_adapter(adapter_t *adapter) { int i, save_and_restore_pcie = adapter->params.rev < T3_REV_B2 && is_pcie(adapter); uint16_t devid = 0; if (save_and_restore_pcie) t3_os_pci_save_state(adapter); t3_write_reg(adapter, A_PL_RST, F_CRSTWRM | F_CRSTWRMMODE); /* * Delay. Give Some time to device to reset fully. * XXX The delay time should be modified. */ for (i = 0; i < 10; i++) { t3_os_sleep(50); t3_os_pci_read_config_2(adapter, 0x00, &devid); if (devid == 0x1425) break; } if (devid != 0x1425) return -1; if (save_and_restore_pcie) t3_os_pci_restore_state(adapter); return 0; } /* * Initialize adapter SW state for the various HW modules, set initial values * for some adapter tunables, take PHYs out of reset, and initialize the MDIO * interface. */ int __devinit t3_prep_adapter(adapter_t *adapter, const struct adapter_info *ai, int reset) { int ret; unsigned int i, j = 0; get_pci_mode(adapter, &adapter->params.pci); adapter->params.info = ai; adapter->params.nports = ai->nports; adapter->params.rev = t3_read_reg(adapter, A_PL_REV); adapter->params.linkpoll_period = 0; adapter->params.stats_update_period = is_10G(adapter) ? MAC_STATS_ACCUM_SECS : (MAC_STATS_ACCUM_SECS * 10); adapter->params.pci.vpd_cap_addr = t3_os_find_pci_capability(adapter, PCI_CAP_ID_VPD); ret = get_vpd_params(adapter, &adapter->params.vpd); if (ret < 0) return ret; if (reset && t3_reset_adapter(adapter)) return -1; t3_sge_prep(adapter, &adapter->params.sge); if (adapter->params.vpd.mclk) { struct tp_params *p = &adapter->params.tp; mc7_prep(adapter, &adapter->pmrx, MC7_PMRX_BASE_ADDR, "PMRX"); mc7_prep(adapter, &adapter->pmtx, MC7_PMTX_BASE_ADDR, "PMTX"); mc7_prep(adapter, &adapter->cm, MC7_CM_BASE_ADDR, "CM"); p->nchan = ai->nports; p->pmrx_size = t3_mc7_size(&adapter->pmrx); p->pmtx_size = t3_mc7_size(&adapter->pmtx); p->cm_size = t3_mc7_size(&adapter->cm); p->chan_rx_size = p->pmrx_size / 2; /* only 1 Rx channel */ p->chan_tx_size = p->pmtx_size / p->nchan; p->rx_pg_size = 64 * 1024; p->tx_pg_size = is_10G(adapter) ? 64 * 1024 : 16 * 1024; p->rx_num_pgs = pm_num_pages(p->chan_rx_size, p->rx_pg_size); p->tx_num_pgs = pm_num_pages(p->chan_tx_size, p->tx_pg_size); p->ntimer_qs = p->cm_size >= (128 << 20) || adapter->params.rev > 0 ? 12 : 6; p->dack_re = fls(adapter->params.vpd.cclk / 10) - 1; /* 100us */ + } + adapter->params.offload = t3_mc7_size(&adapter->pmrx) && + t3_mc7_size(&adapter->pmtx) && + t3_mc7_size(&adapter->cm); + if (is_offload(adapter)) { adapter->params.mc5.nservers = DEFAULT_NSERVERS; adapter->params.mc5.nfilters = adapter->params.rev > 0 ? DEFAULT_NFILTERS : 0; adapter->params.mc5.nroutes = 0; t3_mc5_prep(adapter, &adapter->mc5, MC5_MODE_144_BIT); #ifdef CONFIG_CHELSIO_T3_CORE init_mtus(adapter->params.mtus); init_cong_ctrl(adapter->params.a_wnd, adapter->params.b_wnd); #endif } early_hw_init(adapter, ai); for_each_port(adapter, i) { u8 hw_addr[6]; struct port_info *p = &adapter->port[i]; while (!adapter->params.vpd.port_type[j]) ++j; p->port_type = &port_types[adapter->params.vpd.port_type[j]]; p->port_type->phy_prep(&p->phy, adapter, ai->phy_base_addr + j, ai->mdio_ops); mac_prep(&p->mac, adapter, j); ++j; /* * The VPD EEPROM stores the base Ethernet address for the * card. A port's address is derived from the base by adding * the port's index to the base's low octet. */ memcpy(hw_addr, adapter->params.vpd.eth_base, 5); hw_addr[5] = adapter->params.vpd.eth_base[5] + i; t3_os_set_hw_addr(adapter, i, hw_addr); init_link_config(&p->link_config, p->port_type->caps); p->phy.ops->power_down(&p->phy, 1); if (!(p->port_type->caps & SUPPORTED_IRQ)) adapter->params.linkpoll_period = 10; } return 0; } void t3_led_ready(adapter_t *adapter) { t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL, F_GPIO0_OUT_VAL); } void t3_port_failover(adapter_t *adapter, int port) { u32 val; val = port ? F_PORT1ACTIVE : F_PORT0ACTIVE; t3_set_reg_field(adapter, A_MPS_CFG, F_PORT0ACTIVE | F_PORT1ACTIVE, val); } void t3_failover_done(adapter_t *adapter, int port) { t3_set_reg_field(adapter, A_MPS_CFG, F_PORT0ACTIVE | F_PORT1ACTIVE, F_PORT0ACTIVE | F_PORT1ACTIVE); } void t3_failover_clear(adapter_t *adapter) { t3_set_reg_field(adapter, A_MPS_CFG, F_PORT0ACTIVE | F_PORT1ACTIVE, F_PORT0ACTIVE | F_PORT1ACTIVE); } Index: head/sys/dev/cxgb/common/cxgb_version.h =================================================================== --- head/sys/dev/cxgb/common/cxgb_version.h (revision 169977) +++ head/sys/dev/cxgb/common/cxgb_version.h (revision 169978) @@ -1,45 +1,45 @@ /************************************************************************** Copyright (c) 2007, Chelsio Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the Chelsio Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. $FreeBSD$ ***************************************************************************/ /* * Note that although this driver doesn't contain all of the functionality of the Linux driver * the common code is 99% the same. Hence we keep the same version number to indicate what linux * driver the common code corresponds to. */ #ifndef __CHELSIO_VERSION_H #define __CHELSIO_VERSION_H #define DRV_DESC "Chelsio T3 Network Driver" #define DRV_NAME "cxgb" -#define DRV_VERSION "1.0.071" -#endif +#define DRV_VERSION "1.0.086" +#endif Index: head/sys/dev/cxgb/common/cxgb_xgmac.c =================================================================== --- head/sys/dev/cxgb/common/cxgb_xgmac.c (revision 169977) +++ head/sys/dev/cxgb/common/cxgb_xgmac.c (revision 169978) @@ -1,519 +1,589 @@ /************************************************************************** Copyright (c) 2007, Chelsio Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the Chelsio Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ***************************************************************************/ #include __FBSDID("$FreeBSD$"); #include #include /* * # of exact address filters. The first one is used for the station address, * the rest are available for multicast addresses. */ #define EXACT_ADDR_FILTERS 8 static inline int macidx(const struct cmac *mac) { return mac->offset / (XGMAC0_1_BASE_ADDR - XGMAC0_0_BASE_ADDR); } static void xaui_serdes_reset(struct cmac *mac) { static const unsigned int clear[] = { F_PWRDN0 | F_PWRDN1, F_RESETPLL01, F_RESET0 | F_RESET1, F_PWRDN2 | F_PWRDN3, F_RESETPLL23, F_RESET2 | F_RESET3 }; int i; adapter_t *adap = mac->adapter; u32 ctrl = A_XGM_SERDES_CTRL0 + mac->offset; t3_write_reg(adap, ctrl, adap->params.vpd.xauicfg[macidx(mac)] | F_RESET3 | F_RESET2 | F_RESET1 | F_RESET0 | F_PWRDN3 | F_PWRDN2 | F_PWRDN1 | F_PWRDN0 | F_RESETPLL23 | F_RESETPLL01); (void)t3_read_reg(adap, ctrl); udelay(15); for (i = 0; i < ARRAY_SIZE(clear); i++) { t3_set_reg_field(adap, ctrl, clear[i], 0); udelay(15); } } void t3b_pcs_reset(struct cmac *mac) { t3_set_reg_field(mac->adapter, A_XGM_RESET_CTRL + mac->offset, F_PCS_RESET_, 0); udelay(20); t3_set_reg_field(mac->adapter, A_XGM_RESET_CTRL + mac->offset, 0, F_PCS_RESET_); } int t3_mac_reset(struct cmac *mac) { static struct addr_val_pair mac_reset_avp[] = { { A_XGM_TX_CTRL, 0 }, { A_XGM_RX_CTRL, 0 }, { A_XGM_RX_CFG, F_DISPAUSEFRAMES | F_EN1536BFRAMES | F_RMFCS | F_ENJUMBO | F_ENHASHMCAST }, { A_XGM_RX_HASH_LOW, 0 }, { A_XGM_RX_HASH_HIGH, 0 }, { A_XGM_RX_EXACT_MATCH_LOW_1, 0 }, { A_XGM_RX_EXACT_MATCH_LOW_2, 0 }, { A_XGM_RX_EXACT_MATCH_LOW_3, 0 }, { A_XGM_RX_EXACT_MATCH_LOW_4, 0 }, { A_XGM_RX_EXACT_MATCH_LOW_5, 0 }, { A_XGM_RX_EXACT_MATCH_LOW_6, 0 }, { A_XGM_RX_EXACT_MATCH_LOW_7, 0 }, { A_XGM_RX_EXACT_MATCH_LOW_8, 0 }, { A_XGM_STAT_CTRL, F_CLRSTATS } }; u32 val; adapter_t *adap = mac->adapter; unsigned int oft = mac->offset; t3_write_reg(adap, A_XGM_RESET_CTRL + oft, F_MAC_RESET_); (void) t3_read_reg(adap, A_XGM_RESET_CTRL + oft); /* flush */ t3_write_regs(adap, mac_reset_avp, ARRAY_SIZE(mac_reset_avp), oft); t3_set_reg_field(adap, A_XGM_RXFIFO_CFG + oft, F_RXSTRFRWRD | F_DISERRFRAMES, uses_xaui(adap) ? 0 : F_RXSTRFRWRD); if (uses_xaui(adap)) { if (adap->params.rev == 0) { t3_set_reg_field(adap, A_XGM_SERDES_CTRL + oft, 0, F_RXENABLE | F_TXENABLE); if (t3_wait_op_done(adap, A_XGM_SERDES_STATUS1 + oft, F_CMULOCK, 1, 5, 2)) { CH_ERR(adap, "MAC %d XAUI SERDES CMU lock failed\n", macidx(mac)); return -1; } t3_set_reg_field(adap, A_XGM_SERDES_CTRL + oft, 0, F_SERDESRESET_); } else xaui_serdes_reset(mac); } val = F_MAC_RESET_; if (is_10G(adap)) val |= F_PCS_RESET_; else if (uses_xaui(adap)) val |= F_PCS_RESET_ | F_XG2G_RESET_; else val |= F_RGMII_RESET_ | F_XG2G_RESET_; t3_write_reg(adap, A_XGM_RESET_CTRL + oft, val); (void) t3_read_reg(adap, A_XGM_RESET_CTRL + oft); /* flush */ if ((val & F_PCS_RESET_) && adap->params.rev) { t3_os_sleep(1); t3b_pcs_reset(mac); } memset(&mac->stats, 0, sizeof(mac->stats)); return 0; } static int t3b2_mac_reset(struct cmac *mac) { u32 val; adapter_t *adap = mac->adapter; unsigned int oft = mac->offset; /* Stop egress traffic to xgm*/ if (!macidx(mac)) t3_set_reg_field(adap, A_MPS_CFG, F_PORT0ACTIVE, 0); else t3_set_reg_field(adap, A_MPS_CFG, F_PORT1ACTIVE, 0); /* PCS in reset */ t3_write_reg(adap, A_XGM_RESET_CTRL + oft, F_MAC_RESET_); (void) t3_read_reg(adap, A_XGM_RESET_CTRL + oft); /* flush */ t3_os_sleep(10); /* Check for xgm Rx fifo empty */ if (t3_wait_op_done(adap, A_XGM_RX_MAX_PKT_SIZE_ERR_CNT + oft, 0x80000000, 1, 5, 2)) { CH_ERR(adap, "MAC %d Rx fifo drain failed\n", macidx(mac)); return -1; } t3_write_reg(adap, A_XGM_RESET_CTRL + oft, 0); /*MAC in reset*/ (void) t3_read_reg(adap, A_XGM_RESET_CTRL + oft); /* flush */ val = F_MAC_RESET_; if (is_10G(adap)) val |= F_PCS_RESET_; else if (uses_xaui(adap)) val |= F_PCS_RESET_ | F_XG2G_RESET_; else val |= F_RGMII_RESET_ | F_XG2G_RESET_; t3_write_reg(adap, A_XGM_RESET_CTRL + oft, val); (void) t3_read_reg(adap, A_XGM_RESET_CTRL + oft); /* flush */ if ((val & F_PCS_RESET_) && adap->params.rev) { t3_os_sleep(1); t3b_pcs_reset(mac); } t3_write_reg(adap, A_XGM_RX_CFG + oft, F_DISPAUSEFRAMES | F_EN1536BFRAMES | F_RMFCS | F_ENJUMBO | F_ENHASHMCAST ); /*Resume egress traffic to xgm*/ if (!macidx(mac)) t3_set_reg_field(adap, A_MPS_CFG, 0, F_PORT0ACTIVE); else t3_set_reg_field(adap, A_MPS_CFG, 0, F_PORT1ACTIVE); return 0; } /* * Set the exact match register 'idx' to recognize the given Ethernet address. */ static void set_addr_filter(struct cmac *mac, int idx, const u8 *addr) { u32 addr_lo, addr_hi; unsigned int oft = mac->offset + idx * 8; addr_lo = (addr[3] << 24) | (addr[2] << 16) | (addr[1] << 8) | addr[0]; addr_hi = (addr[5] << 8) | addr[4]; t3_write_reg(mac->adapter, A_XGM_RX_EXACT_MATCH_LOW_1 + oft, addr_lo); t3_write_reg(mac->adapter, A_XGM_RX_EXACT_MATCH_HIGH_1 + oft, addr_hi); } /* Set one of the station's unicast MAC addresses. */ int t3_mac_set_address(struct cmac *mac, unsigned int idx, u8 addr[6]) { if (idx >= mac->nucast) return -EINVAL; set_addr_filter(mac, idx, addr); return 0; } /* * Specify the number of exact address filters that should be reserved for * unicast addresses. Caller should reload the unicast and multicast addresses * after calling this. */ int t3_mac_set_num_ucast(struct cmac *mac, int n) { if (n > EXACT_ADDR_FILTERS) return -EINVAL; mac->nucast = n; return 0; } /* Calculate the RX hash filter index of an Ethernet address */ static int hash_hw_addr(const u8 *addr) { int hash = 0, octet, bit, i = 0, c; for (octet = 0; octet < 6; ++octet) for (c = addr[octet], bit = 0; bit < 8; c >>= 1, ++bit) { hash ^= (c & 1) << i; if (++i == 6) i = 0; } return hash; } int t3_mac_set_rx_mode(struct cmac *mac, struct t3_rx_mode *rm) { u32 val, hash_lo, hash_hi; adapter_t *adap = mac->adapter; unsigned int oft = mac->offset; val = t3_read_reg(adap, A_XGM_RX_CFG + oft) & ~F_COPYALLFRAMES; if (promisc_rx_mode(rm)) val |= F_COPYALLFRAMES; t3_write_reg(adap, A_XGM_RX_CFG + oft, val); if (allmulti_rx_mode(rm)) hash_lo = hash_hi = 0xffffffff; else { u8 *addr; int exact_addr_idx = mac->nucast; hash_lo = hash_hi = 0; while ((addr = t3_get_next_mcaddr(rm))) if (exact_addr_idx < EXACT_ADDR_FILTERS) set_addr_filter(mac, exact_addr_idx++, addr); else { int hash = hash_hw_addr(addr); if (hash < 32) hash_lo |= (1 << hash); else hash_hi |= (1 << (hash - 32)); } } t3_write_reg(adap, A_XGM_RX_HASH_LOW + oft, hash_lo); t3_write_reg(adap, A_XGM_RX_HASH_HIGH + oft, hash_hi); return 0; } int t3_mac_set_mtu(struct cmac *mac, unsigned int mtu) { int hwm, lwm; unsigned int thres, v; adapter_t *adap = mac->adapter; /* * MAX_FRAME_SIZE inludes header + FCS, mtu doesn't. The HW max * packet size register includes header, but not FCS. */ mtu += 14; if (mtu > MAX_FRAME_SIZE - 4) return -EINVAL; t3_write_reg(adap, A_XGM_RX_MAX_PKT_SIZE + mac->offset, mtu); /* * Adjust the PAUSE frame watermarks. We always set the LWM, and the * HWM only if flow-control is enabled. */ hwm = max_t(unsigned int, MAC_RXFIFO_SIZE - 3 * mtu, MAC_RXFIFO_SIZE * 38 / 100); hwm = min(hwm, MAC_RXFIFO_SIZE - 8192); lwm = min(3 * (int) mtu, MAC_RXFIFO_SIZE /4); v = t3_read_reg(adap, A_XGM_RXFIFO_CFG + mac->offset); v &= ~V_RXFIFOPAUSELWM(M_RXFIFOPAUSELWM); v |= V_RXFIFOPAUSELWM(lwm / 8); if (G_RXFIFOPAUSEHWM(v)) v = (v & ~V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM)) | V_RXFIFOPAUSEHWM(hwm / 8); t3_write_reg(adap, A_XGM_RXFIFO_CFG + mac->offset, v); /* Adjust the TX FIFO threshold based on the MTU */ thres = (adap->params.vpd.cclk * 1000) / 15625; thres = (thres * mtu) / 1000; if (is_10G(adap)) thres /= 10; thres = mtu > thres ? (mtu - thres + 7) / 8 : 0; thres = max(thres, 8U); /* need at least 8 */ t3_set_reg_field(adap, A_XGM_TXFIFO_CFG + mac->offset, - V_TXFIFOTHRESH(M_TXFIFOTHRESH) | V_TXIPG(M_TXIPG), - V_TXFIFOTHRESH(thres) | V_TXIPG(1)); + V_TXFIFOTHRESH(M_TXFIFOTHRESH) | V_TXIPG(M_TXIPG), + V_TXFIFOTHRESH(thres) | V_TXIPG(1)); /* Assuming a minimum drain rate of 2.5Gbps... */ if (adap->params.rev > 0) t3_write_reg(adap, A_XGM_PAUSE_TIMER + mac->offset, (hwm-lwm) * 4 / 8); t3_write_reg(adap, A_XGM_TX_PAUSE_QUANTA + mac->offset, MAC_RXFIFO_SIZE * 4 * 8 / 512); return 0; } int t3_mac_set_speed_duplex_fc(struct cmac *mac, int speed, int duplex, int fc) { u32 val; adapter_t *adap = mac->adapter; unsigned int oft = mac->offset; if (duplex >= 0 && duplex != DUPLEX_FULL) return -EINVAL; if (speed >= 0) { if (speed == SPEED_10) val = V_PORTSPEED(0); else if (speed == SPEED_100) val = V_PORTSPEED(1); else if (speed == SPEED_1000) val = V_PORTSPEED(2); else if (speed == SPEED_10000) val = V_PORTSPEED(3); else return -EINVAL; t3_set_reg_field(adap, A_XGM_PORT_CFG + oft, V_PORTSPEED(M_PORTSPEED), val); } -#if 0 +#if 0 val = t3_read_reg(adap, A_XGM_RXFIFO_CFG + oft); val &= ~V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM); if (fc & PAUSE_TX) val |= V_RXFIFOPAUSEHWM(G_RXFIFOPAUSELWM(val) + 128); /* +1KB */ t3_write_reg(adap, A_XGM_RXFIFO_CFG + oft, val); #endif t3_set_reg_field(adap, A_XGM_TX_CFG + oft, F_TXPAUSEEN, (fc & PAUSE_RX) ? F_TXPAUSEEN : 0); return 0; } int t3_mac_enable(struct cmac *mac, int which) { int idx = macidx(mac); adapter_t *adap = mac->adapter; unsigned int oft = mac->offset; + struct mac_stats *s = &mac->stats; if (which & MAC_DIRECTION_TX) { t3_write_reg(adap, A_XGM_TX_CTRL + oft, F_TXEN); t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_CFG_CH0 + idx); t3_write_reg(adap, A_TP_PIO_DATA, 0xc0ede401); t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_MODE); t3_set_reg_field(adap, A_TP_PIO_DATA, 1 << idx, 1 << idx); t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_CNT_CH0 + idx); - mac->tcnt = (G_TXDROPCNTCH0RCVD(t3_read_reg(adap, A_TP_PIO_DATA))); - mac->xcnt = (G_TXSPI4SOPCNT(t3_read_reg(adap, A_XGM_TX_SPI4_SOP_EOP_CNT))); + mac->tx_mcnt = s->tx_frames; + mac->tx_tcnt = (G_TXDROPCNTCH0RCVD(t3_read_reg(adap, + A_TP_PIO_DATA))); + mac->tx_xcnt = (G_TXSPI4SOPCNT(t3_read_reg(adap, + A_XGM_TX_SPI4_SOP_EOP_CNT + + oft))); + mac->rx_mcnt = s->rx_frames; + mac->rx_xcnt = (G_TXSPI4SOPCNT(t3_read_reg(adap, + A_XGM_RX_SPI4_SOP_EOP_CNT + + oft))); mac->txen = F_TXEN; mac->toggle_cnt = 0; } - if (which & MAC_DIRECTION_RX) + if (which & MAC_DIRECTION_RX) t3_write_reg(adap, A_XGM_RX_CTRL + oft, F_RXEN); return 0; } int t3_mac_disable(struct cmac *mac, int which) { int idx = macidx(mac); adapter_t *adap = mac->adapter; + int val; if (which & MAC_DIRECTION_TX) { t3_write_reg(adap, A_XGM_TX_CTRL + mac->offset, 0); t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_CFG_CH0 + idx); t3_write_reg(adap, A_TP_PIO_DATA, 0xc000001f); t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_MODE); t3_set_reg_field(adap, A_TP_PIO_DATA, 1 << idx, 1 << idx); mac->txen = 0; } - if (which & MAC_DIRECTION_RX) + if (which & MAC_DIRECTION_RX) { + t3_set_reg_field(mac->adapter, A_XGM_RESET_CTRL + mac->offset, + F_PCS_RESET_, 0); + t3_os_sleep(100); t3_write_reg(adap, A_XGM_RX_CTRL + mac->offset, 0); + val = F_MAC_RESET_; + if (is_10G(adap)) + val |= F_PCS_RESET_; + else if (uses_xaui(adap)) + val |= F_PCS_RESET_ | F_XG2G_RESET_; + else + val |= F_RGMII_RESET_ | F_XG2G_RESET_; + t3_write_reg(mac->adapter, A_XGM_RESET_CTRL + mac->offset, val); + } return 0; } int t3b2_mac_watchdog_task(struct cmac *mac) { int status; - unsigned int tcnt, xcnt; + unsigned int tx_tcnt, tx_xcnt; adapter_t *adap = mac->adapter; - t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_CNT_CH0 + macidx(mac)); - tcnt = (G_TXDROPCNTCH0RCVD(t3_read_reg(adap, A_TP_PIO_DATA))); - xcnt = (G_TXSPI4SOPCNT(t3_read_reg(adap, A_XGM_TX_SPI4_SOP_EOP_CNT + mac->offset))); + struct mac_stats *s = &mac->stats; + unsigned int tx_mcnt = (unsigned int)s->tx_frames; + unsigned int rx_mcnt = (unsigned int)s->rx_frames; + unsigned int rx_xcnt; - if ((tcnt != mac->tcnt) && (xcnt == 0) && (mac->xcnt == 0)) { + status = 0; + tx_xcnt = 1; /* By default tx_xcnt is making progress*/ + tx_tcnt = mac->tx_tcnt; /* If tx_mcnt is progressing ignore tx_tcnt*/ + rx_xcnt = 1; /* By default rx_xcnt is making progress*/ + if (tx_mcnt == mac->tx_mcnt) { + tx_xcnt = (G_TXSPI4SOPCNT(t3_read_reg(adap, + A_XGM_TX_SPI4_SOP_EOP_CNT + + mac->offset))); + if (tx_xcnt == 0) { + t3_write_reg(adap, A_TP_PIO_ADDR, + A_TP_TX_DROP_CNT_CH0 + macidx(mac)); + tx_tcnt = (G_TXDROPCNTCH0RCVD(t3_read_reg(adap, + A_TP_PIO_DATA))); + } else { + goto rxcheck; + } + } else { + mac->toggle_cnt = 0; + goto rxcheck; + } + + if (((tx_tcnt != mac->tx_tcnt) && + (tx_xcnt == 0) && (mac->tx_xcnt == 0)) || + ((mac->tx_mcnt == tx_mcnt) && + (tx_xcnt != 0) && (mac->tx_xcnt != 0))) { if (mac->toggle_cnt > 4) { - t3b2_mac_reset(mac); - mac->toggle_cnt = 0; status = 2; + goto out; } else { - t3_write_reg(adap, A_XGM_TX_CTRL + mac->offset, 0); - t3_read_reg(adap, A_XGM_TX_CTRL + mac->offset); /* flush */ - t3_write_reg(adap, A_XGM_TX_CTRL + mac->offset, mac->txen); - t3_read_reg(adap, A_XGM_TX_CTRL + mac->offset); /* flush */ - mac->toggle_cnt++; status = 1; - } + goto out; + } } else { mac->toggle_cnt = 0; - status = 0; + goto rxcheck; } - mac->tcnt = tcnt; - mac->xcnt = xcnt; + +rxcheck: + if (rx_mcnt != mac->rx_mcnt) + rx_xcnt = (G_TXSPI4SOPCNT(t3_read_reg(adap, + A_XGM_RX_SPI4_SOP_EOP_CNT + + mac->offset))); + else + goto out; + + if (mac->rx_mcnt != s->rx_frames && rx_xcnt == 0 && mac->rx_xcnt == 0) { + status = 2; + goto out; + } + +out: + mac->tx_tcnt = tx_tcnt; + mac->tx_xcnt = tx_xcnt; + mac->tx_mcnt = s->tx_frames; + mac->rx_xcnt = rx_xcnt; + mac->rx_mcnt = s->rx_frames; + if (status == 1) { + t3_write_reg(adap, A_XGM_TX_CTRL + mac->offset, 0); + t3_read_reg(adap, A_XGM_TX_CTRL + mac->offset); /* flush */ + t3_write_reg(adap, A_XGM_TX_CTRL + mac->offset, mac->txen); + t3_read_reg(adap, A_XGM_TX_CTRL + mac->offset); /* flush */ + mac->toggle_cnt++; + } else if (status == 2) { + t3b2_mac_reset(mac); + mac->toggle_cnt = 0; + } return status; } /* * This function is called periodically to accumulate the current values of the * RMON counters into the port statistics. Since the packet counters are only * 32 bits they can overflow in ~286 secs at 10G, so the function should be * called more frequently than that. The byte counters are 45-bit wide, they * would overflow in ~7.8 hours. */ const struct mac_stats *t3_mac_update_stats(struct cmac *mac) { #define RMON_READ(mac, addr) t3_read_reg(mac->adapter, addr + mac->offset) #define RMON_UPDATE(mac, name, reg) \ (mac)->stats.name += (u64)RMON_READ(mac, A_XGM_STAT_##reg) #define RMON_UPDATE64(mac, name, reg_lo, reg_hi) \ (mac)->stats.name += RMON_READ(mac, A_XGM_STAT_##reg_lo) + \ ((u64)RMON_READ(mac, A_XGM_STAT_##reg_hi) << 32) u32 v, lo; RMON_UPDATE64(mac, rx_octets, RX_BYTES_LOW, RX_BYTES_HIGH); RMON_UPDATE64(mac, rx_frames, RX_FRAMES_LOW, RX_FRAMES_HIGH); RMON_UPDATE(mac, rx_mcast_frames, RX_MCAST_FRAMES); RMON_UPDATE(mac, rx_bcast_frames, RX_BCAST_FRAMES); RMON_UPDATE(mac, rx_fcs_errs, RX_CRC_ERR_FRAMES); RMON_UPDATE(mac, rx_pause, RX_PAUSE_FRAMES); RMON_UPDATE(mac, rx_jabber, RX_JABBER_FRAMES); RMON_UPDATE(mac, rx_short, RX_SHORT_FRAMES); RMON_UPDATE(mac, rx_symbol_errs, RX_SYM_CODE_ERR_FRAMES); RMON_UPDATE(mac, rx_too_long, RX_OVERSIZE_FRAMES); v = RMON_READ(mac, A_XGM_RX_MAX_PKT_SIZE_ERR_CNT); if (mac->adapter->params.rev == T3_REV_B2) v &= 0x7fffffff; mac->stats.rx_too_long += v; RMON_UPDATE(mac, rx_frames_64, RX_64B_FRAMES); RMON_UPDATE(mac, rx_frames_65_127, RX_65_127B_FRAMES); RMON_UPDATE(mac, rx_frames_128_255, RX_128_255B_FRAMES); RMON_UPDATE(mac, rx_frames_256_511, RX_256_511B_FRAMES); RMON_UPDATE(mac, rx_frames_512_1023, RX_512_1023B_FRAMES); RMON_UPDATE(mac, rx_frames_1024_1518, RX_1024_1518B_FRAMES); RMON_UPDATE(mac, rx_frames_1519_max, RX_1519_MAXB_FRAMES); RMON_UPDATE64(mac, tx_octets, TX_BYTE_LOW, TX_BYTE_HIGH); RMON_UPDATE64(mac, tx_frames, TX_FRAME_LOW, TX_FRAME_HIGH); RMON_UPDATE(mac, tx_mcast_frames, TX_MCAST); RMON_UPDATE(mac, tx_bcast_frames, TX_BCAST); RMON_UPDATE(mac, tx_pause, TX_PAUSE); /* This counts error frames in general (bad FCS, underrun, etc). */ RMON_UPDATE(mac, tx_underrun, TX_ERR_FRAMES); RMON_UPDATE(mac, tx_frames_64, TX_64B_FRAMES); RMON_UPDATE(mac, tx_frames_65_127, TX_65_127B_FRAMES); RMON_UPDATE(mac, tx_frames_128_255, TX_128_255B_FRAMES); RMON_UPDATE(mac, tx_frames_256_511, TX_256_511B_FRAMES); RMON_UPDATE(mac, tx_frames_512_1023, TX_512_1023B_FRAMES); RMON_UPDATE(mac, tx_frames_1024_1518, TX_1024_1518B_FRAMES); RMON_UPDATE(mac, tx_frames_1519_max, TX_1519_MAXB_FRAMES); /* The next stat isn't clear-on-read. */ t3_write_reg(mac->adapter, A_TP_MIB_INDEX, mac->offset ? 51 : 50); v = t3_read_reg(mac->adapter, A_TP_MIB_RDATA); lo = (u32)mac->stats.rx_cong_drops; mac->stats.rx_cong_drops += (u64)(v - lo); return &mac->stats; } Index: head/sys/dev/cxgb/common/jhash.h =================================================================== --- head/sys/dev/cxgb/common/jhash.h (nonexistent) +++ head/sys/dev/cxgb/common/jhash.h (revision 169978) @@ -0,0 +1,140 @@ +#ifndef _JHASH_H +#define _JHASH_H + +/* jhash.h: Jenkins hash support. + * + * Copyright (C) 1996 Bob Jenkins (bob_jenkins@burtleburtle.net) + * + * http://burtleburtle.net/bob/hash/ + * + * These are the credits from Bob's sources: + * + * lookup2.c, by Bob Jenkins, December 1996, Public Domain. + * hash(), hash2(), hash3, and mix() are externally useful functions. + * Routines to test the hash are included if SELF_TEST is defined. + * You can use this free for any purpose. It has no warranty. + * + * $FreeBSD$ + */ + +/* NOTE: Arguments are modified. */ +#define __jhash_mix(a, b, c) \ +{ \ + a -= b; a -= c; a ^= (c>>13); \ + b -= c; b -= a; b ^= (a<<8); \ + c -= a; c -= b; c ^= (b>>13); \ + a -= b; a -= c; a ^= (c>>12); \ + b -= c; b -= a; b ^= (a<<16); \ + c -= a; c -= b; c ^= (b>>5); \ + a -= b; a -= c; a ^= (c>>3); \ + b -= c; b -= a; b ^= (a<<10); \ + c -= a; c -= b; c ^= (b>>15); \ +} + +/* The golden ration: an arbitrary value */ +#define JHASH_GOLDEN_RATIO 0x9e3779b9 + +/* The most generic version, hashes an arbitrary sequence + * of bytes. No alignment or length assumptions are made about + * the input key. + */ +static inline u32 jhash(const void *key, u32 length, u32 initval) +{ + u32 a, b, c, len; + const u8 *k = key; + + len = length; + a = b = JHASH_GOLDEN_RATIO; + c = initval; + + while (len >= 12) { + a += (k[0] +((u32)k[1]<<8) +((u32)k[2]<<16) +((u32)k[3]<<24)); + b += (k[4] +((u32)k[5]<<8) +((u32)k[6]<<16) +((u32)k[7]<<24)); + c += (k[8] +((u32)k[9]<<8) +((u32)k[10]<<16)+((u32)k[11]<<24)); + + __jhash_mix(a,b,c); + + k += 12; + len -= 12; + } + + c += length; + switch (len) { + case 11: c += ((u32)k[10]<<24); + case 10: c += ((u32)k[9]<<16); + case 9 : c += ((u32)k[8]<<8); + case 8 : b += ((u32)k[7]<<24); + case 7 : b += ((u32)k[6]<<16); + case 6 : b += ((u32)k[5]<<8); + case 5 : b += k[4]; + case 4 : a += ((u32)k[3]<<24); + case 3 : a += ((u32)k[2]<<16); + case 2 : a += ((u32)k[1]<<8); + case 1 : a += k[0]; + }; + + __jhash_mix(a,b,c); + + return c; +} + +/* A special optimized version that handles 1 or more of u32s. + * The length parameter here is the number of u32s in the key. + */ +static inline u32 jhash2(u32 *k, u32 length, u32 initval) +{ + u32 a, b, c, len; + + a = b = JHASH_GOLDEN_RATIO; + c = initval; + len = length; + + while (len >= 3) { + a += k[0]; + b += k[1]; + c += k[2]; + __jhash_mix(a, b, c); + k += 3; len -= 3; + } + + c += length * 4; + + switch (len) { + case 2 : b += k[1]; + case 1 : a += k[0]; + }; + + __jhash_mix(a,b,c); + + return c; +} + + +/* A special ultra-optimized versions that knows they are hashing exactly + * 3, 2 or 1 word(s). + * + * NOTE: In partilar the "c += length; __jhash_mix(a,b,c);" normally + * done at the end is not done here. + */ +static inline u32 jhash_3words(u32 a, u32 b, u32 c, u32 initval) +{ + a += JHASH_GOLDEN_RATIO; + b += JHASH_GOLDEN_RATIO; + c += initval; + + __jhash_mix(a, b, c); + + return c; +} + +static inline u32 jhash_2words(u32 a, u32 b, u32 initval) +{ + return jhash_3words(a, b, 0, initval); +} + +static inline u32 jhash_1word(u32 a, u32 initval) +{ + return jhash_3words(a, 0, 0, initval); +} + +#endif /* _JHASH_H */ Property changes on: head/sys/dev/cxgb/common/jhash.h ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/dev/cxgb/cxgb_adapter.h =================================================================== --- head/sys/dev/cxgb/cxgb_adapter.h (revision 169977) +++ head/sys/dev/cxgb/cxgb_adapter.h (revision 169978) @@ -1,442 +1,465 @@ /************************************************************************** Copyright (c) 2007, Chelsio Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Chelsio Corporation nor the names of its + 2. Neither the name of the Chelsio Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. $FreeBSD$ ***************************************************************************/ #ifndef _CXGB_ADAPTER_H_ #define _CXGB_ADAPTER_H_ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include +#include + struct adapter; struct sge_qset; extern int cxgb_debug; struct port_info { struct adapter *adapter; struct ifnet *ifp; int if_flags; const struct port_type_info *port_type; struct cphy phy; struct cmac mac; struct link_config link_config; struct ifmedia media; struct mtx lock; int port; uint8_t hw_addr[ETHER_ADDR_LEN]; uint8_t nqsets; uint8_t first_qset; struct taskqueue *tq; struct task start_task; struct cdev *port_cdev; }; enum { /* adapter flags */ FULL_INIT_DONE = (1 << 0), USING_MSI = (1 << 1), USING_MSIX = (1 << 2), QUEUES_BOUND = (1 << 3), FW_UPTODATE = (1 << 4), }; -/* Max active LRO sessions per queue set */ -#define MAX_LRO_PER_QSET 8 - #define FL_Q_SIZE 4096 #define JUMBO_Q_SIZE 512 #define RSPQ_Q_SIZE 1024 #define TX_ETH_Q_SIZE 1024 /* * Types of Tx queues in each queue set. Order here matters, do not change. * XXX TOE is not implemented yet, so the extra queues are just placeholders. */ enum { TXQ_ETH, TXQ_OFLD, TXQ_CTRL }; /* careful, the following are set on priv_flags and must not collide with * IFF_ flags! */ enum { LRO_ACTIVE = (1 << 8), }; -struct sge_lro_session { - struct mbuf *m; +/* Max concurrent LRO sessions per queue set */ +#define MAX_LRO_SES 8 + +struct t3_lro_session { + struct mbuf *head; + struct mbuf *tail; uint32_t seq; uint16_t ip_len; + uint16_t vtag; + uint8_t npkts; }; -struct sge_lro { - unsigned int enabled; - unsigned int num_active; - struct sge_lro_session *last_s; - struct sge_lro_session s[MAX_LRO_PER_QSET]; +struct lro_state { + unsigned short enabled; + unsigned short active_idx; + unsigned int nactive; + struct t3_lro_session sess[MAX_LRO_SES]; }; -/* has its own header on linux XXX - * but I don't even know what it is :-/ - */ - -struct t3cdev { - int foo; /* XXX fill in */ -}; - #define RX_BUNDLE_SIZE 8 struct rsp_desc; struct sge_rspq { uint32_t credits; uint32_t size; uint32_t cidx; uint32_t gen; uint32_t polling; uint32_t holdoff_tmr; uint32_t next_holdoff; uint32_t imm_data; - uint32_t pure_rsps; struct rsp_desc *desc; - bus_addr_t phys_addr; uint32_t cntxt_id; + struct mtx lock; + struct mbuf *rx_head; /* offload packet receive queue head */ + struct mbuf *rx_tail; /* offload packet receive queue tail */ + + uint32_t offload_pkts; + uint32_t offload_bundles; + uint32_t pure_rsps; + uint32_t unhandled_irqs; + + bus_addr_t phys_addr; bus_dma_tag_t desc_tag; bus_dmamap_t desc_map; struct mbuf *m; - struct mtx lock; }; struct rx_desc; struct rx_sw_desc; struct sge_fl { uint32_t buf_size; uint32_t credits; uint32_t size; uint32_t cidx; uint32_t pidx; uint32_t gen; struct rx_desc *desc; struct rx_sw_desc *sdesc; bus_addr_t phys_addr; uint32_t cntxt_id; uint64_t empty; bus_dma_tag_t desc_tag; bus_dmamap_t desc_map; bus_dma_tag_t entry_tag; uma_zone_t zone; int type; }; struct tx_desc; struct tx_sw_desc; struct sge_txq { uint64_t flags; uint32_t in_use; uint32_t size; uint32_t processed; uint32_t cleaned; uint32_t stop_thres; uint32_t cidx; uint32_t pidx; uint32_t gen; uint32_t unacked; struct tx_desc *desc; struct tx_sw_desc *sdesc; uint32_t token; bus_addr_t phys_addr; + struct task qresume_tsk; uint32_t cntxt_id; uint64_t stops; uint64_t restarts; bus_dma_tag_t desc_tag; bus_dmamap_t desc_map; bus_dma_tag_t entry_tag; + struct mbuf_head sendq; struct mtx lock; }; enum { SGE_PSTAT_TSO, /* # of TSO requests */ SGE_PSTAT_RX_CSUM_GOOD, /* # of successful RX csum offloads */ SGE_PSTAT_TX_CSUM, /* # of TX checksum offloads */ SGE_PSTAT_VLANEX, /* # of VLAN tag extractions */ SGE_PSTAT_VLANINS, /* # of VLAN tag insertions */ SGE_PSTATS_LRO_QUEUED, /* # of LRO appended packets */ SGE_PSTATS_LRO_FLUSHED, /* # of LRO flushed packets */ SGE_PSTATS_LRO_X_STREAMS, /* # of exceeded LRO contexts */ }; #define SGE_PSTAT_MAX (SGE_PSTATS_LRO_X_STREAMS+1) struct sge_qset { struct sge_rspq rspq; struct sge_fl fl[SGE_RXQ_PER_SET]; - struct sge_lro lro; + struct lro_state lro; struct sge_txq txq[SGE_TXQ_PER_SET]; - unsigned long txq_stopped; /* which Tx queues are stopped */ + uint32_t txq_stopped; /* which Tx queues are stopped */ uint64_t port_stats[SGE_PSTAT_MAX]; struct port_info *port; int idx; /* qset # */ }; struct sge { struct sge_qset qs[SGE_QSETS]; struct mtx reg_lock; }; struct adapter { device_t dev; int flags; + TAILQ_ENTRY(adapter) adapter_entry; /* PCI register resources */ uint32_t regs_rid; struct resource *regs_res; bus_space_handle_t bh; bus_space_tag_t bt; bus_size_t mmio_len; uint32_t link_width; + /* DMA resources */ bus_dma_tag_t parent_dmat; bus_dma_tag_t rx_dmat; bus_dma_tag_t rx_jumbo_dmat; bus_dma_tag_t tx_dmat; /* Interrupt resources */ struct resource *irq_res; int irq_rid; void *intr_tag; uint32_t msix_regs_rid; struct resource *msix_regs_res; struct resource *msix_irq_res[SGE_QSETS]; int msix_irq_rid[SGE_QSETS]; void *msix_intr_tag[SGE_QSETS]; /* Tasks */ struct task ext_intr_task; struct task timer_reclaim_task; struct task slow_intr_task; struct task process_responses_task; struct task mr_refresh_task; struct taskqueue *tq; struct callout cxgb_tick_ch; struct callout sge_timer_ch; /* Register lock for use by the hardware layer */ struct mtx mdio_lock; /* Bookkeeping for the hardware layer */ struct adapter_params params; unsigned int slow_intr_mask; unsigned long irq_stats[IRQ_NUM_STATS]; struct sge sge; struct mc7 pmrx; struct mc7 pmtx; struct mc7 cm; struct mc5 mc5; struct port_info port[MAX_NPORTS]; device_t portdev[MAX_NPORTS]; - struct t3cdev tdev; + struct toedev tdev; char fw_version[64]; uint32_t open_device_map; + uint32_t registered_device_map; struct mtx lock; + driver_intr_t *cxgb_intr; + int msi_count; }; struct t3_rx_mode { uint32_t idx; struct port_info *port; }; #define MDIO_LOCK(adapter) mtx_lock(&(adapter)->mdio_lock) #define MDIO_UNLOCK(adapter) mtx_unlock(&(adapter)->mdio_lock) #define PORT_LOCK(port) mtx_lock(&(port)->lock); #define PORT_UNLOCK(port) mtx_unlock(&(port)->lock); #define ADAPTER_LOCK(adap) mtx_lock(&(adap)->lock); #define ADAPTER_UNLOCK(adap) mtx_unlock(&(adap)->lock); static __inline uint32_t t3_read_reg(adapter_t *adapter, uint32_t reg_addr) { return (bus_space_read_4(adapter->bt, adapter->bh, reg_addr)); } static __inline void t3_write_reg(adapter_t *adapter, uint32_t reg_addr, uint32_t val) { bus_space_write_4(adapter->bt, adapter->bh, reg_addr, val); } static __inline void t3_os_pci_read_config_4(adapter_t *adapter, int reg, uint32_t *val) { *val = pci_read_config(adapter->dev, reg, 4); } static __inline void t3_os_pci_write_config_4(adapter_t *adapter, int reg, uint32_t val) { pci_write_config(adapter->dev, reg, val, 4); } static __inline void t3_os_pci_read_config_2(adapter_t *adapter, int reg, uint16_t *val) { *val = pci_read_config(adapter->dev, reg, 2); } static __inline void t3_os_pci_write_config_2(adapter_t *adapter, int reg, uint16_t val) { pci_write_config(adapter->dev, reg, val, 2); } static __inline uint8_t * t3_get_next_mcaddr(struct t3_rx_mode *rm) { uint8_t *macaddr = NULL; if (rm->idx == 0) macaddr = rm->port->hw_addr; rm->idx++; return (macaddr); } static __inline void t3_init_rx_mode(struct t3_rx_mode *rm, struct port_info *port) { rm->idx = 0; rm->port = port; } static __inline struct port_info * adap2pinfo(struct adapter *adap, int idx) { return &adap->port[idx]; } int t3_os_find_pci_capability(adapter_t *adapter, int cap); int t3_os_pci_save_state(struct adapter *adapter); int t3_os_pci_restore_state(struct adapter *adapter); void t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, int duplex, int fc); void t3_sge_err_intr_handler(adapter_t *adapter); +int t3_offload_tx(struct toedev *, struct mbuf *); void t3_os_ext_intr_handler(adapter_t *adapter); void t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]); int t3_mgmt_tx(adapter_t *adap, struct mbuf *m); int t3_sge_alloc(struct adapter *); int t3_sge_free(struct adapter *); int t3_sge_alloc_qset(adapter_t *, uint32_t, int, int, const struct qset_params *, int, struct port_info *); void t3_free_sge_resources(adapter_t *); void t3_sge_start(adapter_t *); +void t3_sge_stop(adapter_t *); void t3b_intr(void *data); void t3_intr_msi(void *data); void t3_intr_msix(void *data); int t3_encap(struct port_info *, struct mbuf **); int t3_sge_init_sw(adapter_t *); void t3_sge_deinit_sw(adapter_t *); void t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad, uint32_t rss_hash, uint32_t rss_csum, int lro); void t3_rx_eth(struct port_info *p, struct sge_rspq *rq, struct mbuf *m, int ethpad); -void t3_sge_lro_flush_all(adapter_t *adap, struct sge_qset *qs); +void t3_lro_flush(adapter_t *adap, struct sge_qset *qs, struct lro_state *state); void t3_add_sysctls(adapter_t *sc); int t3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx, unsigned char *data); void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p); /* * XXX figure out how we can return this to being private to sge */ #define desc_reclaimable(q) ((int)((q)->processed - (q)->cleaned - TX_MAX_DESC)) #define container_of(p, stype, field) ((stype *)(((uint8_t *)(p)) - offsetof(stype, field))) static __inline struct sge_qset * fl_to_qset(struct sge_fl *q, int qidx) { return container_of(q, struct sge_qset, fl[qidx]); } static __inline struct sge_qset * rspq_to_qset(struct sge_rspq *q) { return container_of(q, struct sge_qset, rspq); } static __inline struct sge_qset * txq_to_qset(struct sge_txq *q, int qidx) { return container_of(q, struct sge_qset, txq[qidx]); } +static __inline struct adapter * +tdev2adap(struct toedev *d) +{ + return container_of(d, struct adapter, tdev); +} + #undef container_of + +#define OFFLOAD_DEVMAP_BIT 15 +static inline int offload_running(adapter_t *adapter) +{ + return isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); +} + #endif Index: head/sys/dev/cxgb/cxgb_config.h =================================================================== --- head/sys/dev/cxgb/cxgb_config.h (revision 169977) +++ head/sys/dev/cxgb/cxgb_config.h (revision 169978) @@ -1,44 +1,40 @@ /************************************************************************** Copyright (c) 2007, Chelsio Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Chelsio Corporation nor the names of its + 2. Neither the name of the Chelsio Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. $FreeBSD$ ***************************************************************************/ #ifndef _CXGB_CONFIG_H_ #define _CXGB_CONFIG_H_ #ifndef CONFIG_DEFINED #define CONFIG_CHELSIO_T3_CORE #define DEFAULT_JUMBO #endif #endif Index: head/sys/dev/cxgb/cxgb_ioctl.h =================================================================== --- head/sys/dev/cxgb/cxgb_ioctl.h (revision 169977) +++ head/sys/dev/cxgb/cxgb_ioctl.h (revision 169978) @@ -1,234 +1,236 @@ /************************************************************************** Copyright (c) 2007, Chelsio Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Chelsio Corporation nor the names of its + 2. Neither the name of the Chelsio Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. $FreeBSD$ ***************************************************************************/ #ifndef __CHIOCTL_H__ #define __CHIOCTL_H__ /* * Ioctl commands specific to this driver. */ enum { CH_SETREG = 0x40, CH_GETREG, CH_SETTPI, CH_GETTPI, CH_DEVUP, CH_GETMTUTAB, CH_SETMTUTAB, CH_GETMTU, CH_SET_PM, CH_GET_PM, CH_GET_TCAM, CH_SET_TCAM, CH_GET_TCB, CH_READ_TCAM_WORD, CH_GET_MEM, CH_GET_SGE_CONTEXT, CH_GET_SGE_DESC, CH_LOAD_FW, CH_GET_PROTO, CH_SET_PROTO, CH_SET_TRACE_FILTER, CH_SET_QSET_PARAMS, CH_GET_QSET_PARAMS, CH_SET_QSET_NUM, CH_GET_QSET_NUM, CH_SET_PKTSCHED, CH_IFCONF_GETREGS, CH_GETMIIREGS, CH_SETMIIREGS, - + CH_SET_FILTER, + CH_SET_HW_SCHED, }; struct ch_reg { uint32_t addr; uint32_t val; }; struct ch_cntxt { uint32_t cntxt_type; uint32_t cntxt_id; uint32_t data[4]; }; /* context types */ enum { CNTXT_TYPE_EGRESS, CNTXT_TYPE_FL, CNTXT_TYPE_RSP, CNTXT_TYPE_CQ }; struct ch_desc { uint32_t cmd; uint32_t queue_num; uint32_t idx; uint32_t size; uint8_t data[128]; }; struct ch_mem_range { uint32_t cmd; uint32_t mem_id; uint32_t addr; uint32_t len; uint32_t version; uint8_t *buf; }; struct ch_qset_params { uint32_t qset_idx; int32_t txq_size[3]; int32_t rspq_size; int32_t fl_size[2]; int32_t intr_lat; int32_t polling; int32_t cong_thres; int32_t vector; int32_t qnum; }; struct ch_pktsched_params { uint32_t cmd; uint8_t sched; uint8_t idx; uint8_t min; uint8_t max; uint8_t binding; }; struct ch_hw_sched { uint32_t cmd; uint8_t sched; int8_t mode; int8_t channel; int32_t kbps; /* rate in Kbps */ int32_t class_ipg; /* tenths of nanoseconds */ int32_t flow_ipg; /* usec */ }; #ifndef TCB_SIZE # define TCB_SIZE 128 #endif /* TCB size in 32-bit words */ #define TCB_WORDS (TCB_SIZE / 4) enum { MEM_CM, MEM_PMRX, MEM_PMTX }; /* ch_mem_range.mem_id values */ struct ch_mtus { uint32_t cmd; uint32_t nmtus; uint16_t mtus[NMTUS]; }; struct ch_pm { uint32_t cmd; uint32_t tx_pg_sz; uint32_t tx_num_pg; uint32_t rx_pg_sz; uint32_t rx_num_pg; uint32_t pm_total; }; struct ch_tcam { uint32_t cmd; uint32_t tcam_size; uint32_t nservers; uint32_t nroutes; uint32_t nfilters; }; struct ch_tcb { uint32_t cmd; uint32_t tcb_index; uint32_t tcb_data[TCB_WORDS]; }; struct ch_tcam_word { uint32_t cmd; uint32_t addr; uint32_t buf[3]; }; struct ch_trace { uint32_t cmd; uint32_t sip; uint32_t sip_mask; uint32_t dip; uint32_t dip_mask; uint16_t sport; uint16_t sport_mask; uint16_t dport; uint16_t dport_mask; uint32_t vlan:12, vlan_mask:12, intf:4, intf_mask:4; uint8_t proto; uint8_t proto_mask; uint8_t invert_match:1, config_tx:1, config_rx:1, trace_tx:1, trace_rx:1; }; #define REGDUMP_SIZE (4 * 1024) struct ifconf_regs { uint32_t version; uint32_t len; /* bytes */ uint8_t *data; }; struct mii_data { uint32_t phy_id; uint32_t reg_num; uint32_t val_in; uint32_t val_out; }; #define CHELSIO_SETREG _IOW('f', CH_SETREG, struct ch_reg) #define CHELSIO_GETREG _IOWR('f', CH_GETREG, struct ch_reg) +#define CHELSIO_READ_TCAM_WORD _IOR('f', CH_READ_TCAM_WORD, struct ch_tcam) #define CHELSIO_GET_MEM _IOWR('f', CH_GET_MEM, struct ch_mem_range) #define CHELSIO_GET_SGE_CONTEXT _IOWR('f', CH_GET_SGE_CONTEXT, struct ch_cntxt) #define CHELSIO_GET_SGE_DESC _IOWR('f', CH_GET_SGE_DESC, struct ch_desc) #define CHELSIO_GET_QSET_PARAMS _IOWR('f', CH_GET_QSET_PARAMS, struct ch_qset_params) #define CHELSIO_SET_QSET_PARAMS _IOW('f', CH_SET_QSET_PARAMS, struct ch_qset_params) #define CHELSIO_GET_QSET_NUM _IOWR('f', CH_GET_QSET_NUM, struct ch_reg) #define CHELSIO_SET_QSET_NUM _IOW('f', CH_SET_QSET_NUM, struct ch_reg) +#define CHELSIO_GETMTUTAB _IOR('f', CH_GET_QSET_NUM, struct ch_mtus) +#define CHELSIO_SETMTUTAB _IOW('f', CH_SET_QSET_NUM, struct ch_mtus) #define CHELSIO_SET_TRACE_FILTER _IOW('f', CH_SET_TRACE_FILTER, struct ch_trace) #define CHELSIO_SET_PKTSCHED _IOW('f', CH_SET_PKTSCHED, struct ch_pktsched_params) #define CHELSIO_IFCONF_GETREGS _IOWR('f', CH_IFCONF_GETREGS, struct ifconf_regs) #define SIOCGMIIREG _IOWR('f', CH_GETMIIREGS, struct mii_data) #define SIOCSMIIREG _IOWR('f', CH_SETMIIREGS, struct mii_data) +#define CHELSIO_SET_HW_SCHED _IOWR('f', CH_SET_HW_SCHED, struct ch_hw_sched) +#define CHELSIO_DEVUP _IO('f', CH_DEVUP) #endif Index: head/sys/dev/cxgb/cxgb_l2t.c =================================================================== --- head/sys/dev/cxgb/cxgb_l2t.c (nonexistent) +++ head/sys/dev/cxgb/cxgb_l2t.c (revision 169978) @@ -0,0 +1,670 @@ +/************************************************************************** + +Copyright (c) 2007, Chelsio Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Neither the name of the Chelsio Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +***************************************************************************/ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include + + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#include +#include +#include +#include +#include +#include +#include + +#define VLAN_NONE 0xfff +#define SDL(s) ((struct sockaddr_dl *)s) +#define RT_ENADDR(rt) ((char *)LLADDR(SDL((rt)))) +#define rt_expire rt_rmx.rmx_expire + +struct llinfo_arp { + struct callout la_timer; + struct rtentry *la_rt; + struct mbuf *la_hold; /* last packet until resolved/timeout */ + u_short la_preempt; /* countdown for pre-expiry arps */ + u_short la_asked; /* # requests sent */ +}; + +/* + * Module locking notes: There is a RW lock protecting the L2 table as a + * whole plus a spinlock per L2T entry. Entry lookups and allocations happen + * under the protection of the table lock, individual entry changes happen + * while holding that entry's spinlock. The table lock nests outside the + * entry locks. Allocations of new entries take the table lock as writers so + * no other lookups can happen while allocating new entries. Entry updates + * take the table lock as readers so multiple entries can be updated in + * parallel. An L2T entry can be dropped by decrementing its reference count + * and therefore can happen in parallel with entry allocation but no entry + * can change state or increment its ref count during allocation as both of + * these perform lookups. + */ + +static inline unsigned int +vlan_prio(const struct l2t_entry *e) +{ + return e->vlan >> 13; +} + +static inline unsigned int +arp_hash(u32 key, int ifindex, const struct l2t_data *d) +{ + return jhash_2words(key, ifindex, 0) & (d->nentries - 1); +} + +static inline void +neigh_replace(struct l2t_entry *e, struct rtentry *rt) +{ + RT_LOCK(rt); + RT_ADDREF(rt); + RT_UNLOCK(rt); + + if (e->neigh) { + RT_LOCK(e->neigh); + RT_REMREF(e->neigh); + RT_UNLOCK(e->neigh); + } + e->neigh = rt; +} + +/* + * Set up an L2T entry and send any packets waiting in the arp queue. The + * supplied mbuf is used for the CPL_L2T_WRITE_REQ. Must be called with the + * entry locked. + */ +static int +setup_l2e_send_pending(struct toedev *dev, struct mbuf *m, + struct l2t_entry *e) +{ + struct cpl_l2t_write_req *req; + + if (!m) { + if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) + return (ENOMEM); + } + /* + * XXX MH_ALIGN + */ + req = mtod(m, struct cpl_l2t_write_req *); + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, e->idx)); + req->params = htonl(V_L2T_W_IDX(e->idx) | V_L2T_W_IFF(e->smt_idx) | + V_L2T_W_VLAN(e->vlan & EVL_VLID_MASK) | + V_L2T_W_PRIO(vlan_prio(e))); + + memcpy(e->dmac, RT_ENADDR(e->neigh), sizeof(e->dmac)); + memcpy(req->dst_mac, e->dmac, sizeof(req->dst_mac)); + m_set_priority(m, CPL_PRIORITY_CONTROL); + cxgb_ofld_send(dev, m); + while (e->arpq_head) { + m = e->arpq_head; + e->arpq_head = m->m_next; + m->m_next = NULL; + cxgb_ofld_send(dev, m); + } + e->arpq_tail = NULL; + e->state = L2T_STATE_VALID; + + return 0; +} + +/* + * Add a packet to the an L2T entry's queue of packets awaiting resolution. + * Must be called with the entry's lock held. + */ +static inline void +arpq_enqueue(struct l2t_entry *e, struct mbuf *m) +{ + m->m_next = NULL; + if (e->arpq_head) + e->arpq_tail->m_next = m; + else + e->arpq_head = m; + e->arpq_tail = m; +} + +int +t3_l2t_send_slow(struct toedev *dev, struct mbuf *m, + struct l2t_entry *e) +{ + struct rtentry *rt; + struct mbuf *m0; + + if ((m0 = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) + return (ENOMEM); + + rt = e->neigh; + +again: + switch (e->state) { + case L2T_STATE_STALE: /* entry is stale, kick off revalidation */ + arpresolve(rt->rt_ifp, rt, m0, rt->rt_gateway, RT_ENADDR(rt)); + mtx_lock(&e->lock); + if (e->state == L2T_STATE_STALE) + e->state = L2T_STATE_VALID; + mtx_unlock(&e->lock); + case L2T_STATE_VALID: /* fast-path, send the packet on */ + return cxgb_ofld_send(dev, m); + case L2T_STATE_RESOLVING: + mtx_lock(&e->lock); + if (e->state != L2T_STATE_RESOLVING) { // ARP already completed + mtx_unlock(&e->lock); + goto again; + } + arpq_enqueue(e, m); + mtx_unlock(&e->lock); + + if ((m0 = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) + return (ENOMEM); + /* + * Only the first packet added to the arpq should kick off + * resolution. However, because the m_gethdr below can fail, + * we allow each packet added to the arpq to retry resolution + * as a way of recovering from transient memory exhaustion. + * A better way would be to use a work request to retry L2T + * entries when there's no memory. + */ + if (arpresolve(rt->rt_ifp, rt, m0, rt->rt_gateway, RT_ENADDR(rt)) == 0) { + + mtx_lock(&e->lock); + if (e->arpq_head) + setup_l2e_send_pending(dev, m, e); + else + m_freem(m); + mtx_unlock(&e->lock); + } + } + return 0; +} + +void +t3_l2t_send_event(struct toedev *dev, struct l2t_entry *e) +{ + struct rtentry *rt; + struct mbuf *m0; + + if ((m0 = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) + return; + + rt = e->neigh; +again: + switch (e->state) { + case L2T_STATE_STALE: /* entry is stale, kick off revalidation */ + arpresolve(rt->rt_ifp, rt, m0, rt->rt_gateway, RT_ENADDR(rt)); + mtx_lock(&e->lock); + if (e->state == L2T_STATE_STALE) { + e->state = L2T_STATE_VALID; + } + mtx_unlock(&e->lock); + return; + case L2T_STATE_VALID: /* fast-path, send the packet on */ + return; + case L2T_STATE_RESOLVING: + mtx_lock(&e->lock); + if (e->state != L2T_STATE_RESOLVING) { // ARP already completed + mtx_unlock(&e->lock); + goto again; + } + mtx_unlock(&e->lock); + + if ((m0 = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) + return; + /* + * Only the first packet added to the arpq should kick off + * resolution. However, because the alloc_skb below can fail, + * we allow each packet added to the arpq to retry resolution + * as a way of recovering from transient memory exhaustion. + * A better way would be to use a work request to retry L2T + * entries when there's no memory. + */ + arpresolve(rt->rt_ifp, rt, m0, rt->rt_gateway, RT_ENADDR(rt)); + + } + return; +} +/* + * Allocate a free L2T entry. Must be called with l2t_data.lock held. + */ +static struct l2t_entry * +alloc_l2e(struct l2t_data *d) +{ + struct l2t_entry *end, *e, **p; + + if (!atomic_load_acq_int(&d->nfree)) + return NULL; + + /* there's definitely a free entry */ + for (e = d->rover, end = &d->l2tab[d->nentries]; e != end; ++e) + if (atomic_load_acq_int(&e->refcnt) == 0) + goto found; + + for (e = &d->l2tab[1]; atomic_load_acq_int(&e->refcnt); ++e) ; +found: + d->rover = e + 1; + atomic_add_int(&d->nfree, -1); + + /* + * The entry we found may be an inactive entry that is + * presently in the hash table. We need to remove it. + */ + if (e->state != L2T_STATE_UNUSED) { + int hash = arp_hash(e->addr, e->ifindex, d); + + for (p = &d->l2tab[hash].first; *p; p = &(*p)->next) + if (*p == e) { + *p = e->next; + break; + } + e->state = L2T_STATE_UNUSED; + } + return e; +} + +/* + * Called when an L2T entry has no more users. The entry is left in the hash + * table since it is likely to be reused but we also bump nfree to indicate + * that the entry can be reallocated for a different neighbor. We also drop + * the existing neighbor reference in case the neighbor is going away and is + * waiting on our reference. + * + * Because entries can be reallocated to other neighbors once their ref count + * drops to 0 we need to take the entry's lock to avoid races with a new + * incarnation. + */ +void +t3_l2e_free(struct l2t_data *d, struct l2t_entry *e) +{ + mtx_lock(&e->lock); + if (atomic_load_acq_int(&e->refcnt) == 0) { /* hasn't been recycled */ + if (e->neigh) { + RT_LOCK(e->neigh); + RT_REMREF(e->neigh); + RT_UNLOCK(e->neigh); + e->neigh = NULL; + } + } + mtx_unlock(&e->lock); + atomic_add_int(&d->nfree, 1); +} + +/* + * Update an L2T entry that was previously used for the same next hop as neigh. + * Must be called with softirqs disabled. + */ +static inline void +reuse_entry(struct l2t_entry *e, struct rtentry *neigh) +{ + struct llinfo_arp *la; + + la = (struct llinfo_arp *)neigh->rt_llinfo; + + mtx_lock(&e->lock); /* avoid race with t3_l2t_free */ + if (neigh != e->neigh) + neigh_replace(e, neigh); + + if (memcmp(e->dmac, RT_ENADDR(neigh), sizeof(e->dmac)) || + (neigh->rt_expire > time_uptime)) + e->state = L2T_STATE_RESOLVING; + else if (la->la_hold == NULL) + e->state = L2T_STATE_VALID; + else + e->state = L2T_STATE_STALE; + mtx_unlock(&e->lock); +} + +struct l2t_entry * +t3_l2t_get(struct toedev *dev, struct rtentry *neigh, + unsigned int smt_idx) +{ + struct l2t_entry *e; + struct l2t_data *d = L2DATA(dev); + u32 addr = *(u32 *) rt_key(neigh); + int ifidx = neigh->rt_ifp->if_index; + int hash = arp_hash(addr, ifidx, d); + + rw_wlock(&d->lock); + for (e = d->l2tab[hash].first; e; e = e->next) + if (e->addr == addr && e->ifindex == ifidx && + e->smt_idx == smt_idx) { + l2t_hold(d, e); + if (atomic_load_acq_int(&e->refcnt) == 1) + reuse_entry(e, neigh); + goto done; + } + + /* Need to allocate a new entry */ + e = alloc_l2e(d); + if (e) { + mtx_lock(&e->lock); /* avoid race with t3_l2t_free */ + e->next = d->l2tab[hash].first; + d->l2tab[hash].first = e; + e->state = L2T_STATE_RESOLVING; + e->addr = addr; + e->ifindex = ifidx; + e->smt_idx = smt_idx; + atomic_store_rel_int(&e->refcnt, 1); + neigh_replace(e, neigh); +#ifdef notyet + /* + * XXX need to add accessor function for vlan tag + */ + if (neigh->rt_ifp->if_vlantrunk) + e->vlan = VLAN_DEV_INFO(neigh->dev)->vlan_id; + else +#endif + e->vlan = VLAN_NONE; + mtx_unlock(&e->lock); + } +done: + rw_wunlock(&d->lock); + return e; +} + +/* + * Called when address resolution fails for an L2T entry to handle packets + * on the arpq head. If a packet specifies a failure handler it is invoked, + * otherwise the packets is sent to the TOE. + * + * XXX: maybe we should abandon the latter behavior and just require a failure + * handler. + */ +static void +handle_failed_resolution(struct toedev *dev, struct mbuf *arpq) +{ + + while (arpq) { + struct mbuf *m = arpq; +#ifdef notyet + struct l2t_mbuf_cb *cb = L2T_MBUF_CB(m); +#endif + arpq = m->m_next; + m->m_next = NULL; +#ifdef notyet + if (cb->arp_failure_handler) + cb->arp_failure_handler(dev, m); + else +#endif + cxgb_ofld_send(dev, m); + } + +} + +#if defined(NETEVENT) || !defined(CONFIG_CHELSIO_T3_MODULE) +/* + * Called when the host's ARP layer makes a change to some entry that is + * loaded into the HW L2 table. + */ +void +t3_l2t_update(struct toedev *dev, struct rtentry *neigh) +{ + struct l2t_entry *e; + struct mbuf *arpq = NULL; + struct l2t_data *d = L2DATA(dev); + u32 addr = *(u32 *) rt_key(neigh); + int ifidx = neigh->rt_ifp->if_index; + int hash = arp_hash(addr, ifidx, d); + struct llinfo_arp *la; + + rw_rlock(&d->lock); + for (e = d->l2tab[hash].first; e; e = e->next) + if (e->addr == addr && e->ifindex == ifidx) { + mtx_lock(&e->lock); + goto found; + } + rw_runlock(&d->lock); + return; + +found: + rw_runlock(&d->lock); + if (atomic_load_acq_int(&e->refcnt)) { + if (neigh != e->neigh) + neigh_replace(e, neigh); + + la = (struct llinfo_arp *)neigh->rt_llinfo; + if (e->state == L2T_STATE_RESOLVING) { + + if (la->la_asked >= 5 /* arp_maxtries */) { + arpq = e->arpq_head; + e->arpq_head = e->arpq_tail = NULL; + } else if (la->la_hold == NULL) + setup_l2e_send_pending(dev, NULL, e); + } else { + e->state = (la->la_hold == NULL) ? + L2T_STATE_VALID : L2T_STATE_STALE; + if (memcmp(e->dmac, RT_ENADDR(neigh), 6)) + setup_l2e_send_pending(dev, NULL, e); + } + } + mtx_unlock(&e->lock); + + if (arpq) + handle_failed_resolution(dev, arpq); +} +#else +/* + * Called from a kprobe, interrupts are off. + */ +void +t3_l2t_update(struct toedev *dev, struct rtentry *neigh) +{ + struct l2t_entry *e; + struct l2t_data *d = L2DATA(dev); + u32 addr = *(u32 *) rt_key(neigh); + int ifidx = neigh->dev->ifindex; + int hash = arp_hash(addr, ifidx, d); + + rw_rlock(&d->lock); + for (e = d->l2tab[hash].first; e; e = e->next) + if (e->addr == addr && e->ifindex == ifidx) { + mtx_lock(&e->lock); + if (atomic_load_acq_int(&e->refcnt)) { + if (neigh != e->neigh) + neigh_replace(e, neigh); + e->tdev = dev; + mod_timer(&e->update_timer, jiffies + 1); + } + mtx_unlock(&e->lock); + break; + } + rw_runlock(&d->lock); +} + +static void +update_timer_cb(unsigned long data) +{ + struct mbuf *arpq = NULL; + struct l2t_entry *e = (struct l2t_entry *)data; + struct rtentry *neigh = e->neigh; + struct toedev *dev = e->tdev; + + barrier(); + if (!atomic_load_acq_int(&e->refcnt)) + return; + + rw_rlock(&neigh->lock); + mtx_lock(&e->lock); + + if (atomic_load_acq_int(&e->refcnt)) { + if (e->state == L2T_STATE_RESOLVING) { + if (neigh->nud_state & NUD_FAILED) { + arpq = e->arpq_head; + e->arpq_head = e->arpq_tail = NULL; + } else if (neigh_is_connected(neigh) && e->arpq_head) + setup_l2e_send_pending(dev, NULL, e); + } else { + e->state = neigh_is_connected(neigh) ? + L2T_STATE_VALID : L2T_STATE_STALE; + if (memcmp(e->dmac, RT_ENADDR(neigh), sizeof(e->dmac))) + setup_l2e_send_pending(dev, NULL, e); + } + } + mtx_unlock(&e->lock); + rw_runlock(&neigh->lock); + + if (arpq) + handle_failed_resolution(dev, arpq); +} +#endif + +struct l2t_data * +t3_init_l2t(unsigned int l2t_capacity) +{ + struct l2t_data *d; + int i, size = sizeof(*d) + l2t_capacity * sizeof(struct l2t_entry); + + d = cxgb_alloc_mem(size); + if (!d) + return NULL; + + d->nentries = l2t_capacity; + d->rover = &d->l2tab[1]; /* entry 0 is not used */ + atomic_store_rel_int(&d->nfree, l2t_capacity - 1); + rw_init(&d->lock, "L2T"); + + for (i = 0; i < l2t_capacity; ++i) { + d->l2tab[i].idx = i; + d->l2tab[i].state = L2T_STATE_UNUSED; + mtx_init(&d->l2tab[i].lock, "L2TAB", NULL, MTX_DEF); + atomic_store_rel_int(&d->l2tab[i].refcnt, 0); +#ifndef NETEVENT +#ifdef CONFIG_CHELSIO_T3_MODULE + setup_timer(&d->l2tab[i].update_timer, update_timer_cb, + (unsigned long)&d->l2tab[i]); +#endif +#endif + } + return d; +} + +void +t3_free_l2t(struct l2t_data *d) +{ +#ifndef NETEVENT +#ifdef CONFIG_CHELSIO_T3_MODULE + int i; + + /* Stop all L2T timers */ + for (i = 0; i < d->nentries; ++i) + del_timer_sync(&d->l2tab[i].update_timer); +#endif +#endif + cxgb_free_mem(d); +} + +#ifdef CONFIG_PROC_FS +#include +#include +#include + +static inline void * +l2t_get_idx(struct seq_file *seq, loff_t pos) +{ + struct l2t_data *d = seq->private; + + return pos >= d->nentries ? NULL : &d->l2tab[pos]; +} + +static void * +l2t_seq_start(struct seq_file *seq, loff_t *pos) +{ + return *pos ? l2t_get_idx(seq, *pos) : SEQ_START_TOKEN; +} + +static void * +l2t_seq_next(struct seq_file *seq, void *v, loff_t *pos) +{ + v = l2t_get_idx(seq, *pos + 1); + if (v) + ++*pos; + return v; +} + +static void +l2t_seq_stop(struct seq_file *seq, void *v) +{ +} + +static char +l2e_state(const struct l2t_entry *e) +{ + switch (e->state) { + case L2T_STATE_VALID: return 'V'; /* valid, fast-path entry */ + case L2T_STATE_STALE: return 'S'; /* needs revalidation, but usable */ + case L2T_STATE_RESOLVING: + return e->arpq_head ? 'A' : 'R'; + default: + return 'U'; + } +} + +static int +l2t_seq_show(struct seq_file *seq, void *v) +{ + if (v == SEQ_START_TOKEN) + seq_puts(seq, "Index IP address Ethernet address VLAN " + "Prio State Users SMTIDX Port\n"); + else { + char ip[20]; + struct l2t_entry *e = v; + + mtx_lock(&e->lock); + sprintf(ip, "%u.%u.%u.%u", NIPQUAD(e->addr)); + seq_printf(seq, "%-5u %-15s %02x:%02x:%02x:%02x:%02x:%02x %4d" + " %3u %c %7u %4u %s\n", + e->idx, ip, e->dmac[0], e->dmac[1], e->dmac[2], + e->dmac[3], e->dmac[4], e->dmac[5], + e->vlan & EVL_VLID_MASK, vlan_prio(e), + l2e_state(e), atomic_load_acq_int(&e->refcnt), e->smt_idx, + e->neigh ? e->neigh->dev->name : ""); + mtx_unlock(&e->lock); + } + return 0; +} + +#endif Property changes on: head/sys/dev/cxgb/cxgb_l2t.c ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/dev/cxgb/cxgb_l2t.h =================================================================== --- head/sys/dev/cxgb/cxgb_l2t.h (nonexistent) +++ head/sys/dev/cxgb/cxgb_l2t.h (revision 169978) @@ -0,0 +1,154 @@ +/************************************************************************** + +Copyright (c) 2007, Chelsio Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Neither the name of the Chelsio Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +$FreeBSD$ + +***************************************************************************/ +#ifndef _CHELSIO_L2T_H +#define _CHELSIO_L2T_H + +#include +#include +#include + +enum { + L2T_STATE_VALID, /* entry is up to date */ + L2T_STATE_STALE, /* entry may be used but needs revalidation */ + L2T_STATE_RESOLVING, /* entry needs address resolution */ + L2T_STATE_UNUSED /* entry not in use */ +}; + +/* + * Each L2T entry plays multiple roles. First of all, it keeps state for the + * corresponding entry of the HW L2 table and maintains a queue of offload + * packets awaiting address resolution. Second, it is a node of a hash table + * chain, where the nodes of the chain are linked together through their next + * pointer. Finally, each node is a bucket of a hash table, pointing to the + * first element in its chain through its first pointer. + */ +struct l2t_entry { + uint16_t state; /* entry state */ + uint16_t idx; /* entry index */ + uint32_t addr; /* dest IP address */ + int ifindex; /* neighbor's net_device's ifindex */ + uint16_t smt_idx; /* SMT index */ + uint16_t vlan; /* VLAN TCI (id: bits 0-11, prio: 13-15 */ + struct rtentry *neigh; /* associated neighbour */ + struct l2t_entry *first; /* start of hash chain */ + struct l2t_entry *next; /* next l2t_entry on chain */ + struct mbuf *arpq_head; /* queue of packets awaiting resolution */ + struct mbuf *arpq_tail; + struct mtx lock; + volatile uint32_t refcnt; /* entry reference count */ + uint8_t dmac[6]; /* neighbour's MAC address */ +#ifndef NETEVENT +#ifdef CONFIG_CHELSIO_T3_MODULE + struct timer_list update_timer; + struct toedev *tdev; +#endif +#endif +}; + +struct l2t_data { + unsigned int nentries; /* number of entries */ + struct l2t_entry *rover; /* starting point for next allocation */ + volatile uint32_t nfree; /* number of free entries */ + struct rwlock lock; + struct l2t_entry l2tab[0]; +}; + +typedef void (*arp_failure_handler_func)(struct toedev *dev, + struct mbuf *m); + +/* + * Callback stored in an skb to handle address resolution failure. + */ +struct l2t_mbuf_cb { + arp_failure_handler_func arp_failure_handler; +}; + +/* + * XXX + */ +#define L2T_MBUF_CB(skb) ((struct l2t_mbuf_cb *)(skb)->cb) + + +static __inline void set_arp_failure_handler(struct mbuf *m, + arp_failure_handler_func hnd) +{ +#if 0 + L2T_SKB_CB(skb)->arp_failure_handler = hnd; +#endif + panic("implement me"); +} + +/* + * Getting to the L2 data from an offload device. + */ +#define L2DATA(dev) ((dev)->l2opt) + +void t3_l2e_free(struct l2t_data *d, struct l2t_entry *e); +void t3_l2t_update(struct toedev *dev, struct rtentry *ifp); +struct l2t_entry *t3_l2t_get(struct toedev *dev, struct rtentry *neigh, + unsigned int smt_idx); +int t3_l2t_send_slow(struct toedev *dev, struct mbuf *m, + struct l2t_entry *e); +void t3_l2t_send_event(struct toedev *dev, struct l2t_entry *e); +struct l2t_data *t3_init_l2t(unsigned int l2t_capacity); +void t3_free_l2t(struct l2t_data *d); + +#ifdef CONFIG_PROC_FS +int t3_l2t_proc_setup(struct proc_dir_entry *dir, struct l2t_data *d); +void t3_l2t_proc_free(struct proc_dir_entry *dir); +#else +#define l2t_proc_setup(dir, d) 0 +#define l2t_proc_free(dir) +#endif + +int cxgb_ofld_send(struct toedev *dev, struct mbuf *m); + +static inline int l2t_send(struct toedev *dev, struct mbuf *m, + struct l2t_entry *e) +{ + if (__predict_true(e->state == L2T_STATE_VALID)) + return cxgb_ofld_send(dev, m); + return t3_l2t_send_slow(dev, m, e); +} + +static inline void l2t_release(struct l2t_data *d, struct l2t_entry *e) +{ + if (atomic_fetchadd_int(&e->refcnt, -1) == 1) + t3_l2e_free(d, e); +} + +static inline void l2t_hold(struct l2t_data *d, struct l2t_entry *e) +{ + if (atomic_fetchadd_int(&e->refcnt, 1) == 1) /* 0 -> 1 transition */ + atomic_add_int(&d->nfree, 1); +} + +#endif Property changes on: head/sys/dev/cxgb/cxgb_l2t.h ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/dev/cxgb/cxgb_lro.c =================================================================== --- head/sys/dev/cxgb/cxgb_lro.c (revision 169977) +++ head/sys/dev/cxgb/cxgb_lro.c (revision 169978) @@ -1,418 +1,397 @@ /************************************************************************** Copyright (c) 2007, Chelsio Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Chelsio Corporation nor the names of its +2. Neither the name of the Chelsio Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ***************************************************************************/ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef M_LRO #define M_LRO 0x0200 #endif #ifdef DEBUG #define MBUF_HEADER_CHECK(m) do { \ if ((m->m_len == 0) || (m->m_pkthdr.len == 0) \ || ((m->m_flags & M_PKTHDR) == 0)) \ panic("lro_flush_session - mbuf len=%d pktlen=%d flags=0x%x\n", \ m->m_len, m->m_pkthdr.len, m->m_flags); \ if ((m->m_flags & M_PKTHDR) == 0) \ panic("first mbuf is not packet header - flags=0x%x\n", \ m->m_flags); \ if ((m->m_len < ETHER_HDR_LEN) || (m->m_pkthdr.len < ETHER_HDR_LEN)) \ panic("packet too small len=%d pktlen=%d\n", \ m->m_len, m->m_pkthdr.len);\ } while (0) #else #define MBUF_HEADER_CHECK(m) #endif #define IPH_OFFSET (2 + sizeof (struct cpl_rx_pkt) + ETHER_HDR_LEN) -#define LRO_SESSION_IDX_HINT_HASH(hash) (hash & (MAX_LRO_PER_QSET - 1)) -#define LRO_IDX_INC(idx) idx = (idx + 1) & (MAX_LRO_PER_QSET - 1) +#define LRO_SESSION_IDX_HINT_HASH(hash) (hash & (MAX_LRO_SES - 1)) +#define LRO_IDX_INC(idx) idx = (idx + 1) & (MAX_LRO_SES - 1) -static __inline struct sge_lro_session * -lro_session(struct sge_lro *l, int idx) -{ - return l->s + idx; -} - static __inline int -lro_match_session(struct sge_lro_session *s, - struct ip *ih, struct tcphdr *th) +lro_match(struct mbuf *m, struct ip *ih, struct tcphdr *th) { - struct ip *sih = (struct ip *)(s->m->m_data + IPH_OFFSET); + struct ip *sih = (struct ip *)(m->m_data + IPH_OFFSET); struct tcphdr *sth = (struct tcphdr *) (sih + 1); /* - * Linux driver doesn't include destination port check -- - * need to find out why XXX + * Why don't we check dest ports? */ return (*(uint32_t *)&th->th_sport == *(uint32_t *)&sth->th_sport && - *(uint32_t *)&th->th_dport == *(uint32_t *)&sth->th_dport && ih->ip_src.s_addr == ih->ip_src.s_addr && ih->ip_dst.s_addr == sih->ip_dst.s_addr); } -static __inline struct sge_lro_session * -lro_find_session(struct sge_lro *l, int idx, struct ip *ih, struct tcphdr *th) +static __inline struct t3_lro_session * +lro_lookup(struct lro_state *l, int idx, struct ip *ih, struct tcphdr *th) { - struct sge_lro_session *s; - int active = 0; + struct t3_lro_session *s = NULL; + int active = l->nactive; - while (active < l->num_active) { - s = lro_session(l, idx); - if (s->m) { - if (lro_match_session(s, ih, th)) { - l->last_s = s; - return s; - } - active++; + while (active) { + s = &l->sess[idx]; + if (s->head) { + if (lro_match(s->head, ih, th)) + break; + active--; } LRO_IDX_INC(idx); } - return NULL; + return (s); } static __inline int can_lro_packet(struct cpl_rx_pkt *cpl, unsigned int rss_hi) { struct ether_header *eh = (struct ether_header *)(cpl + 1); struct ip *ih = (struct ip *)(eh + 1); /* * XXX VLAN support? */ if (__predict_false(G_HASHTYPE(ntohl(rss_hi)) != RSS_HASH_4_TUPLE || (*((uint8_t *)cpl + 1) & 0x90) != 0x10 || cpl->csum != 0xffff || eh->ether_type != ntohs(ETHERTYPE_IP) || ih->ip_hl != (sizeof (*ih) >> 2))) { return 0; } return 1; } static int can_lro_tcpsegment(struct tcphdr *th) { int olen = (th->th_off << 2) - sizeof (*th); u8 control_bits = *((u8 *)th + 13); if (__predict_false((control_bits & 0xB7) != 0x10)) goto no_lro; if (olen) { uint32_t *ptr = (u32 *)(th + 1); if (__predict_false(olen != TCPOLEN_TSTAMP_APPA || *ptr != ntohl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP))) goto no_lro; } return 1; no_lro: return 0; } static __inline void -lro_new_session_init(struct sge_lro_session *s, struct mbuf *m) +lro_new_session_init(struct t3_lro_session *s, struct mbuf *m) { struct ip *ih = (struct ip *)(m->m_data + IPH_OFFSET); struct tcphdr *th = (struct tcphdr *) (ih + 1); int ip_len = ntohs(ih->ip_len); DPRINTF("%s(s=%p, m=%p)\n", __FUNCTION__, s, m); - s->m = m; + s->head = m; MBUF_HEADER_CHECK(m); s->ip_len = ip_len; s->seq = ntohl(th->th_seq) + ip_len - sizeof(*ih) - (th->th_off << 2); } static void -lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct mbuf *m) +lro_flush_session(struct sge_qset *qs, struct t3_lro_session *s, struct mbuf *m) { - struct sge_lro *l = &qs->lro; - struct mbuf *sm = s->m; + struct lro_state *l = &qs->lro; + struct mbuf *sm = s->head; struct ip *ih = (struct ip *)(sm->m_data + IPH_OFFSET); DPRINTF("%s(qs=%p, s=%p, ", __FUNCTION__, qs, s); if (m) DPRINTF("m=%p)\n", m); else DPRINTF("m=NULL)\n"); ih->ip_len = htons(s->ip_len); ih->ip_sum = 0; ih->ip_sum = in_cksum_hdr(ih); MBUF_HEADER_CHECK(sm); sm->m_flags |= M_LRO; t3_rx_eth(qs->port, &qs->rspq, sm, 2); if (m) { - s->m = m; + s->head = m; lro_new_session_init(s, m); } else { - s->m = NULL; - l->num_active--; + s->head = NULL; + l->nactive--; } qs->port_stats[SGE_PSTATS_LRO_FLUSHED]++; } -static __inline struct sge_lro_session * +static __inline struct t3_lro_session * lro_new_session(struct sge_qset *qs, struct mbuf *m, uint32_t rss_hash) { - struct sge_lro *l = &qs->lro; + struct lro_state *l = &qs->lro; int idx = LRO_SESSION_IDX_HINT_HASH(rss_hash); - struct sge_lro_session *s = lro_session(l, idx); + struct t3_lro_session *s = &l->sess[idx]; DPRINTF("%s(qs=%p, m=%p, rss_hash=0x%x)\n", __FUNCTION__, qs, m, rss_hash); - if (__predict_true(!s->m)) + if (__predict_true(!s->head)) goto done; - if (l->num_active > MAX_LRO_PER_QSET) + if (l->nactive > MAX_LRO_SES) panic("MAX_LRO_PER_QSET exceeded"); - if (l->num_active == MAX_LRO_PER_QSET) { + if (l->nactive == MAX_LRO_SES) { lro_flush_session(qs, s, m); qs->port_stats[SGE_PSTATS_LRO_X_STREAMS]++; return s; } while (1) { LRO_IDX_INC(idx); - s = lro_session(l, idx); - if (!s->m) + s = &l->sess[idx]; + if (!s->head) break; } done: lro_new_session_init(s, m); - l->num_active++; + l->nactive++; return s; } static __inline int -lro_update_session(struct sge_lro_session *s, struct mbuf *m) +lro_update_session(struct t3_lro_session *s, struct mbuf *m) { - struct mbuf *sm = s->m; + struct mbuf *sm = s->head; struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(sm->m_data + 2); struct cpl_rx_pkt *ncpl = (struct cpl_rx_pkt *)(m->m_data + 2); struct ip *nih = (struct ip *)(m->m_data + IPH_OFFSET); struct tcphdr *th, *nth = (struct tcphdr *)(nih + 1); uint32_t seq = ntohl(nth->th_seq); int plen, tcpiphlen, olen = (nth->th_off << 2) - sizeof (*nth); DPRINTF("%s(s=%p, m=%p)\n", __FUNCTION__, s, m); if (cpl->vlan_valid && cpl->vlan != ncpl->vlan) { return -1; } if (__predict_false(seq != s->seq)) { DPRINTF("sequence mismatch\n"); return -1; } MBUF_HEADER_CHECK(sm); th = (struct tcphdr *)(sm->m_data + IPH_OFFSET + sizeof (struct ip)); if (olen) { uint32_t *ptr = (uint32_t *)(th + 1); uint32_t *nptr = (uint32_t *)(nth + 1); if (__predict_false(ntohl(*(ptr + 1)) > ntohl(*(nptr + 1)) || !*(nptr + 2))) { return -1; } *(ptr + 1) = *(nptr + 1); *(ptr + 2) = *(nptr + 2); } th->th_ack = nth->th_ack; th->th_win = nth->th_win; tcpiphlen = (nth->th_off << 2) + sizeof (*nih); plen = ntohs(nih->ip_len) - tcpiphlen; s->seq += plen; s->ip_len += plen; sm->m_pkthdr.len += plen; /* * XXX FIX ME * * */ #if 0 /* XXX this I *do not* understand */ if (plen > skb_shinfo(s->skb)->gso_size) skb_shinfo(s->skb)->gso_size = plen; #endif #if __FreeBSD_version > 700000 if (plen > sm->m_pkthdr.tso_segsz) sm->m_pkthdr.tso_segsz = plen; #endif DPRINTF("m_adj(%d)\n", (int)(IPH_OFFSET + tcpiphlen)); m_adj(m, IPH_OFFSET + tcpiphlen); #if 0 if (__predict_false(!skb_shinfo(s->skb)->frag_list)) skb_shinfo(s->skb)->frag_list = skb; #endif #if 0 /* * XXX we really need to be able to * support vectors of buffers in FreeBSD */ int nr = skb_shinfo(s->skb)->nr_frags; skb_shinfo(s->skb)->frags[nr].page = frag->page; skb_shinfo(s->skb)->frags[nr].page_offset = frag->page_offset + IPH_OFFSET + tcpiphlen; skb_shinfo(s->skb)->frags[nr].size = plen; skb_shinfo(s->skb)->nr_frags = ++nr; #endif return (0); } void t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad, uint32_t rss_hash, uint32_t rss_csum, int lro) { struct sge_qset *qs = rspq_to_qset(rq); struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(m->m_data + ethpad); struct ether_header *eh = (struct ether_header *)(cpl + 1); struct ip *ih; struct tcphdr *th; - struct sge_lro_session *s = NULL; + struct t3_lro_session *s = NULL; struct port_info *pi = qs->port; if (lro == 0) goto no_lro; if (!can_lro_packet(cpl, rss_csum)) goto no_lro; if (&adap->port[cpl->iff] != pi) panic("bad port index %d\n", cpl->iff); ih = (struct ip *)(eh + 1); th = (struct tcphdr *)(ih + 1); - s = lro_find_session(&qs->lro, + s = lro_lookup(&qs->lro, LRO_SESSION_IDX_HINT_HASH(rss_hash), ih, th); if (__predict_false(!can_lro_tcpsegment(th))) { goto no_lro; } else if (__predict_false(!s)) { s = lro_new_session(qs, m, rss_hash); } else { if (lro_update_session(s, m)) { lro_flush_session(qs, s, m); } - if (__predict_false(s->m->m_pkthdr.len + pi->ifp->if_mtu > 65535)) { + if (__predict_false(s->head->m_pkthdr.len + pi->ifp->if_mtu > 65535)) { lro_flush_session(qs, s, NULL); } } qs->port_stats[SGE_PSTATS_LRO_QUEUED]++; return; no_lro: if (s) lro_flush_session(qs, s, NULL); if (m->m_len == 0 || m->m_pkthdr.len == 0 || (m->m_flags & M_PKTHDR) == 0) DPRINTF("rx_eth_lro mbuf len=%d pktlen=%d flags=0x%x\n", m->m_len, m->m_pkthdr.len, m->m_flags); t3_rx_eth(pi, rq, m, ethpad); } void -t3_sge_lro_flush_all(adapter_t *adap, struct sge_qset *qs) +t3_lro_flush(adapter_t *adap, struct sge_qset *qs, struct lro_state *state) { - struct sge_lro *l = &qs->lro; - struct sge_lro_session *s = l->last_s; - int active = 0, idx = 0, num_active = l->num_active; + unsigned int idx = state->active_idx; - if (__predict_false(!s)) - s = lro_session(l, idx); - - while (active < num_active) { - if (s->m) { + while (state->nactive) { + struct t3_lro_session *s = &state->sess[idx]; + + if (s->head) lro_flush_session(qs, s, NULL); - active++; - } LRO_IDX_INC(idx); - s = lro_session(l, idx); } } Index: head/sys/dev/cxgb/cxgb_main.c =================================================================== --- head/sys/dev/cxgb/cxgb_main.c (revision 169977) +++ head/sys/dev/cxgb/cxgb_main.c (revision 169978) @@ -1,1920 +1,2232 @@ /************************************************************************** Copyright (c) 2007, Chelsio Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Chelsio Corporation nor the names of its +2. Neither the name of the Chelsio Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ***************************************************************************/ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include #include #include #include #include #ifdef PRIV_SUPPORTED #include #endif static int cxgb_setup_msix(adapter_t *, int); static void cxgb_init(void *); static void cxgb_init_locked(struct port_info *); static void cxgb_stop_locked(struct port_info *); static void cxgb_set_rxmode(struct port_info *); static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t); static void cxgb_start(struct ifnet *); static void cxgb_start_proc(void *, int ncount); static int cxgb_media_change(struct ifnet *); static void cxgb_media_status(struct ifnet *, struct ifmediareq *); static int setup_sge_qsets(adapter_t *); static void cxgb_async_intr(void *); static void cxgb_ext_intr_handler(void *, int); +static void cxgb_down(struct adapter *sc); static void cxgb_tick(void *); static void setup_rss(adapter_t *sc); /* Attachment glue for the PCI controller end of the device. Each port of * the device is attached separately, as defined later. */ static int cxgb_controller_probe(device_t); static int cxgb_controller_attach(device_t); static int cxgb_controller_detach(device_t); static void cxgb_free(struct adapter *); static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, unsigned int end); static void cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf); static int cxgb_get_regs_len(void); +static int offload_open(struct port_info *pi); +static int offload_close(struct toedev *tdev); + + static device_method_t cxgb_controller_methods[] = { DEVMETHOD(device_probe, cxgb_controller_probe), DEVMETHOD(device_attach, cxgb_controller_attach), DEVMETHOD(device_detach, cxgb_controller_detach), /* bus interface */ DEVMETHOD(bus_print_child, bus_generic_print_child), DEVMETHOD(bus_driver_added, bus_generic_driver_added), { 0, 0 } }; static driver_t cxgb_controller_driver = { "cxgbc", cxgb_controller_methods, sizeof(struct adapter) }; static devclass_t cxgb_controller_devclass; DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0); /* * Attachment glue for the ports. Attachment is done directly to the * controller device. */ static int cxgb_port_probe(device_t); static int cxgb_port_attach(device_t); static int cxgb_port_detach(device_t); static device_method_t cxgb_port_methods[] = { DEVMETHOD(device_probe, cxgb_port_probe), DEVMETHOD(device_attach, cxgb_port_attach), DEVMETHOD(device_detach, cxgb_port_detach), { 0, 0 } }; static driver_t cxgb_port_driver = { "cxgb", cxgb_port_methods, 0 }; static d_ioctl_t cxgb_extension_ioctl; static devclass_t cxgb_port_devclass; DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0); #define SGE_MSIX_COUNT (SGE_QSETS + 1) extern int collapse_mbufs; /* * The driver uses the best interrupt scheme available on a platform in the * order MSI-X, MSI, legacy pin interrupts. This parameter determines which * of these schemes the driver may consider as follows: * * msi = 2: choose from among all three options * msi = 1 : only consider MSI and pin interrupts * msi = 0: force pin interrupts */ static int msi_allowed = 2; TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed); - SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters"); SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0, "MSI-X, MSI, INTx selector"); + /* - * Multiple queues need further tuning + * The driver enables offload as a default. + * To disable it, use ofld_disable = 1. */ +static int ofld_disable = 0; +TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable); +SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0, + "disable ULP offload"); + +/* + * The driver uses an auto-queue algorithm by default. + * To disable it and force a single queue-set per port, use singleq = 1. + */ static int singleq = 1; +TUNABLE_INT("hw.cxgb.singleq", &singleq); +SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0, + "use a single queue-set per port"); enum { MAX_TXQ_ENTRIES = 16384, MAX_CTRL_TXQ_ENTRIES = 1024, MAX_RSPQ_ENTRIES = 16384, MAX_RX_BUFFERS = 16384, MAX_RX_JUMBO_BUFFERS = 16384, MIN_TXQ_ENTRIES = 4, MIN_CTRL_TXQ_ENTRIES = 4, MIN_RSPQ_ENTRIES = 32, MIN_FL_ENTRIES = 32 }; #define PORT_MASK ((1 << MAX_NPORTS) - 1) /* Table for probing the cards. The desc field isn't actually used */ struct cxgb_ident { uint16_t vendor; uint16_t device; int index; char *desc; } cxgb_identifiers[] = { {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"}, {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"}, {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"}, {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"}, {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"}, {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"}, {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"}, {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"}, {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"}, {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"}, {0, 0, 0, NULL} }; static struct cxgb_ident * cxgb_get_ident(device_t dev) { struct cxgb_ident *id; for (id = cxgb_identifiers; id->desc != NULL; id++) { if ((id->vendor == pci_get_vendor(dev)) && (id->device == pci_get_device(dev))) { return (id); } } return (NULL); } static const struct adapter_info * cxgb_get_adapter_info(device_t dev) { struct cxgb_ident *id; const struct adapter_info *ai; id = cxgb_get_ident(dev); if (id == NULL) return (NULL); ai = t3_get_adapter_info(id->index); return (ai); } static int cxgb_controller_probe(device_t dev) { const struct adapter_info *ai; char *ports, buf[80]; ai = cxgb_get_adapter_info(dev); if (ai == NULL) return (ENXIO); if (ai->nports == 1) ports = "port"; else ports = "ports"; snprintf(buf, sizeof(buf), "%s RNIC, %d %s", ai->desc, ai->nports, ports); device_set_desc_copy(dev, buf); return (BUS_PROBE_DEFAULT); } static int -cxgb_fw_download(adapter_t *sc, device_t dev) +upgrade_fw(adapter_t *sc) { char buf[32]; #ifdef FIRMWARE_LATEST const struct firmware *fw; #else struct firmware *fw; #endif int status; - snprintf(&buf[0], sizeof(buf), "t3fw%d%d", FW_VERSION_MAJOR, - FW_VERSION_MINOR); + snprintf(&buf[0], sizeof(buf), "t3fw%d%d%d", FW_VERSION_MAJOR, + FW_VERSION_MINOR, FW_VERSION_MICRO); fw = firmware_get(buf); - if (fw == NULL) { - device_printf(dev, "Could not find firmware image %s\n", buf); - return ENOENT; + device_printf(sc->dev, "Could not find firmware image %s\n", buf); + return (ENOENT); } status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize); firmware_put(fw, FIRMWARE_UNLOAD); return (status); } - static int cxgb_controller_attach(device_t dev) { driver_intr_t *cxgb_intr = NULL; device_t child; const struct adapter_info *ai; struct adapter *sc; - int i, reg, msi_needed, msi_count = 0, error = 0; + int i, reg, msi_needed, error = 0; uint32_t vers; int port_qsets = 1; sc = device_get_softc(dev); sc->dev = dev; - + sc->msi_count = 0; + /* find the PCIe link width and set max read request to 4KB*/ if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) { uint16_t lnk, pectl; lnk = pci_read_config(dev, reg + 0x12, 2); sc->link_width = (lnk >> 4) & 0x3f; pectl = pci_read_config(dev, reg + 0x8, 2); pectl = (pectl & ~0x7000) | (5 << 12); pci_write_config(dev, reg + 0x8, pectl, 2); } if (sc->link_width != 0 && sc->link_width <= 4) { device_printf(sc->dev, "PCIe x%d Link, expect reduced performance\n", sc->link_width); } pci_enable_busmaster(dev); /* * Allocate the registers and make them available to the driver. * The registers that we care about for NIC mode are in BAR 0 */ sc->regs_rid = PCIR_BAR(0); if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->regs_rid, RF_ACTIVE)) == NULL) { device_printf(dev, "Cannot allocate BAR\n"); return (ENXIO); } mtx_init(&sc->sge.reg_lock, "SGE reg lock", NULL, MTX_DEF); mtx_init(&sc->lock, "cxgb controller lock", NULL, MTX_DEF); mtx_init(&sc->mdio_lock, "cxgb mdio", NULL, MTX_DEF); sc->bt = rman_get_bustag(sc->regs_res); sc->bh = rman_get_bushandle(sc->regs_res); sc->mmio_len = rman_get_size(sc->regs_res); ai = cxgb_get_adapter_info(dev); if (t3_prep_adapter(sc, ai, 1) < 0) { error = ENODEV; goto out; } /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate * enough messages for the queue sets. If that fails, try falling * back to MSI. If that fails, then try falling back to the legacy * interrupt pin model. */ #ifdef MSI_SUPPORTED sc->msix_regs_rid = 0x20; if ((msi_allowed >= 2) && (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->msix_regs_rid, RF_ACTIVE)) != NULL) { - msi_needed = msi_count = SGE_MSIX_COUNT; + msi_needed = sc->msi_count = SGE_MSIX_COUNT; - if ((pci_alloc_msix(dev, &msi_count) != 0) || - (msi_count != msi_needed)) { - device_printf(dev, "msix allocation failed" - " will try msi\n"); - msi_count = 0; + if (((error = pci_alloc_msix(dev, &sc->msi_count)) != 0) || + (sc->msi_count != msi_needed)) { + device_printf(dev, "msix allocation failed - msi_count = %d" + " msi_needed=%d will try msi err=%d\n", sc->msi_count, + msi_needed, error); + sc->msi_count = 0; pci_release_msi(dev); bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_regs_rid, sc->msix_regs_res); sc->msix_regs_res = NULL; } else { sc->flags |= USING_MSIX; cxgb_intr = t3_intr_msix; } } - if ((msi_allowed >= 1) && (msi_count == 0)) { - msi_count = 1; - if (pci_alloc_msi(dev, &msi_count)) { + if ((msi_allowed >= 1) && (sc->msi_count == 0)) { + sc->msi_count = 1; + if (pci_alloc_msi(dev, &sc->msi_count)) { device_printf(dev, "alloc msi failed - will try INTx\n"); - msi_count = 0; + sc->msi_count = 0; pci_release_msi(dev); } else { sc->flags |= USING_MSI; sc->irq_rid = 1; cxgb_intr = t3_intr_msi; } } #endif - if (msi_count == 0) { + if (sc->msi_count == 0) { device_printf(dev, "using line interrupts\n"); sc->irq_rid = 0; cxgb_intr = t3b_intr; } /* Create a private taskqueue thread for handling driver events */ #ifdef TASKQUEUE_CURRENT sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT, taskqueue_thread_enqueue, &sc->tq); #else sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT, taskqueue_thread_enqueue, &sc->tq); #endif if (sc->tq == NULL) { device_printf(dev, "failed to allocate controller task queue\n"); goto out; } taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", device_get_nameunit(dev)); TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc); /* Create a periodic callout for checking adapter status */ callout_init_mtx(&sc->cxgb_tick_ch, &sc->lock, CALLOUT_RETURNUNLOCKED); if (t3_check_fw_version(sc) != 0) { /* * Warn user that a firmware update will be attempted in init. */ - device_printf(dev, "firmware needs to be updated to version %d.%d\n", - FW_VERSION_MAJOR, FW_VERSION_MINOR); + device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n", + FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO); sc->flags &= ~FW_UPTODATE; } else { sc->flags |= FW_UPTODATE; } - if (t3_init_hw(sc, 0) != 0) { - device_printf(dev, "hw initialization failed\n"); - error = ENXIO; - goto out; - } - t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); - - - if ((singleq == 0) && (sc->flags & USING_MSIX)) + if ((sc->flags & USING_MSIX) && !singleq) port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus); /* * Create a child device for each MAC. The ethernet attachment * will be done in these children. */ for (i = 0; i < (sc)->params.nports; i++) { if ((child = device_add_child(dev, "cxgb", -1)) == NULL) { device_printf(dev, "failed to add child port\n"); error = EINVAL; goto out; } sc->portdev[i] = child; sc->port[i].adapter = sc; sc->port[i].nqsets = port_qsets; sc->port[i].first_qset = i*port_qsets; sc->port[i].port = i; device_set_softc(child, &sc->port[i]); } if ((error = bus_generic_attach(dev)) != 0) - goto out;; - - if ((error = setup_sge_qsets(sc)) != 0) goto out; - - setup_rss(sc); - - /* If it's MSI or INTx, allocate a single interrupt for everything */ - if ((sc->flags & USING_MSIX) == 0) { - if ((sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, - &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { - device_printf(dev, "Cannot allocate interrupt rid=%d\n", sc->irq_rid); - error = EINVAL; - goto out; - } - device_printf(dev, "allocated irq_res=%p\n", sc->irq_res); - if (bus_setup_intr(dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, -#ifdef INTR_FILTERS - NULL, -#endif - cxgb_intr, sc, &sc->intr_tag)) { - device_printf(dev, "Cannot set up interrupt\n"); - error = EINVAL; - goto out; - } - } else { - cxgb_setup_msix(sc, msi_count); - } - + /* + * XXX need to poll for link status + */ sc->params.stats_update_period = 1; /* initialize sge private state */ t3_sge_init_sw(sc); t3_led_ready(sc); - + + cxgb_offload_init(); + if (is_offload(sc)) { + setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT); + cxgb_adapter_ofld(sc); + } error = t3_get_fw_version(sc, &vers); if (error) goto out; - - snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d", G_FW_VERSION_MAJOR(vers), - G_FW_VERSION_MINOR(vers)); + snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", + G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), + G_FW_VERSION_MICRO(vers)); + t3_add_sysctls(sc); - out: if (error) cxgb_free(sc); return (error); } static int cxgb_controller_detach(device_t dev) { struct adapter *sc; sc = device_get_softc(dev); cxgb_free(sc); return (0); } static void cxgb_free(struct adapter *sc) { int i; + cxgb_down(sc); + +#ifdef MSI_SUPPORTED + if (sc->flags & (USING_MSI | USING_MSIX)) { + device_printf(sc->dev, "releasing msi message(s)\n"); + pci_release_msi(sc->dev); + } else { + device_printf(sc->dev, "no msi message to release\n"); + } +#endif + if (sc->msix_regs_res != NULL) { + bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, + sc->msix_regs_res); + } + /* * XXX need to drain the ifq by hand until * it is taught about mbuf iovecs */ - callout_drain(&sc->cxgb_tick_ch); t3_sge_deinit_sw(sc); if (sc->tq != NULL) { taskqueue_drain(sc->tq, &sc->ext_intr_task); taskqueue_free(sc->tq); } for (i = 0; i < (sc)->params.nports; ++i) { if (sc->portdev[i] != NULL) device_delete_child(sc->dev, sc->portdev[i]); } bus_generic_detach(sc->dev); + if (is_offload(sc)) { + cxgb_adapter_unofld(sc); + if (isset(&sc->open_device_map, OFFLOAD_DEVMAP_BIT)) + offload_close(&sc->tdev); + } t3_free_sge_resources(sc); t3_sge_free(sc); - for (i = 0; i < SGE_QSETS; i++) { - if (sc->msix_intr_tag[i] != NULL) { - bus_teardown_intr(sc->dev, sc->msix_irq_res[i], - sc->msix_intr_tag[i]); - } - if (sc->msix_irq_res[i] != NULL) { - bus_release_resource(sc->dev, SYS_RES_IRQ, - sc->msix_irq_rid[i], sc->msix_irq_res[i]); - } - } - - if (sc->intr_tag != NULL) { - bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); - } - - if (sc->irq_res != NULL) { - device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n", - sc->irq_rid, sc->irq_res); - bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, - sc->irq_res); - } -#ifdef MSI_SUPPORTED - if (sc->flags & (USING_MSI | USING_MSIX)) { - device_printf(sc->dev, "releasing msi message(s)\n"); - pci_release_msi(sc->dev); - } -#endif - if (sc->msix_regs_res != NULL) { - bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, - sc->msix_regs_res); - } - if (sc->regs_res != NULL) bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid, sc->regs_res); mtx_destroy(&sc->mdio_lock); mtx_destroy(&sc->sge.reg_lock); mtx_destroy(&sc->lock); return; } /** * setup_sge_qsets - configure SGE Tx/Rx/response queues * @sc: the controller softc * * Determines how many sets of SGE queues to use and initializes them. * We support multiple queue sets per port if we have MSI-X, otherwise * just one queue set per port. */ static int setup_sge_qsets(adapter_t *sc) { int i, j, err, irq_idx, qset_idx; - u_int ntxq = 3; + u_int ntxq = SGE_TXQ_PER_SET; if ((err = t3_sge_alloc(sc)) != 0) { device_printf(sc->dev, "t3_sge_alloc returned %d\n", err); return (err); } if (sc->params.rev > 0 && !(sc->flags & USING_MSI)) irq_idx = -1; else irq_idx = 0; for (qset_idx = 0, i = 0; i < (sc)->params.nports; ++i) { struct port_info *pi = &sc->port[i]; for (j = 0; j < pi->nqsets; ++j, ++qset_idx) { err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports, (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx, &sc->params.sge.qset[qset_idx], ntxq, pi); if (err) { t3_free_sge_resources(sc); device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n", err); return (err); } } } return (0); } static int cxgb_setup_msix(adapter_t *sc, int msix_count) { int i, j, k, nqsets, rid; /* The first message indicates link changes and error conditions */ sc->irq_rid = 1; if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { device_printf(sc->dev, "Cannot allocate msix interrupt\n"); return (EINVAL); } if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, #ifdef INTR_FILTERS NULL, #endif cxgb_async_intr, sc, &sc->intr_tag)) { device_printf(sc->dev, "Cannot set up interrupt\n"); return (EINVAL); } for (i = 0, k = 0; i < (sc)->params.nports; ++i) { nqsets = sc->port[i].nqsets; for (j = 0; j < nqsets; ++j, k++) { struct sge_qset *qs = &sc->sge.qs[k]; rid = k + 2; if (cxgb_debug) printf("rid=%d ", rid); if ((sc->msix_irq_res[k] = bus_alloc_resource_any( sc->dev, SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { device_printf(sc->dev, "Cannot allocate " "interrupt for message %d\n", rid); return (EINVAL); } sc->msix_irq_rid[k] = rid; if (bus_setup_intr(sc->dev, sc->msix_irq_res[j], INTR_MPSAFE|INTR_TYPE_NET, #ifdef INTR_FILTERS NULL, #endif t3_intr_msix, qs, &sc->msix_intr_tag[k])) { device_printf(sc->dev, "Cannot set up " "interrupt for message %d\n", rid); return (EINVAL); } } } return (0); } static int cxgb_port_probe(device_t dev) { struct port_info *p; char buf[80]; p = device_get_softc(dev); snprintf(buf, sizeof(buf), "Port %d %s", p->port, p->port_type->desc); device_set_desc_copy(dev, buf); return (0); } static int cxgb_makedev(struct port_info *pi) { struct cdevsw *cxgb_cdevsw; if ((cxgb_cdevsw = malloc(sizeof(struct cdevsw), M_DEVBUF, M_NOWAIT|M_ZERO)) == NULL) return (ENOMEM); cxgb_cdevsw->d_version = D_VERSION; cxgb_cdevsw->d_name = strdup(pi->ifp->if_xname, M_DEVBUF); cxgb_cdevsw->d_ioctl = cxgb_extension_ioctl; pi->port_cdev = make_dev(cxgb_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, pi->ifp->if_xname); if (pi->port_cdev == NULL) return (ENOMEM); pi->port_cdev->si_drv1 = (void *)pi; return (0); } #ifdef TSO_SUPPORTED #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU) /* Don't enable TSO6 yet */ #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU) #else #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) /* Don't enable TSO6 yet */ #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) #define IFCAP_TSO4 0x0 #define CSUM_TSO 0x0 #endif static int cxgb_port_attach(device_t dev) { struct port_info *p; struct ifnet *ifp; int media_flags; int err; char buf[64]; p = device_get_softc(dev); snprintf(buf, sizeof(buf), "cxgb port %d", p->port); mtx_init(&p->lock, buf, 0, MTX_DEF); /* Allocate an ifnet object and set it up */ ifp = p->ifp = if_alloc(IFT_ETHER); if (ifp == NULL) { device_printf(dev, "Cannot allocate ifnet\n"); return (ENOMEM); } /* * Note that there is currently no watchdog timer. */ if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_init = cxgb_init; ifp->if_softc = p; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = cxgb_ioctl; ifp->if_start = cxgb_start; ifp->if_timer = 0; /* Disable ifnet watchdog */ ifp->if_watchdog = NULL; ifp->if_snd.ifq_drv_maxlen = TX_ETH_Q_SIZE; IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); IFQ_SET_READY(&ifp->if_snd); ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0; ifp->if_capabilities |= CXGB_CAP; ifp->if_capenable |= CXGB_CAP_ENABLE; ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO); ifp->if_baudrate = 100000000; ether_ifattach(ifp, p->hw_addr); #ifdef DEFAULT_JUMBO ifp->if_mtu = 9000; #endif if ((err = cxgb_makedev(p)) != 0) { printf("makedev failed %d\n", err); return (err); } ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change, cxgb_media_status); if (!strcmp(p->port_type->desc, "10GBASE-CX4")) media_flags = IFM_ETHER | IFM_10G_CX4; else if (!strcmp(p->port_type->desc, "10GBASE-SR")) media_flags = IFM_ETHER | IFM_10G_SR; else if (!strcmp(p->port_type->desc, "10GBASE-XR")) media_flags = IFM_ETHER | IFM_10G_LR; else { printf("unsupported media type %s\n", p->port_type->desc); return (ENXIO); } ifmedia_add(&p->media, media_flags, 0, NULL); ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL); ifmedia_set(&p->media, media_flags); - snprintf(buf, sizeof(buf), "cxgb_port_taskq%d", p->port); + snprintf(buf, sizeof(buf), "cxgb_port_taskq%d", p->port); #ifdef TASKQUEUE_CURRENT /* Create a port for handling TX without starvation */ p->tq = taskqueue_create(buf, M_NOWAIT, taskqueue_thread_enqueue, &p->tq); #else /* Create a port for handling TX without starvation */ p->tq = taskqueue_create_fast(buf, M_NOWAIT, taskqueue_thread_enqueue, &p->tq); #endif if (p->tq == NULL) { device_printf(dev, "failed to allocate port task queue\n"); return (ENOMEM); } taskqueue_start_threads(&p->tq, 1, PI_NET, "%s taskq", device_get_nameunit(dev)); TASK_INIT(&p->start_task, 0, cxgb_start_proc, ifp); return (0); } static int cxgb_port_detach(device_t dev) { struct port_info *p; p = device_get_softc(dev); + + PORT_LOCK(p); + cxgb_stop_locked(p); + PORT_UNLOCK(p); + mtx_destroy(&p->lock); if (p->tq != NULL) { taskqueue_drain(p->tq, &p->start_task); taskqueue_free(p->tq); p->tq = NULL; } ether_ifdetach(p->ifp); if_free(p->ifp); destroy_dev(p->port_cdev); return (0); } void t3_fatal_err(struct adapter *sc) { u_int fw_status[4]; device_printf(sc->dev,"encountered fatal error, operation suspended\n"); if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n", fw_status[0], fw_status[1], fw_status[2], fw_status[3]); } int t3_os_find_pci_capability(adapter_t *sc, int cap) { device_t dev; struct pci_devinfo *dinfo; pcicfgregs *cfg; uint32_t status; uint8_t ptr; dev = sc->dev; dinfo = device_get_ivars(dev); cfg = &dinfo->cfg; status = pci_read_config(dev, PCIR_STATUS, 2); if (!(status & PCIM_STATUS_CAPPRESENT)) return (0); switch (cfg->hdrtype & PCIM_HDRTYPE) { case 0: case 1: ptr = PCIR_CAP_PTR; break; case 2: ptr = PCIR_CAP_PTR_2; break; default: return (0); break; } ptr = pci_read_config(dev, ptr, 1); while (ptr != 0) { if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap) return (ptr); ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1); } return (0); } int t3_os_pci_save_state(struct adapter *sc) { device_t dev; struct pci_devinfo *dinfo; dev = sc->dev; dinfo = device_get_ivars(dev); pci_cfg_save(dev, dinfo, 0); return (0); } int t3_os_pci_restore_state(struct adapter *sc) { device_t dev; struct pci_devinfo *dinfo; dev = sc->dev; dinfo = device_get_ivars(dev); pci_cfg_restore(dev, dinfo); return (0); } /** * t3_os_link_changed - handle link status changes * @adapter: the adapter associated with the link change * @port_id: the port index whose limk status has changed * @link_stat: the new status of the link * @speed: the new speed setting * @duplex: the new duplex setting * @fc: the new flow-control setting * * This is the OS-dependent handler for link status changes. The OS * neutral handler takes care of most of the processing for these events, * then calls this handler for any OS-specific processing. */ void t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, int duplex, int fc) { struct port_info *pi = &adapter->port[port_id]; + struct cmac *mac = &adapter->port[port_id].mac; if ((pi->ifp->if_flags & IFF_UP) == 0) return; - - if (link_status) + + if (link_status) { + t3_mac_enable(mac, MAC_DIRECTION_RX); if_link_state_change(pi->ifp, LINK_STATE_UP); - else + } else { if_link_state_change(pi->ifp, LINK_STATE_DOWN); - + pi->phy.ops->power_down(&pi->phy, 1); + t3_mac_disable(mac, MAC_DIRECTION_RX); + t3_link_start(&pi->phy, mac, &pi->link_config); + } } /* * Interrupt-context handler for external (PHY) interrupts. */ void t3_os_ext_intr_handler(adapter_t *sc) { if (cxgb_debug) printf("t3_os_ext_intr_handler\n"); /* * Schedule a task to handle external interrupts as they may be slow * and we use a mutex to protect MDIO registers. We disable PHY * interrupts in the meantime and let the task reenable them when * it's done. */ + ADAPTER_LOCK(sc); if (sc->slow_intr_mask) { sc->slow_intr_mask &= ~F_T3DBG; t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); taskqueue_enqueue(sc->tq, &sc->ext_intr_task); } + ADAPTER_UNLOCK(sc); } void t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]) { /* * The ifnet might not be allocated before this gets called, * as this is called early on in attach by t3_prep_adapter * save the address off in the port structure */ if (cxgb_debug) printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":"); bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); } /** * link_start - enable a port * @p: the port to enable * * Performs the MAC and PHY actions needed to enable a port. */ static void cxgb_link_start(struct port_info *p) { struct ifnet *ifp; struct t3_rx_mode rm; struct cmac *mac = &p->mac; ifp = p->ifp; t3_init_rx_mode(&rm, p); t3_mac_reset(mac); t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN); t3_mac_set_address(mac, 0, p->hw_addr); t3_mac_set_rx_mode(mac, &rm); t3_link_start(&p->phy, mac, &p->link_config); t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); } /** * setup_rss - configure Receive Side Steering (per-queue connection demux) * @adap: the adapter * * Sets up RSS to distribute packets to multiple receive queues. We * configure the RSS CPU lookup table to distribute to the number of HW * receive queues, and the response queue lookup table to narrow that * down to the response queues actually configured for each port. * We always configure the RSS mapping for two ports since the mapping * table has plenty of entries. */ static void setup_rss(adapter_t *adap) { int i; u_int nq0 = adap->port[0].nqsets; u_int nq1 = max((u_int)adap->port[1].nqsets, 1U); uint8_t cpus[SGE_QSETS + 1]; uint16_t rspq_map[RSS_TABLE_SIZE]; for (i = 0; i < SGE_QSETS; ++i) cpus[i] = i; cpus[SGE_QSETS] = 0xff; for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { rspq_map[i] = i % nq0; rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0; } t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | V_RRCPLCPUSIZE(6), cpus, rspq_map); } +/* + * Sends an mbuf to an offload queue driver + * after dealing with any active network taps. + */ +static inline int +offload_tx(struct toedev *tdev, struct mbuf *m) +{ + int ret; + + critical_enter(); + ret = t3_offload_tx(tdev, m); + critical_exit(); + return ret; +} + +static int +write_smt_entry(struct adapter *adapter, int idx) +{ + struct port_info *pi = &adapter->port[idx]; + struct cpl_smt_write_req *req; + struct mbuf *m; + + if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) + return (ENOMEM); + + req = mtod(m, struct cpl_smt_write_req *); + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx)); + req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */ + req->iff = idx; + memset(req->src_mac1, 0, sizeof(req->src_mac1)); + memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN); + + m_set_priority(m, 1); + + offload_tx(&adapter->tdev, m); + + return (0); +} + +static int +init_smt(struct adapter *adapter) +{ + int i; + + for_each_port(adapter, i) + write_smt_entry(adapter, i); + return 0; +} + static void +init_port_mtus(adapter_t *adapter) +{ + unsigned int mtus = adapter->port[0].ifp->if_mtu; + + if (adapter->port[1].ifp) + mtus |= adapter->port[1].ifp->if_mtu << 16; + t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus); +} + +static void send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, int hi, int port) { struct mbuf *m; struct mngt_pktsched_wr *req; m = m_gethdr(M_NOWAIT, MT_DATA); if (m) { - req = (struct mngt_pktsched_wr *)m->m_data; + req = mtod(m, struct mngt_pktsched_wr *); req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; req->sched = sched; req->idx = qidx; req->min = lo; req->max = hi; req->binding = port; m->m_len = m->m_pkthdr.len = sizeof(*req); t3_mgmt_tx(adap, m); } } static void bind_qsets(adapter_t *sc) { int i, j; for (i = 0; i < (sc)->params.nports; ++i) { const struct port_info *pi = adap2pinfo(sc, i); for (j = 0; j < pi->nqsets; ++j) send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, -1, i); } } +/** + * cxgb_up - enable the adapter + * @adap: adapter being enabled + * + * Called when the first port is enabled, this function performs the + * actions necessary to make an adapter operational, such as completing + * the initialization of HW modules, and enabling interrupts. + * + */ +static int +cxgb_up(struct adapter *sc) +{ + int err = 0; + + if ((sc->flags & FULL_INIT_DONE) == 0) { + + if ((sc->flags & FW_UPTODATE) == 0) + err = upgrade_fw(sc); + + if (err) + goto out; + + err = t3_init_hw(sc, 0); + if (err) + goto out; + + t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); + + err = setup_sge_qsets(sc); + if (err) + goto out; + + setup_rss(sc); + sc->flags |= FULL_INIT_DONE; + } + + t3_intr_clear(sc); + + /* If it's MSI or INTx, allocate a single interrupt for everything */ + if ((sc->flags & USING_MSIX) == 0) { + if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, + &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { + device_printf(sc->dev, "Cannot allocate interrupt rid=%d\n", sc->irq_rid); + err = EINVAL; + goto out; + } + device_printf(sc->dev, "allocated irq_res=%p\n", sc->irq_res); + + if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, +#ifdef INTR_FILTERS + NULL, +#endif + sc->cxgb_intr, sc, &sc->intr_tag)) { + device_printf(sc->dev, "Cannot set up interrupt\n"); + err = EINVAL; + goto irq_err; + } + } else { + cxgb_setup_msix(sc, sc->msi_count); + } + + t3_sge_start(sc); + t3_intr_enable(sc); + + if ((sc->flags & (USING_MSIX | QUEUES_BOUND)) == USING_MSIX) + bind_qsets(sc); + sc->flags |= QUEUES_BOUND; +out: + return (err); +irq_err: + CH_ERR(sc, "request_irq failed, err %d\n", err); + goto out; +} + + +/* + * Release resources when all the ports and offloading have been stopped. + */ static void +cxgb_down(struct adapter *sc) +{ + int i; + + t3_sge_stop(sc); + t3_intr_disable(sc); + + for (i = 0; i < SGE_QSETS; i++) { + if (sc->msix_intr_tag[i] != NULL) { + bus_teardown_intr(sc->dev, sc->msix_irq_res[i], + sc->msix_intr_tag[i]); + sc->msix_intr_tag[i] = NULL; + } + if (sc->msix_irq_res[i] != NULL) { + bus_release_resource(sc->dev, SYS_RES_IRQ, + sc->msix_irq_rid[i], sc->msix_irq_res[i]); + sc->msix_irq_res[i] = NULL; + } + } + + if (sc->intr_tag != NULL) { + bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); + sc->intr_tag = NULL; + } + if (sc->irq_res != NULL) { + device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n", + sc->irq_rid, sc->irq_res); + bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, + sc->irq_res); + sc->irq_res = NULL; + } + + callout_drain(&sc->sge_timer_ch); + taskqueue_drain(sc->tq, &sc->slow_intr_task); + taskqueue_drain(sc->tq, &sc->timer_reclaim_task); +} + +static int +offload_open(struct port_info *pi) +{ + struct adapter *adapter = pi->adapter; + struct toedev *tdev = TOEDEV(pi->ifp); + int adap_up = adapter->open_device_map & PORT_MASK; + int err = 0; + + if (atomic_cmpset_int(&adapter->open_device_map, + (adapter->open_device_map & ~OFFLOAD_DEVMAP_BIT), + (adapter->open_device_map | OFFLOAD_DEVMAP_BIT)) == 0) + return (0); + + ADAPTER_LOCK(pi->adapter); + if (!adap_up) + err = cxgb_up(adapter); + ADAPTER_UNLOCK(pi->adapter); + if (err < 0) + return (err); + + t3_tp_set_offload_mode(adapter, 1); + tdev->lldev = adapter->port[0].ifp; + err = cxgb_offload_activate(adapter); + if (err) + goto out; + + init_port_mtus(adapter); + t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd, + adapter->params.b_wnd, + adapter->params.rev == 0 ? + adapter->port[0].ifp->if_mtu : 0xffff); + init_smt(adapter); + + /* Call back all registered clients */ + cxgb_add_clients(tdev); + +out: + /* restore them in case the offload module has changed them */ + if (err) { + t3_tp_set_offload_mode(adapter, 0); + clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); + cxgb_set_dummy_ops(tdev); + } + return (err); +} + +static int +offload_close(struct toedev *tdev) +{ + struct adapter *adapter = tdev2adap(tdev); + + if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) + return 0; + + /* Call back all registered clients */ + cxgb_remove_clients(tdev); + tdev->lldev = NULL; + cxgb_set_dummy_ops(tdev); + t3_tp_set_offload_mode(adapter, 0); + clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); + + ADAPTER_LOCK(adapter); + if (!adapter->open_device_map) + cxgb_down(adapter); + ADAPTER_UNLOCK(adapter); + + cxgb_offload_deactivate(adapter); + return 0; +} + +static void cxgb_init(void *arg) { struct port_info *p = arg; PORT_LOCK(p); cxgb_init_locked(p); PORT_UNLOCK(p); } static void cxgb_init_locked(struct port_info *p) { struct ifnet *ifp; adapter_t *sc = p->adapter; - int error; + int err; mtx_assert(&p->lock, MA_OWNED); - ifp = p->ifp; - if ((sc->flags & FW_UPTODATE) == 0) { - device_printf(sc->dev, "updating firmware to version %d.%d\n", - FW_VERSION_MAJOR, FW_VERSION_MINOR); - if ((error = cxgb_fw_download(sc, sc->dev)) != 0) { - device_printf(sc->dev, "firmware download failed err: %d" - "interface will be unavailable\n", error); - return; - } - sc->flags |= FW_UPTODATE; - } - cxgb_link_start(p); ADAPTER_LOCK(p->adapter); + if ((sc->open_device_map == 0) && ((err = cxgb_up(sc)) < 0)) { + ADAPTER_UNLOCK(p->adapter); + cxgb_stop_locked(p); + return; + } if (p->adapter->open_device_map == 0) t3_intr_clear(sc); - t3_sge_start(sc); - p->adapter->open_device_map |= (1 << p->port); + setbit(&p->adapter->open_device_map, p->port); + ADAPTER_UNLOCK(p->adapter); - t3_intr_enable(sc); + if (is_offload(sc) && !ofld_disable) { + err = offload_open(p); + if (err) + log(LOG_WARNING, + "Could not initialize offload capabilities\n"); + } + cxgb_link_start(p); t3_port_intr_enable(sc, p->port); - if ((p->adapter->flags & (USING_MSIX | QUEUES_BOUND)) == USING_MSIX) - bind_qsets(sc); - p->adapter->flags |= QUEUES_BOUND; - callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz, cxgb_tick, sc); - + PORT_LOCK(p); ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; + PORT_UNLOCK(p); } static void cxgb_set_rxmode(struct port_info *p) { struct t3_rx_mode rm; struct cmac *mac = &p->mac; mtx_assert(&p->lock, MA_OWNED); t3_init_rx_mode(&rm, p); t3_mac_set_rx_mode(mac, &rm); } static void cxgb_stop_locked(struct port_info *p) { struct ifnet *ifp; mtx_assert(&p->lock, MA_OWNED); mtx_assert(&p->adapter->lock, MA_NOTOWNED); ifp = p->ifp; - ADAPTER_LOCK(p->adapter); + t3_port_intr_disable(p->adapter, p->port); + PORT_LOCK(p); ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); - p->adapter->open_device_map &= ~(1 << p->port); + PORT_UNLOCK(p); + p->phy.ops->power_down(&p->phy, 1); + t3_mac_disable(&p->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX); + + ADAPTER_LOCK(p->adapter); + clrbit(&p->adapter->open_device_map, p->port); + /* + * XXX cancel check_task + */ if (p->adapter->open_device_map == 0) - t3_intr_disable(p->adapter); + cxgb_down(p->adapter); ADAPTER_UNLOCK(p->adapter); - t3_port_intr_disable(p->adapter, p->port); - t3_mac_disable(&p->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX); - } static int cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) { struct port_info *p = ifp->if_softc; struct ifaddr *ifa = (struct ifaddr *)data; struct ifreq *ifr = (struct ifreq *)data; int flags, error = 0; uint32_t mask; /* * XXX need to check that we aren't in the middle of an unload */ switch (command) { case SIOCSIFMTU: if ((ifr->ifr_mtu < ETHERMIN) || (ifr->ifr_mtu > ETHER_MAX_LEN_JUMBO)) error = EINVAL; else if (ifp->if_mtu != ifr->ifr_mtu) { PORT_LOCK(p); ifp->if_mtu = ifr->ifr_mtu; t3_mac_set_mtu(&p->mac, ifp->if_mtu + ETHER_HDR_LEN); PORT_UNLOCK(p); } break; case SIOCSIFADDR: case SIOCGIFADDR: if (ifa->ifa_addr->sa_family == AF_INET) { ifp->if_flags |= IFF_UP; if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { cxgb_init(p); } arp_ifinit(ifp, ifa); } else error = ether_ioctl(ifp, command, data); break; case SIOCSIFFLAGS: if (ifp->if_flags & IFF_UP) { PORT_LOCK(p); if (ifp->if_drv_flags & IFF_DRV_RUNNING) { flags = p->if_flags; if (((ifp->if_flags ^ flags) & IFF_PROMISC) || ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) cxgb_set_rxmode(p); } else cxgb_init_locked(p); p->if_flags = ifp->if_flags; PORT_UNLOCK(p); } else { callout_drain(&p->adapter->cxgb_tick_ch); PORT_LOCK(p); if (ifp->if_drv_flags & IFF_DRV_RUNNING) { cxgb_stop_locked(p); } else { adapter_t *sc = p->adapter; callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz, cxgb_tick, sc); } PORT_UNLOCK(p); } break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: error = ifmedia_ioctl(ifp, ifr, &p->media, command); break; case SIOCSIFCAP: PORT_LOCK(p); mask = ifr->ifr_reqcap ^ ifp->if_capenable; if (mask & IFCAP_TXCSUM) { if (IFCAP_TXCSUM & ifp->if_capenable) { ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4); ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP | CSUM_TSO); } else { ifp->if_capenable |= IFCAP_TXCSUM; ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); } } else if (mask & IFCAP_RXCSUM) { if (IFCAP_RXCSUM & ifp->if_capenable) { ifp->if_capenable &= ~IFCAP_RXCSUM; } else { ifp->if_capenable |= IFCAP_RXCSUM; } } if (mask & IFCAP_TSO4) { if (IFCAP_TSO4 & ifp->if_capenable) { ifp->if_capenable &= ~IFCAP_TSO4; ifp->if_hwassist &= ~CSUM_TSO; } else if (IFCAP_TXCSUM & ifp->if_capenable) { ifp->if_capenable |= IFCAP_TSO4; ifp->if_hwassist |= CSUM_TSO; } else { if (cxgb_debug) printf("cxgb requires tx checksum offload" " be enabled to use TSO\n"); error = EINVAL; } } PORT_UNLOCK(p); break; default: error = ether_ioctl(ifp, command, data); break; } return (error); } static int cxgb_start_tx(struct ifnet *ifp, uint32_t txmax) { struct sge_qset *qs; struct sge_txq *txq; struct port_info *p = ifp->if_softc; struct mbuf *m0, *m = NULL; int err, in_use_init; if (!p->link_config.link_ok) return (ENXIO); if (IFQ_DRV_IS_EMPTY(&ifp->if_snd)) return (ENOBUFS); qs = &p->adapter->sge.qs[p->first_qset]; txq = &qs->txq[TXQ_ETH]; err = 0; mtx_lock(&txq->lock); in_use_init = txq->in_use; while ((txq->in_use - in_use_init < txmax) && (txq->size > txq->in_use + TX_MAX_DESC)) { IFQ_DRV_DEQUEUE(&ifp->if_snd, m); if (m == NULL) break; /* * Convert chain to M_IOVEC */ KASSERT((m->m_flags & M_IOVEC) == 0, ("IOVEC set too early")); m0 = m; #ifdef INVARIANTS /* * Clean up after net stack sloppiness * before calling m_sanity */ m0 = m->m_next; while (m0) { m0->m_flags &= ~M_PKTHDR; m0 = m0->m_next; } m_sanity(m0, 0); m0 = m; #endif if (collapse_mbufs && m->m_pkthdr.len > MCLBYTES && m_collapse(m, TX_MAX_SEGS, &m0) == EFBIG) { if ((m0 = m_defrag(m, M_NOWAIT)) != NULL) { m = m0; m_collapse(m, TX_MAX_SEGS, &m0); } else break; } m = m0; if ((err = t3_encap(p, &m)) != 0) break; - BPF_MTAP(ifp, m); + BPF_MTAP(ifp, m); } mtx_unlock(&txq->lock); if (__predict_false(err)) { if (cxgb_debug) printf("would set OFLAGS\n"); if (err == ENOMEM) { IFQ_LOCK(&ifp->if_snd); IFQ_DRV_PREPEND(&ifp->if_snd, m); IFQ_UNLOCK(&ifp->if_snd); } } if (err == 0 && m == NULL) err = ENOBUFS; return (err); } static void cxgb_start_proc(void *arg, int ncount) { struct ifnet *ifp = arg; struct port_info *pi = ifp->if_softc; struct sge_qset *qs; struct sge_txq *txq; int error = 0; qs = &pi->adapter->sge.qs[pi->first_qset]; txq = &qs->txq[TXQ_ETH]; while (error == 0) { if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC) taskqueue_enqueue(pi->adapter->tq, &pi->adapter->timer_reclaim_task); error = cxgb_start_tx(ifp, TX_START_MAX_DESC); } } static void cxgb_start(struct ifnet *ifp) { struct port_info *pi = ifp->if_softc; struct sge_qset *qs; struct sge_txq *txq; int err; qs = &pi->adapter->sge.qs[pi->first_qset]; txq = &qs->txq[TXQ_ETH]; if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC) taskqueue_enqueue(pi->adapter->tq, &pi->adapter->timer_reclaim_task); err = cxgb_start_tx(ifp, TX_START_MAX_DESC); if (err == 0) taskqueue_enqueue(pi->tq, &pi->start_task); } static int cxgb_media_change(struct ifnet *ifp) { if_printf(ifp, "media change not supported\n"); return (ENXIO); } static void cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) { struct port_info *p = ifp->if_softc; ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; if (!p->link_config.link_ok) return; ifmr->ifm_status |= IFM_ACTIVE; if (p->link_config.duplex) ifmr->ifm_active |= IFM_FDX; else ifmr->ifm_active |= IFM_HDX; } static void cxgb_async_intr(void *data) { adapter_t *sc = data; if (cxgb_debug) device_printf(sc->dev, "cxgb_async_intr\n"); t3_slow_intr_handler(sc); } static void cxgb_ext_intr_handler(void *arg, int count) { adapter_t *sc = (adapter_t *)arg; if (cxgb_debug) printf("cxgb_ext_intr_handler\n"); t3_phy_intr_handler(sc); /* Now reenable external interrupts */ + ADAPTER_LOCK(sc); if (sc->slow_intr_mask) { sc->slow_intr_mask |= F_T3DBG; t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG); t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); } + ADAPTER_UNLOCK(sc); } static void check_link_status(adapter_t *sc) { int i; for (i = 0; i < (sc)->params.nports; ++i) { struct port_info *p = &sc->port[i]; if (!(p->port_type->caps & SUPPORTED_IRQ)) t3_link_changed(sc, i); } } static void check_t3b2_mac(struct adapter *adapter) { int i; for_each_port(adapter, i) { struct port_info *p = &adapter->port[i]; struct ifnet *ifp = p->ifp; int status; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) continue; status = 0; PORT_LOCK(p); if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) status = t3b2_mac_watchdog_task(&p->mac); if (status == 1) p->mac.stats.num_toggled++; else if (status == 2) { struct cmac *mac = &p->mac; t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN); t3_mac_set_address(mac, 0, p->hw_addr); cxgb_set_rxmode(p); t3_link_start(&p->phy, mac, &p->link_config); t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); t3_port_intr_enable(adapter, p->port); p->mac.stats.num_resets++; } PORT_UNLOCK(p); } } static void cxgb_tick(void *arg) { adapter_t *sc = (adapter_t *)arg; const struct adapter_params *p = &sc->params; if (p->linkpoll_period) check_link_status(sc); callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz, cxgb_tick, sc); /* * adapter lock can currently only be acquire after the * port lock */ ADAPTER_UNLOCK(sc); if (p->rev == T3_REV_B2) check_t3b2_mac(sc); } static int in_range(int val, int lo, int hi) { return val < 0 || (val <= hi && val >= lo); } static int cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, struct thread *td) { int mmd, error = 0; struct port_info *pi = dev->si_drv1; adapter_t *sc = pi->adapter; #ifdef PRIV_SUPPORTED if (priv_check(td, PRIV_DRIVER)) { if (cxgb_debug) printf("user does not have access to privileged ioctls\n"); return (EPERM); } #else if (suser(td)) { if (cxgb_debug) printf("user does not have access to privileged ioctls\n"); return (EPERM); } #endif switch (cmd) { case SIOCGMIIREG: { uint32_t val; struct cphy *phy = &pi->phy; struct mii_data *mid = (struct mii_data *)data; if (!phy->mdio_read) return (EOPNOTSUPP); if (is_10G(sc)) { mmd = mid->phy_id >> 8; if (!mmd) mmd = MDIO_DEV_PCS; else if (mmd > MDIO_DEV_XGXS) return -EINVAL; error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd, mid->reg_num, &val); } else error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0, mid->reg_num & 0x1f, &val); if (error == 0) mid->val_out = val; break; } case SIOCSMIIREG: { struct cphy *phy = &pi->phy; struct mii_data *mid = (struct mii_data *)data; if (!phy->mdio_write) return (EOPNOTSUPP); if (is_10G(sc)) { mmd = mid->phy_id >> 8; if (!mmd) mmd = MDIO_DEV_PCS; else if (mmd > MDIO_DEV_XGXS) return (EINVAL); error = phy->mdio_write(sc, mid->phy_id & 0x1f, mmd, mid->reg_num, mid->val_in); } else error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0, mid->reg_num & 0x1f, mid->val_in); break; } case CHELSIO_SETREG: { struct ch_reg *edata = (struct ch_reg *)data; if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) return (EFAULT); t3_write_reg(sc, edata->addr, edata->val); break; } case CHELSIO_GETREG: { struct ch_reg *edata = (struct ch_reg *)data; if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) return (EFAULT); edata->val = t3_read_reg(sc, edata->addr); break; } case CHELSIO_GET_SGE_CONTEXT: { struct ch_cntxt *ecntxt = (struct ch_cntxt *)data; mtx_lock(&sc->sge.reg_lock); switch (ecntxt->cntxt_type) { case CNTXT_TYPE_EGRESS: error = t3_sge_read_ecntxt(sc, ecntxt->cntxt_id, ecntxt->data); break; case CNTXT_TYPE_FL: error = t3_sge_read_fl(sc, ecntxt->cntxt_id, ecntxt->data); break; case CNTXT_TYPE_RSP: error = t3_sge_read_rspq(sc, ecntxt->cntxt_id, ecntxt->data); break; case CNTXT_TYPE_CQ: error = t3_sge_read_cq(sc, ecntxt->cntxt_id, ecntxt->data); break; default: error = EINVAL; break; } mtx_unlock(&sc->sge.reg_lock); break; } case CHELSIO_GET_SGE_DESC: { struct ch_desc *edesc = (struct ch_desc *)data; int ret; if (edesc->queue_num >= SGE_QSETS * 6) return (EINVAL); ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6], edesc->queue_num % 6, edesc->idx, edesc->data); if (ret < 0) return (EINVAL); edesc->size = ret; break; } case CHELSIO_SET_QSET_PARAMS: { struct qset_params *q; struct ch_qset_params *t = (struct ch_qset_params *)data; if (t->qset_idx >= SGE_QSETS) return -EINVAL; if (!in_range(t->intr_lat, 0, M_NEWTIMER) || !in_range(t->cong_thres, 0, 255) || !in_range(t->txq_size[0], MIN_TXQ_ENTRIES, MAX_TXQ_ENTRIES) || !in_range(t->txq_size[1], MIN_TXQ_ENTRIES, MAX_TXQ_ENTRIES) || !in_range(t->txq_size[2], MIN_CTRL_TXQ_ENTRIES, MAX_CTRL_TXQ_ENTRIES) || !in_range(t->fl_size[0], MIN_FL_ENTRIES, MAX_RX_BUFFERS) || !in_range(t->fl_size[1], MIN_FL_ENTRIES, MAX_RX_JUMBO_BUFFERS) || !in_range(t->rspq_size, MIN_RSPQ_ENTRIES, MAX_RSPQ_ENTRIES)) return -EINVAL; if ((sc->flags & FULL_INIT_DONE) && (t->rspq_size >= 0 || t->fl_size[0] >= 0 || t->fl_size[1] >= 0 || t->txq_size[0] >= 0 || t->txq_size[1] >= 0 || t->txq_size[2] >= 0 || t->polling >= 0 || t->cong_thres >= 0)) return -EBUSY; q = &sc->params.sge.qset[t->qset_idx]; if (t->rspq_size >= 0) q->rspq_size = t->rspq_size; if (t->fl_size[0] >= 0) q->fl_size = t->fl_size[0]; if (t->fl_size[1] >= 0) q->jumbo_size = t->fl_size[1]; if (t->txq_size[0] >= 0) q->txq_size[0] = t->txq_size[0]; if (t->txq_size[1] >= 0) q->txq_size[1] = t->txq_size[1]; if (t->txq_size[2] >= 0) q->txq_size[2] = t->txq_size[2]; if (t->cong_thres >= 0) q->cong_thres = t->cong_thres; if (t->intr_lat >= 0) { struct sge_qset *qs = &sc->sge.qs[t->qset_idx]; q->coalesce_nsecs = t->intr_lat*1000; t3_update_qset_coalesce(qs, q); } break; } case CHELSIO_GET_QSET_PARAMS: { struct qset_params *q; struct ch_qset_params *t = (struct ch_qset_params *)data; if (t->qset_idx >= SGE_QSETS) return (EINVAL); q = &(sc)->params.sge.qset[t->qset_idx]; t->rspq_size = q->rspq_size; t->txq_size[0] = q->txq_size[0]; t->txq_size[1] = q->txq_size[1]; t->txq_size[2] = q->txq_size[2]; t->fl_size[0] = q->fl_size; t->fl_size[1] = q->jumbo_size; t->polling = q->polling; t->intr_lat = q->coalesce_nsecs / 1000; t->cong_thres = q->cong_thres; break; } case CHELSIO_SET_QSET_NUM: { struct ch_reg *edata = (struct ch_reg *)data; unsigned int port_idx = pi->port; if (sc->flags & FULL_INIT_DONE) return (EBUSY); if (edata->val < 1 || (edata->val > 1 && !(sc->flags & USING_MSIX))) return (EINVAL); if (edata->val + sc->port[!port_idx].nqsets > SGE_QSETS) return (EINVAL); sc->port[port_idx].nqsets = edata->val; + sc->port[0].first_qset = 0; /* - * XXX we're hardcoding ourselves to 2 ports - * just like the LEENUX + * XXX hardcode ourselves to 2 ports just like LEEENUX */ sc->port[1].first_qset = sc->port[0].nqsets; break; } case CHELSIO_GET_QSET_NUM: { struct ch_reg *edata = (struct ch_reg *)data; edata->val = pi->nqsets; break; } -#ifdef notyet - /* - * XXX FreeBSD driver does not currently support any - * offload functionality - */ +#ifdef notyet case CHELSIO_LOAD_FW: - case CHELSIO_DEVUP: - case CHELSIO_SETMTUTAB: case CHELSIO_GET_PM: case CHELSIO_SET_PM: - case CHELSIO_READ_TCAM_WORD: return (EOPNOTSUPP); break; #endif + case CHELSIO_SETMTUTAB: { + struct ch_mtus *m = (struct ch_mtus *)data; + int i; + + if (!is_offload(sc)) + return (EOPNOTSUPP); + if (offload_running(sc)) + return (EBUSY); + if (m->nmtus != NMTUS) + return (EINVAL); + if (m->mtus[0] < 81) /* accommodate SACK */ + return (EINVAL); + + /* + * MTUs must be in ascending order + */ + for (i = 1; i < NMTUS; ++i) + if (m->mtus[i] < m->mtus[i - 1]) + return (EINVAL); + + memcpy(sc->params.mtus, m->mtus, + sizeof(sc->params.mtus)); + break; + } + case CHELSIO_GETMTUTAB: { + struct ch_mtus *m = (struct ch_mtus *)data; + + if (!is_offload(sc)) + return (EOPNOTSUPP); + + memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus)); + m->nmtus = NMTUS; + break; + } + case CHELSIO_DEVUP: + if (!is_offload(sc)) + return (EOPNOTSUPP); + return offload_open(pi); + break; case CHELSIO_GET_MEM: { struct ch_mem_range *t = (struct ch_mem_range *)data; struct mc7 *mem; uint8_t *useraddr; u64 buf[32]; if (!is_offload(sc)) return (EOPNOTSUPP); if (!(sc->flags & FULL_INIT_DONE)) return (EIO); /* need the memory controllers */ if ((t->addr & 0x7) || (t->len & 0x7)) return (EINVAL); if (t->mem_id == MEM_CM) mem = &sc->cm; else if (t->mem_id == MEM_PMRX) mem = &sc->pmrx; else if (t->mem_id == MEM_PMTX) mem = &sc->pmtx; else return (EINVAL); /* * Version scheme: * bits 0..9: chip version * bits 10..15: chip revision */ t->version = 3 | (sc->params.rev << 10); /* * Read 256 bytes at a time as len can be large and we don't * want to use huge intermediate buffers. */ useraddr = (uint8_t *)(t + 1); /* advance to start of buffer */ while (t->len) { unsigned int chunk = min(t->len, sizeof(buf)); error = t3_mc7_bd_read(mem, t->addr / 8, chunk / 8, buf); if (error) return (-error); if (copyout(buf, useraddr, chunk)) return (EFAULT); useraddr += chunk; t->addr += chunk; t->len -= chunk; } break; } + case CHELSIO_READ_TCAM_WORD: { + struct ch_tcam_word *t = (struct ch_tcam_word *)data; + + if (!is_offload(sc)) + return (EOPNOTSUPP); + return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf); + break; + } case CHELSIO_SET_TRACE_FILTER: { struct ch_trace *t = (struct ch_trace *)data; const struct trace_params *tp; tp = (const struct trace_params *)&t->sip; if (t->config_tx) t3_config_trace_filter(sc, tp, 0, t->invert_match, t->trace_tx); if (t->config_rx) t3_config_trace_filter(sc, tp, 1, t->invert_match, t->trace_rx); break; } case CHELSIO_SET_PKTSCHED: { struct ch_pktsched_params *p = (struct ch_pktsched_params *)data; if (sc->open_device_map == 0) return (EAGAIN); send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max, p->binding); break; } case CHELSIO_IFCONF_GETREGS: { struct ifconf_regs *regs = (struct ifconf_regs *)data; int reglen = cxgb_get_regs_len(); uint8_t *buf = malloc(REGDUMP_SIZE, M_DEVBUF, M_NOWAIT); if (buf == NULL) { return (ENOMEM); } if (regs->len > reglen) regs->len = reglen; else if (regs->len < reglen) { error = E2BIG; goto done; } cxgb_get_regs(sc, regs, buf); error = copyout(buf, regs->data, reglen); done: free(buf, M_DEVBUF); break; } + case CHELSIO_SET_HW_SCHED: { + struct ch_hw_sched *t = (struct ch_hw_sched *)data; + unsigned int ticks_per_usec = core_ticks_per_usec(sc); + + if ((sc->flags & FULL_INIT_DONE) == 0) + return (EAGAIN); /* need TP to be initialized */ + if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) || + !in_range(t->channel, 0, 1) || + !in_range(t->kbps, 0, 10000000) || + !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) || + !in_range(t->flow_ipg, 0, + dack_ticks_to_usec(sc, 0x7ff))) + return (EINVAL); + + if (t->kbps >= 0) { + error = t3_config_sched(sc, t->kbps, t->sched); + if (error < 0) + return (-error); + } + if (t->class_ipg >= 0) + t3_set_sched_ipg(sc, t->sched, t->class_ipg); + if (t->flow_ipg >= 0) { + t->flow_ipg *= 1000; /* us -> ns */ + t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1); + } + if (t->mode >= 0) { + int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched); + + t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, + bit, t->mode ? bit : 0); + } + if (t->channel >= 0) + t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, + 1 << t->sched, t->channel << t->sched); + break; + } default: return (EOPNOTSUPP); break; } return (error); } static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, unsigned int end) { uint32_t *p = (uint32_t *)buf + start; for ( ; start <= end; start += sizeof(uint32_t)) *p++ = t3_read_reg(ap, start); } #define T3_REGMAP_SIZE (3 * 1024) static int cxgb_get_regs_len(void) { return T3_REGMAP_SIZE; } #undef T3_REGMAP_SIZE static void cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf) { /* * Version scheme: * bits 0..9: chip version * bits 10..15: chip revision * bit 31: set for PCIe cards */ regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31); /* * We skip the MAC statistics registers because they are clear-on-read. * Also reading multi-register stats would need to synchronize with the * periodic mac stats accumulation. Hard to justify the complexity. */ memset(buf, 0, REGDUMP_SIZE); reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN); reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0, XGM_REG(A_XGM_SERDES_STAT3, 1)); reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); } Index: head/sys/dev/cxgb/cxgb_offload.c =================================================================== --- head/sys/dev/cxgb/cxgb_offload.c (nonexistent) +++ head/sys/dev/cxgb/cxgb_offload.c (revision 169978) @@ -0,0 +1,1634 @@ + +/************************************************************************** + +Copyright (c) 2007, Chelsio Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Neither the name of the Chelsio Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +***************************************************************************/ + + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/* + * XXX + */ +#define LOG_ERR 1 +#define LOG_NOTICE 2 +#define BUG_ON(...) +#define VALIDATE_TID 0 + + +TAILQ_HEAD(, cxgb_client) client_list; +TAILQ_HEAD(, toedev) ofld_dev_list; +TAILQ_HEAD(, adapter) adapter_list; + +static struct mtx cxgb_db_lock; +static struct rwlock adapter_list_lock; + + +static const unsigned int MAX_ATIDS = 64 * 1024; +static const unsigned int ATID_BASE = 0x100000; +static int inited = 0; + +static inline int +offload_activated(struct toedev *tdev) +{ + struct adapter *adapter = tdev2adap(tdev); + + return (isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)); +} + +/** + * cxgb_register_client - register an offload client + * @client: the client + * + * Add the client to the client list, + * and call backs the client for each activated offload device + */ +void +cxgb_register_client(struct cxgb_client *client) +{ + struct toedev *tdev; + + mtx_lock(&cxgb_db_lock); + TAILQ_INSERT_TAIL(&client_list, client, client_entry); + + if (client->add) { + TAILQ_FOREACH(tdev, &ofld_dev_list, ofld_entry) { + if (offload_activated(tdev)) + client->add(tdev); + } + } + mtx_unlock(&cxgb_db_lock); +} + +/** + * cxgb_unregister_client - unregister an offload client + * @client: the client + * + * Remove the client to the client list, + * and call backs the client for each activated offload device. + */ +void +cxgb_unregister_client(struct cxgb_client *client) +{ + struct toedev *tdev; + + mtx_lock(&cxgb_db_lock); + TAILQ_REMOVE(&client_list, client, client_entry); + + if (client->remove) { + TAILQ_FOREACH(tdev, &ofld_dev_list, ofld_entry) { + if (offload_activated(tdev)) + client->remove(tdev); + } + } + mtx_unlock(&cxgb_db_lock); +} + +/** + * cxgb_add_clients - activate register clients for an offload device + * @tdev: the offload device + * + * Call backs all registered clients once a offload device is activated + */ +void +cxgb_add_clients(struct toedev *tdev) +{ + struct cxgb_client *client; + + mtx_lock(&cxgb_db_lock); + TAILQ_FOREACH(client, &client_list, client_entry) { + if (client->add) + client->add(tdev); + } + mtx_unlock(&cxgb_db_lock); +} + +/** + * cxgb_remove_clients - activate register clients for an offload device + * @tdev: the offload device + * + * Call backs all registered clients once a offload device is deactivated + */ +void +cxgb_remove_clients(struct toedev *tdev) +{ + struct cxgb_client *client; + + mtx_lock(&cxgb_db_lock); + TAILQ_FOREACH(client, &client_list, client_entry) { + if (client->remove) + client->remove(tdev); + } + mtx_unlock(&cxgb_db_lock); +} + +static int +is_offloading(struct ifnet *ifp) +{ + struct adapter *adapter; + int port; + + rw_rlock(&adapter_list_lock); + TAILQ_FOREACH(adapter, &adapter_list, adapter_entry) { + for_each_port(adapter, port) { + if (ifp == adapter->port[port].ifp) { + rw_runlock(&adapter_list_lock); + return 1; + } + } + } + rw_runlock(&adapter_list_lock); + return 0; +} + +static struct ifnet * +get_iff_from_mac(adapter_t *adapter, const uint8_t *mac, unsigned int vlan) +{ +#ifdef notyet + int i; + + for_each_port(adapter, i) { + const struct vlan_group *grp; + const struct port_info *p = &adapter->port[i]; + struct ifnet *ifnet = p->ifp; + + if (!memcmp(p->hw_addr, mac, ETHER_ADDR_LEN)) { + if (vlan && vlan != EVL_VLID_MASK) { + grp = p->vlan_grp; + dev = grp ? grp->vlan_devices[vlan] : NULL; + } else + while (dev->master) + dev = dev->master; + return dev; + } + } +#endif + return NULL; +} + +static inline void +failover_fixup(adapter_t *adapter, int port) +{ + if (adapter->params.rev == 0) { + struct ifnet *ifp = adapter->port[port].ifp; + struct cmac *mac = &adapter->port[port].mac; + if (!(ifp->if_flags & IFF_UP)) { + /* Failover triggered by the interface ifdown */ + t3_write_reg(adapter, A_XGM_TX_CTRL + mac->offset, + F_TXEN); + t3_read_reg(adapter, A_XGM_TX_CTRL + mac->offset); + } else { + /* Failover triggered by the interface link down */ + t3_write_reg(adapter, A_XGM_RX_CTRL + mac->offset, 0); + t3_read_reg(adapter, A_XGM_RX_CTRL + mac->offset); + t3_write_reg(adapter, A_XGM_RX_CTRL + mac->offset, + F_RXEN); + } + } +} + +static int +cxgb_ulp_iscsi_ctl(adapter_t *adapter, unsigned int req, void *data) +{ + int ret = 0; + struct ulp_iscsi_info *uiip = data; + + switch (req) { + case ULP_ISCSI_GET_PARAMS: + uiip->llimit = t3_read_reg(adapter, A_ULPRX_ISCSI_LLIMIT); + uiip->ulimit = t3_read_reg(adapter, A_ULPRX_ISCSI_ULIMIT); + uiip->tagmask = t3_read_reg(adapter, A_ULPRX_ISCSI_TAGMASK); + /* + * On tx, the iscsi pdu has to be <= tx page size and has to + * fit into the Tx PM FIFO. + */ + uiip->max_txsz = min(adapter->params.tp.tx_pg_size, + t3_read_reg(adapter, A_PM1_TX_CFG) >> 17); + /* on rx, the iscsi pdu has to be < rx page size and the + whole pdu + cpl headers has to fit into one sge buffer */ + uiip->max_rxsz = + (unsigned int)min(adapter->params.tp.rx_pg_size, + (adapter->sge.qs[0].fl[1].buf_size - + sizeof(struct cpl_rx_data) * 2 - + sizeof(struct cpl_rx_data_ddp)) ); + break; + case ULP_ISCSI_SET_PARAMS: + t3_write_reg(adapter, A_ULPRX_ISCSI_TAGMASK, uiip->tagmask); + break; + default: + ret = -EOPNOTSUPP; + } + return ret; +} + +/* Response queue used for RDMA events. */ +#define ASYNC_NOTIF_RSPQ 0 + +static int +cxgb_rdma_ctl(adapter_t *adapter, unsigned int req, void *data) +{ + int ret = 0; + + switch (req) { + case RDMA_GET_PARAMS: { + struct rdma_info *req = data; + + req->udbell_physbase = rman_get_start(adapter->regs_res); + req->udbell_len = rman_get_size(adapter->regs_res); + req->tpt_base = t3_read_reg(adapter, A_ULPTX_TPT_LLIMIT); + req->tpt_top = t3_read_reg(adapter, A_ULPTX_TPT_ULIMIT); + req->pbl_base = t3_read_reg(adapter, A_ULPTX_PBL_LLIMIT); + req->pbl_top = t3_read_reg(adapter, A_ULPTX_PBL_ULIMIT); + req->rqt_base = t3_read_reg(adapter, A_ULPRX_RQ_LLIMIT); + req->rqt_top = t3_read_reg(adapter, A_ULPRX_RQ_ULIMIT); + req->kdb_addr = (void *)(rman_get_start(adapter->regs_res) + A_SG_KDOORBELL); + break; + } + case RDMA_CQ_OP: { + struct rdma_cq_op *req = data; + + /* may be called in any context */ + mtx_lock(&adapter->sge.reg_lock); + ret = t3_sge_cqcntxt_op(adapter, req->id, req->op, + req->credits); + mtx_unlock(&adapter->sge.reg_lock); + break; + } + case RDMA_GET_MEM: { + struct ch_mem_range *t = data; + struct mc7 *mem; + + if ((t->addr & 7) || (t->len & 7)) + return -EINVAL; + if (t->mem_id == MEM_CM) + mem = &adapter->cm; + else if (t->mem_id == MEM_PMRX) + mem = &adapter->pmrx; + else if (t->mem_id == MEM_PMTX) + mem = &adapter->pmtx; + else + return -EINVAL; + + ret = t3_mc7_bd_read(mem, t->addr/8, t->len/8, (u64 *)t->buf); + if (ret) + return ret; + break; + } + case RDMA_CQ_SETUP: { + struct rdma_cq_setup *req = data; + + mtx_lock(&adapter->sge.reg_lock); + ret = t3_sge_init_cqcntxt(adapter, req->id, req->base_addr, + req->size, ASYNC_NOTIF_RSPQ, + req->ovfl_mode, req->credits, + req->credit_thres); + mtx_unlock(&adapter->sge.reg_lock); + break; + } + case RDMA_CQ_DISABLE: + mtx_lock(&adapter->sge.reg_lock); + ret = t3_sge_disable_cqcntxt(adapter, *(unsigned int *)data); + mtx_unlock(&adapter->sge.reg_lock); + break; + case RDMA_CTRL_QP_SETUP: { + struct rdma_ctrlqp_setup *req = data; + + mtx_lock(&adapter->sge.reg_lock); + ret = t3_sge_init_ecntxt(adapter, FW_RI_SGEEC_START, 0, + SGE_CNTXT_RDMA, ASYNC_NOTIF_RSPQ, + req->base_addr, req->size, + FW_RI_TID_START, 1, 0); + mtx_unlock(&adapter->sge.reg_lock); + break; + } + default: + ret = -EOPNOTSUPP; + } + return ret; +} + +static int +cxgb_offload_ctl(struct toedev *tdev, unsigned int req, void *data) +{ + struct adapter *adapter = tdev2adap(tdev); + struct tid_range *tid; + struct mtutab *mtup; + struct iff_mac *iffmacp; + struct ddp_params *ddpp; + struct adap_ports *ports; + int port; + + switch (req) { + case GET_MAX_OUTSTANDING_WR: + *(unsigned int *)data = FW_WR_NUM; + break; + case GET_WR_LEN: + *(unsigned int *)data = WR_FLITS; + break; + case GET_TX_MAX_CHUNK: + *(unsigned int *)data = 1 << 20; /* 1MB */ + break; + case GET_TID_RANGE: + tid = data; + tid->num = t3_mc5_size(&adapter->mc5) - + adapter->params.mc5.nroutes - + adapter->params.mc5.nfilters - + adapter->params.mc5.nservers; + tid->base = 0; + break; + case GET_STID_RANGE: + tid = data; + tid->num = adapter->params.mc5.nservers; + tid->base = t3_mc5_size(&adapter->mc5) - tid->num - + adapter->params.mc5.nfilters - + adapter->params.mc5.nroutes; + break; + case GET_L2T_CAPACITY: + *(unsigned int *)data = 2048; + break; + case GET_MTUS: + mtup = data; + mtup->size = NMTUS; + mtup->mtus = adapter->params.mtus; + break; + case GET_IFF_FROM_MAC: + iffmacp = data; + iffmacp->dev = get_iff_from_mac(adapter, iffmacp->mac_addr, + iffmacp->vlan_tag & EVL_VLID_MASK); + break; + case GET_DDP_PARAMS: + ddpp = data; + ddpp->llimit = t3_read_reg(adapter, A_ULPRX_TDDP_LLIMIT); + ddpp->ulimit = t3_read_reg(adapter, A_ULPRX_TDDP_ULIMIT); + ddpp->tag_mask = t3_read_reg(adapter, A_ULPRX_TDDP_TAGMASK); + break; + case GET_PORTS: + ports = data; + ports->nports = adapter->params.nports; + for_each_port(adapter, port) + ports->lldevs[port] = adapter->port[port].ifp; + break; + case FAILOVER: + port = *(int *)data; + t3_port_failover(adapter, port); + failover_fixup(adapter, port); + break; + case FAILOVER_DONE: + port = *(int *)data; + t3_failover_done(adapter, port); + break; + case FAILOVER_CLEAR: + t3_failover_clear(adapter); + break; + case ULP_ISCSI_GET_PARAMS: + case ULP_ISCSI_SET_PARAMS: + if (!offload_running(adapter)) + return -EAGAIN; + return cxgb_ulp_iscsi_ctl(adapter, req, data); + case RDMA_GET_PARAMS: + case RDMA_CQ_OP: + case RDMA_CQ_SETUP: + case RDMA_CQ_DISABLE: + case RDMA_CTRL_QP_SETUP: + case RDMA_GET_MEM: + if (!offload_running(adapter)) + return -EAGAIN; + return cxgb_rdma_ctl(adapter, req, data); + default: + return -EOPNOTSUPP; + } + return 0; +} + +/* + * Dummy handler for Rx offload packets in case we get an offload packet before + * proper processing is setup. This complains and drops the packet as it isn't + * normal to get offload packets at this stage. + */ +static int +rx_offload_blackhole(struct toedev *dev, struct mbuf **m, int n) +{ + CH_ERR(tdev2adap(dev), "%d unexpected offload packets, first data %u\n", + n, ntohl(*mtod(m[0], uint32_t *))); + while (n--) + m_freem(m[n]); + return 0; +} + +static void +dummy_neigh_update(struct toedev *dev, struct rtentry *neigh) +{ +} + +void +cxgb_set_dummy_ops(struct toedev *dev) +{ + dev->recv = rx_offload_blackhole; + dev->neigh_update = dummy_neigh_update; +} + +/* + * Free an active-open TID. + */ +void * +cxgb_free_atid(struct toedev *tdev, int atid) +{ + struct tid_info *t = &(TOE_DATA(tdev))->tid_maps; + union active_open_entry *p = atid2entry(t, atid); + void *ctx = p->toe_tid.ctx; + + mtx_lock(&t->atid_lock); + p->next = t->afree; + t->afree = p; + t->atids_in_use--; + mtx_lock(&t->atid_lock); + + return ctx; +} + +/* + * Free a server TID and return it to the free pool. + */ +void +cxgb_free_stid(struct toedev *tdev, int stid) +{ + struct tid_info *t = &(TOE_DATA(tdev))->tid_maps; + union listen_entry *p = stid2entry(t, stid); + + mtx_lock(&t->stid_lock); + p->next = t->sfree; + t->sfree = p; + t->stids_in_use--; + mtx_unlock(&t->stid_lock); +} + +void +cxgb_insert_tid(struct toedev *tdev, struct cxgb_client *client, + void *ctx, unsigned int tid) +{ + struct tid_info *t = &(TOE_DATA(tdev))->tid_maps; + + t->tid_tab[tid].client = client; + t->tid_tab[tid].ctx = ctx; + atomic_add_int(&t->tids_in_use, 1); +} + +/* + * Populate a TID_RELEASE WR. The mbuf must be already propely sized. + */ +static inline void +mk_tid_release(struct mbuf *m, unsigned int tid) +{ + struct cpl_tid_release *req; + + m_set_priority(m, CPL_PRIORITY_SETUP); + req = mtod(m, struct cpl_tid_release *); + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); +} + +static void +t3_process_tid_release_list(void *data, int pending) +{ + struct mbuf *m; + struct toedev *tdev = data; + struct toe_data *td = TOE_DATA(tdev); + + mtx_lock(&td->tid_release_lock); + while (td->tid_release_list) { + struct toe_tid_entry *p = td->tid_release_list; + + td->tid_release_list = (struct toe_tid_entry *)p->ctx; + mtx_unlock(&td->tid_release_lock); + m = m_get(M_WAIT, MT_DATA); + mk_tid_release(m, p - td->tid_maps.tid_tab); + cxgb_ofld_send(tdev, m); + p->ctx = NULL; + mtx_lock(&td->tid_release_lock); + } + mtx_unlock(&td->tid_release_lock); +} + +/* use ctx as a next pointer in the tid release list */ +void +cxgb_queue_tid_release(struct toedev *tdev, unsigned int tid) +{ + struct toe_data *td = TOE_DATA(tdev); + struct toe_tid_entry *p = &td->tid_maps.tid_tab[tid]; + + mtx_lock(&td->tid_release_lock); + p->ctx = td->tid_release_list; + td->tid_release_list = p; + + if (!p->ctx) + taskqueue_enqueue(tdev->adapter->tq, &td->tid_release_task); + + mtx_unlock(&td->tid_release_lock); +} + +/* + * Remove a tid from the TID table. A client may defer processing its last + * CPL message if it is locked at the time it arrives, and while the message + * sits in the client's backlog the TID may be reused for another connection. + * To handle this we atomically switch the TID association if it still points + * to the original client context. + */ +void +cxgb_remove_tid(struct toedev *tdev, void *ctx, unsigned int tid) +{ + struct tid_info *t = &(TOE_DATA(tdev))->tid_maps; + + BUG_ON(tid >= t->ntids); + if (tdev->type == T3A) + atomic_cmpset_ptr((long *)&t->tid_tab[tid].ctx, (long)NULL, (long)ctx); + else { + struct mbuf *m; + + m = m_get(M_NOWAIT, MT_DATA); + if (__predict_true(m != NULL)) { + mk_tid_release(m, tid); + cxgb_ofld_send(tdev, m); + t->tid_tab[tid].ctx = NULL; + } else + cxgb_queue_tid_release(tdev, tid); + } + atomic_add_int(&t->tids_in_use, -1); +} + +int +cxgb_alloc_atid(struct toedev *tdev, struct cxgb_client *client, + void *ctx) +{ + int atid = -1; + struct tid_info *t = &(TOE_DATA(tdev))->tid_maps; + + mtx_lock(&t->atid_lock); + if (t->afree) { + union active_open_entry *p = t->afree; + + atid = (p - t->atid_tab) + t->atid_base; + t->afree = p->next; + p->toe_tid.ctx = ctx; + p->toe_tid.client = client; + t->atids_in_use++; + } + mtx_unlock(&t->atid_lock); + return atid; +} + +int +cxgb_alloc_stid(struct toedev *tdev, struct cxgb_client *client, + void *ctx) +{ + int stid = -1; + struct tid_info *t = &(TOE_DATA(tdev))->tid_maps; + + mtx_lock(&t->stid_lock); + if (t->sfree) { + union listen_entry *p = t->sfree; + + stid = (p - t->stid_tab) + t->stid_base; + t->sfree = p->next; + p->toe_tid.ctx = ctx; + p->toe_tid.client = client; + t->stids_in_use++; + } + mtx_unlock(&t->stid_lock); + return stid; +} + +static int +do_smt_write_rpl(struct toedev *dev, struct mbuf *m) +{ + struct cpl_smt_write_rpl *rpl = cplhdr(m); + + if (rpl->status != CPL_ERR_NONE) + log(LOG_ERR, + "Unexpected SMT_WRITE_RPL status %u for entry %u\n", + rpl->status, GET_TID(rpl)); + + return CPL_RET_BUF_DONE; +} + +static int +do_l2t_write_rpl(struct toedev *dev, struct mbuf *m) +{ + struct cpl_l2t_write_rpl *rpl = cplhdr(m); + + if (rpl->status != CPL_ERR_NONE) + log(LOG_ERR, + "Unexpected L2T_WRITE_RPL status %u for entry %u\n", + rpl->status, GET_TID(rpl)); + + return CPL_RET_BUF_DONE; +} + +static int +do_act_open_rpl(struct toedev *dev, struct mbuf *m) +{ + struct cpl_act_open_rpl *rpl = cplhdr(m); + unsigned int atid = G_TID(ntohl(rpl->atid)); + struct toe_tid_entry *toe_tid; + + toe_tid = lookup_atid(&(TOE_DATA(dev))->tid_maps, atid); + if (toe_tid->ctx && toe_tid->client && toe_tid->client->handlers && + toe_tid->client->handlers[CPL_ACT_OPEN_RPL]) { + return toe_tid->client->handlers[CPL_ACT_OPEN_RPL] (dev, m, + toe_tid->ctx); + } else { + log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", + dev->name, CPL_ACT_OPEN_RPL); + return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; + } +} + +static int +do_stid_rpl(struct toedev *dev, struct mbuf *m) +{ + union opcode_tid *p = cplhdr(m); + unsigned int stid = G_TID(ntohl(p->opcode_tid)); + struct toe_tid_entry *toe_tid; + + toe_tid = lookup_stid(&(TOE_DATA(dev))->tid_maps, stid); + if (toe_tid->ctx && toe_tid->client->handlers && + toe_tid->client->handlers[p->opcode]) { + return toe_tid->client->handlers[p->opcode] (dev, m, toe_tid->ctx); + } else { + log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", + dev->name, p->opcode); + return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; + } +} + +static int +do_hwtid_rpl(struct toedev *dev, struct mbuf *m) +{ + union opcode_tid *p = cplhdr(m); + unsigned int hwtid = G_TID(ntohl(p->opcode_tid)); + struct toe_tid_entry *toe_tid; + + toe_tid = lookup_tid(&(TOE_DATA(dev))->tid_maps, hwtid); + if (toe_tid->ctx && toe_tid->client->handlers && + toe_tid->client->handlers[p->opcode]) { + return toe_tid->client->handlers[p->opcode] + (dev, m, toe_tid->ctx); + } else { + log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", + dev->name, p->opcode); + return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; + } +} + +static int +do_cr(struct toedev *dev, struct mbuf *m) +{ + struct cpl_pass_accept_req *req = cplhdr(m); + unsigned int stid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); + struct toe_tid_entry *toe_tid; + + toe_tid = lookup_stid(&(TOE_DATA(dev))->tid_maps, stid); + if (toe_tid->ctx && toe_tid->client->handlers && + toe_tid->client->handlers[CPL_PASS_ACCEPT_REQ]) { + return toe_tid->client->handlers[CPL_PASS_ACCEPT_REQ] + (dev, m, toe_tid->ctx); + } else { + log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", + dev->name, CPL_PASS_ACCEPT_REQ); + return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; + } +} + +static int +do_abort_req_rss(struct toedev *dev, struct mbuf *m) +{ + union opcode_tid *p = cplhdr(m); + unsigned int hwtid = G_TID(ntohl(p->opcode_tid)); + struct toe_tid_entry *toe_tid; + + toe_tid = lookup_tid(&(TOE_DATA(dev))->tid_maps, hwtid); + if (toe_tid->ctx && toe_tid->client->handlers && + toe_tid->client->handlers[p->opcode]) { + return toe_tid->client->handlers[p->opcode] + (dev, m, toe_tid->ctx); + } else { + struct cpl_abort_req_rss *req = cplhdr(m); + struct cpl_abort_rpl *rpl; + + struct mbuf *m = m_get(M_NOWAIT, MT_DATA); + if (!m) { + log(LOG_NOTICE, "do_abort_req_rss: couldn't get mbuf!\n"); + goto out; + } + + m_set_priority(m, CPL_PRIORITY_DATA); +#if 0 + __skb_put(skb, sizeof(struct cpl_abort_rpl)); +#endif + rpl = cplhdr(m); + rpl->wr.wr_hi = + htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL)); + rpl->wr.wr_lo = htonl(V_WR_TID(GET_TID(req))); + OPCODE_TID(rpl) = + htonl(MK_OPCODE_TID(CPL_ABORT_RPL, GET_TID(req))); + rpl->cmd = req->status; + cxgb_ofld_send(dev, m); + out: + return CPL_RET_BUF_DONE; + } +} + +static int +do_act_establish(struct toedev *dev, struct mbuf *m) +{ + struct cpl_act_establish *req = cplhdr(m); + unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); + struct toe_tid_entry *toe_tid; + + toe_tid = lookup_atid(&(TOE_DATA(dev))->tid_maps, atid); + if (toe_tid->ctx && toe_tid->client->handlers && + toe_tid->client->handlers[CPL_ACT_ESTABLISH]) { + return toe_tid->client->handlers[CPL_ACT_ESTABLISH] + (dev, m, toe_tid->ctx); + } else { + log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", + dev->name, CPL_PASS_ACCEPT_REQ); + return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; + } +} + +static int +do_set_tcb_rpl(struct toedev *dev, struct mbuf *m) +{ + struct cpl_set_tcb_rpl *rpl = cplhdr(m); + + if (rpl->status != CPL_ERR_NONE) + log(LOG_ERR, + "Unexpected SET_TCB_RPL status %u for tid %u\n", + rpl->status, GET_TID(rpl)); + return CPL_RET_BUF_DONE; +} + +static int +do_trace(struct toedev *dev, struct mbuf *m) +{ +#if 0 + struct cpl_trace_pkt *p = cplhdr(m); + + + skb->protocol = 0xffff; + skb->dev = dev->lldev; + skb_pull(skb, sizeof(*p)); + skb->mac.raw = mtod(m, (char *)); + netif_receive_skb(skb); +#endif + return 0; +} + +static int +do_term(struct toedev *dev, struct mbuf *m) +{ + unsigned int hwtid = ntohl(m_get_priority(m)) >> 8 & 0xfffff; + unsigned int opcode = G_OPCODE(ntohl(m->m_pkthdr.csum_data)); + struct toe_tid_entry *toe_tid; + + toe_tid = lookup_tid(&(TOE_DATA(dev))->tid_maps, hwtid); + if (toe_tid->ctx && toe_tid->client->handlers && + toe_tid->client->handlers[opcode]) { + return toe_tid->client->handlers[opcode](dev, m, toe_tid->ctx); + } else { + log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", + dev->name, opcode); + return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; + } + return (0); +} + +#if defined(FOO) +#include +#include +#include +#include + +static int (*orig_arp_constructor)(struct ifnet *); + +static void +neigh_suspect(struct ifnet *neigh) +{ + struct hh_cache *hh; + + neigh->output = neigh->ops->output; + + for (hh = neigh->hh; hh; hh = hh->hh_next) + hh->hh_output = neigh->ops->output; +} + +static void +neigh_connect(struct ifnet *neigh) +{ + struct hh_cache *hh; + + neigh->output = neigh->ops->connected_output; + + for (hh = neigh->hh; hh; hh = hh->hh_next) + hh->hh_output = neigh->ops->hh_output; +} + +static inline int +neigh_max_probes(const struct neighbour *n) +{ + const struct neigh_parms *p = n->parms; + return (n->nud_state & NUD_PROBE ? + p->ucast_probes : + p->ucast_probes + p->app_probes + p->mcast_probes); +} + +static void +neigh_timer_handler_offload(unsigned long arg) +{ + unsigned long now, next; + struct neighbour *neigh = (struct neighbour *)arg; + unsigned state; + int notify = 0; + + write_lock(&neigh->lock); + + state = neigh->nud_state; + now = jiffies; + next = now + HZ; + + if (!(state & NUD_IN_TIMER)) { +#ifndef CONFIG_SMP + log(LOG_WARNING, "neigh: timer & !nud_in_timer\n"); +#endif + goto out; + } + + if (state & NUD_REACHABLE) { + if (time_before_eq(now, + neigh->confirmed + + neigh->parms->reachable_time)) { + next = neigh->confirmed + neigh->parms->reachable_time; + } else if (time_before_eq(now, + neigh->used + + neigh->parms->delay_probe_time)) { + neigh->nud_state = NUD_DELAY; + neigh->updated = jiffies; + neigh_suspect(neigh); + next = now + neigh->parms->delay_probe_time; + } else { + neigh->nud_state = NUD_STALE; + neigh->updated = jiffies; + neigh_suspect(neigh); + cxgb_neigh_update(neigh); + } + } else if (state & NUD_DELAY) { + if (time_before_eq(now, + neigh->confirmed + + neigh->parms->delay_probe_time)) { + neigh->nud_state = NUD_REACHABLE; + neigh->updated = jiffies; + neigh_connect(neigh); + cxgb_neigh_update(neigh); + next = neigh->confirmed + neigh->parms->reachable_time; + } else { + neigh->nud_state = NUD_PROBE; + neigh->updated = jiffies; + atomic_set_int(&neigh->probes, 0); + next = now + neigh->parms->retrans_time; + } + } else { + /* NUD_PROBE|NUD_INCOMPLETE */ + next = now + neigh->parms->retrans_time; + } + /* + * Needed for read of probes + */ + mb(); + if ((neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) && + neigh->probes >= neigh_max_probes(neigh)) { + struct mbuf *m; + + neigh->nud_state = NUD_FAILED; + neigh->updated = jiffies; + notify = 1; + cxgb_neigh_update(neigh); + NEIGH_CACHE_STAT_INC(neigh->tbl, res_failed); + + /* It is very thin place. report_unreachable is very + complicated routine. Particularly, it can hit the same + neighbour entry! + So that, we try to be accurate and avoid dead loop. --ANK + */ + while (neigh->nud_state == NUD_FAILED && + (skb = __skb_dequeue(&neigh->arp_queue)) != NULL) { + write_unlock(&neigh->lock); + neigh->ops->error_report(neigh, skb); + write_lock(&neigh->lock); + } + skb_queue_purge(&neigh->arp_queue); + } + + if (neigh->nud_state & NUD_IN_TIMER) { + if (time_before(next, jiffies + HZ/2)) + next = jiffies + HZ/2; + if (!mod_timer(&neigh->timer, next)) + neigh_hold(neigh); + } + if (neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) { + struct mbuf *m = skb_peek(&neigh->arp_queue); + + write_unlock(&neigh->lock); + neigh->ops->solicit(neigh, skb); + atomic_add_int(&neigh->probes, 1); + if (m) + m_free(m); + } else { +out: + write_unlock(&neigh->lock); + } + +#ifdef CONFIG_ARPD + if (notify && neigh->parms->app_probes) + neigh_app_notify(neigh); +#endif + neigh_release(neigh); +} + +static int +arp_constructor_offload(struct neighbour *neigh) +{ + if (neigh->ifp && is_offloading(neigh->ifp)) + neigh->timer.function = neigh_timer_handler_offload; + return orig_arp_constructor(neigh); +} + +/* + * This must match exactly the signature of neigh_update for jprobes to work. + * It runs from a trap handler with interrupts off so don't disable BH. + */ +static int +neigh_update_offload(struct neighbour *neigh, const u8 *lladdr, + u8 new, u32 flags) +{ + write_lock(&neigh->lock); + cxgb_neigh_update(neigh); + write_unlock(&neigh->lock); + jprobe_return(); + /* NOTREACHED */ + return 0; +} + +static struct jprobe neigh_update_jprobe = { + .entry = (kprobe_opcode_t *) neigh_update_offload, + .kp.addr = (kprobe_opcode_t *) neigh_update +}; + +#ifdef MODULE_SUPPORT +static int +prepare_arp_with_t3core(void) +{ + int err; + + err = register_jprobe(&neigh_update_jprobe); + if (err) { + log(LOG_ERR, "Could not install neigh_update jprobe, " + "error %d\n", err); + return err; + } + + orig_arp_constructor = arp_tbl.constructor; + arp_tbl.constructor = arp_constructor_offload; + + return 0; +} + +static void +restore_arp_sans_t3core(void) +{ + arp_tbl.constructor = orig_arp_constructor; + unregister_jprobe(&neigh_update_jprobe); +} + +#else /* Module suport */ +static inline int +prepare_arp_with_t3core(void) +{ + return 0; +} + +static inline void +restore_arp_sans_t3core(void) +{} +#endif +#endif +/* + * Process a received packet with an unknown/unexpected CPL opcode. + */ +static int +do_bad_cpl(struct toedev *dev, struct mbuf *m) +{ + log(LOG_ERR, "%s: received bad CPL command 0x%x\n", dev->name, + *mtod(m, uint32_t *)); + return (CPL_RET_BUF_DONE | CPL_RET_BAD_MSG); +} + +/* + * Handlers for each CPL opcode + */ +static cpl_handler_func cpl_handlers[NUM_CPL_CMDS]; + +/* + * Add a new handler to the CPL dispatch table. A NULL handler may be supplied + * to unregister an existing handler. + */ +void +t3_register_cpl_handler(unsigned int opcode, cpl_handler_func h) +{ + if (opcode < NUM_CPL_CMDS) + cpl_handlers[opcode] = h ? h : do_bad_cpl; + else + log(LOG_ERR, "T3C: handler registration for " + "opcode %x failed\n", opcode); +} + +/* + * TOEDEV's receive method. + */ +int +process_rx(struct toedev *dev, struct mbuf **m, int n) +{ + while (n--) { + struct mbuf *m0 = *m++; + unsigned int opcode = G_OPCODE(ntohl(m0->m_pkthdr.csum_data)); + int ret = cpl_handlers[opcode] (dev, m0); + +#if VALIDATE_TID + if (ret & CPL_RET_UNKNOWN_TID) { + union opcode_tid *p = cplhdr(m0); + + log(LOG_ERR, "%s: CPL message (opcode %u) had " + "unknown TID %u\n", dev->name, opcode, + G_TID(ntohl(p->opcode_tid))); + } +#endif + if (ret & CPL_RET_BUF_DONE) + m_freem(m0); + } + return 0; +} + +/* + * Sends an sk_buff to a T3C driver after dealing with any active network taps. + */ +int +cxgb_ofld_send(struct toedev *dev, struct mbuf *m) +{ + int r; + + critical_enter(); + r = dev->send(dev, m); + critical_exit(); + return r; +} + + +/** + * cxgb_ofld_recv - process n received offload packets + * @dev: the offload device + * @m: an array of offload packets + * @n: the number of offload packets + * + * Process an array of ingress offload packets. Each packet is forwarded + * to any active network taps and then passed to the offload device's receive + * method. We optimize passing packets to the receive method by passing + * it the whole array at once except when there are active taps. + */ +int +cxgb_ofld_recv(struct toedev *dev, struct mbuf **m, int n) +{ + +#if defined(CONFIG_CHELSIO_T3) + if (likely(!netdev_nit)) + return dev->recv(dev, skb, n); + + for ( ; n; n--, skb++) { + skb[0]->dev = dev->lldev; + dev_queue_xmit_nit(skb[0], dev->lldev); + skb[0]->dev = NULL; + dev->recv(dev, skb, 1); + } + return 0; +#else + return dev->recv(dev, m, n); +#endif +} + +void +cxgb_neigh_update(struct rtentry *rt) +{ + + if (is_offloading(rt->rt_ifp)) { + struct toedev *tdev = TOEDEV(rt->rt_ifp); + + BUG_ON(!tdev); + t3_l2t_update(tdev, rt); + } +} + +static void +set_l2t_ix(struct toedev *tdev, u32 tid, struct l2t_entry *e) +{ + struct mbuf *m; + struct cpl_set_tcb_field *req; + + m = m_gethdr(M_NOWAIT, MT_DATA); + if (!m) { + log(LOG_ERR, "%s: cannot allocate mbuf!\n", __FUNCTION__); + return; + } + + m_set_priority(m, CPL_PRIORITY_CONTROL); + req = mtod(m, struct cpl_set_tcb_field *); + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); + req->reply = 0; + req->cpu_idx = 0; + req->word = htons(W_TCB_L2T_IX); + req->mask = htobe64(V_TCB_L2T_IX(M_TCB_L2T_IX)); + req->val = htobe64(V_TCB_L2T_IX(e->idx)); + tdev->send(tdev, m); +} + +void +cxgb_redirect(struct rtentry *old, struct rtentry *new) +{ + struct ifnet *olddev, *newdev; + struct tid_info *ti; + struct toedev *tdev; + u32 tid; + int update_tcb; + struct l2t_entry *e; + struct toe_tid_entry *te; + + olddev = old->rt_ifp; + newdev = new->rt_ifp; + if (!is_offloading(olddev)) + return; + if (!is_offloading(newdev)) { + log(LOG_WARNING, "%s: Redirect to non-offload" + "device ignored.\n", __FUNCTION__); + return; + } + tdev = TOEDEV(olddev); + BUG_ON(!tdev); + if (tdev != TOEDEV(newdev)) { + log(LOG_WARNING, "%s: Redirect to different " + "offload device ignored.\n", __FUNCTION__); + return; + } + + /* Add new L2T entry */ + e = t3_l2t_get(tdev, new, ((struct port_info *)new->rt_ifp->if_softc)->port); + if (!e) { + log(LOG_ERR, "%s: couldn't allocate new l2t entry!\n", + __FUNCTION__); + return; + } + + /* Walk tid table and notify clients of dst change. */ + ti = &(TOE_DATA(tdev))->tid_maps; + for (tid=0; tid < ti->ntids; tid++) { + te = lookup_tid(ti, tid); + BUG_ON(!te); + if (te->ctx && te->client && te->client->redirect) { + update_tcb = te->client->redirect(te->ctx, old, new, + e); + if (update_tcb) { + l2t_hold(L2DATA(tdev), e); + set_l2t_ix(tdev, tid, e); + } + } + } + l2t_release(L2DATA(tdev), e); +} + +/* + * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc. + * The allocated memory is cleared. + */ +void * +cxgb_alloc_mem(unsigned long size) +{ + + return malloc(size, M_DEVBUF, M_ZERO); +} + +/* + * Free memory allocated through t3_alloc_mem(). + */ +void +cxgb_free_mem(void *addr) +{ + free(addr, M_DEVBUF); +} + + +/* + * Allocate and initialize the TID tables. Returns 0 on success. + */ +static int +init_tid_tabs(struct tid_info *t, unsigned int ntids, + unsigned int natids, unsigned int nstids, + unsigned int atid_base, unsigned int stid_base) +{ + unsigned long size = ntids * sizeof(*t->tid_tab) + + natids * sizeof(*t->atid_tab) + nstids * sizeof(*t->stid_tab); + + t->tid_tab = cxgb_alloc_mem(size); + if (!t->tid_tab) + return -ENOMEM; + + t->stid_tab = (union listen_entry *)&t->tid_tab[ntids]; + t->atid_tab = (union active_open_entry *)&t->stid_tab[nstids]; + t->ntids = ntids; + t->nstids = nstids; + t->stid_base = stid_base; + t->sfree = NULL; + t->natids = natids; + t->atid_base = atid_base; + t->afree = NULL; + t->stids_in_use = t->atids_in_use = 0; + atomic_set_int(&t->tids_in_use, 0); + mtx_init(&t->stid_lock, "stid", NULL, MTX_DEF); + mtx_init(&t->atid_lock, "atid", NULL, MTX_DEF); + + /* + * Setup the free lists for stid_tab and atid_tab. + */ + if (nstids) { + while (--nstids) + t->stid_tab[nstids - 1].next = &t->stid_tab[nstids]; + t->sfree = t->stid_tab; + } + if (natids) { + while (--natids) + t->atid_tab[natids - 1].next = &t->atid_tab[natids]; + t->afree = t->atid_tab; + } + return 0; +} + +static void +free_tid_maps(struct tid_info *t) +{ + cxgb_free_mem(t->tid_tab); +} + +static inline void +add_adapter(adapter_t *adap) +{ + rw_wlock(&adapter_list_lock); + TAILQ_INSERT_TAIL(&adapter_list, adap, adapter_entry); + rw_wunlock(&adapter_list_lock); +} + +static inline void +remove_adapter(adapter_t *adap) +{ + rw_wlock(&adapter_list_lock); + TAILQ_REMOVE(&adapter_list, adap, adapter_entry); + rw_wunlock(&adapter_list_lock); +} + +/* + * XXX + */ +#define t3_free_l2t(...) + +int +cxgb_offload_activate(struct adapter *adapter) +{ + struct toedev *dev = &adapter->tdev; + int natids, err; + struct toe_data *t; + struct tid_range stid_range, tid_range; + struct mtutab mtutab; + unsigned int l2t_capacity; + + t = malloc(sizeof(*t), M_DEVBUF, M_WAITOK); + if (!t) + return (ENOMEM); + + err = (EOPNOTSUPP); + if (dev->ctl(dev, GET_TX_MAX_CHUNK, &t->tx_max_chunk) < 0 || + dev->ctl(dev, GET_MAX_OUTSTANDING_WR, &t->max_wrs) < 0 || + dev->ctl(dev, GET_L2T_CAPACITY, &l2t_capacity) < 0 || + dev->ctl(dev, GET_MTUS, &mtutab) < 0 || + dev->ctl(dev, GET_TID_RANGE, &tid_range) < 0 || + dev->ctl(dev, GET_STID_RANGE, &stid_range) < 0) + goto out_free; + + err = (ENOMEM); + L2DATA(dev) = t3_init_l2t(l2t_capacity); + if (!L2DATA(dev)) + goto out_free; + + natids = min(tid_range.num / 2, MAX_ATIDS); + err = init_tid_tabs(&t->tid_maps, tid_range.num, natids, + stid_range.num, ATID_BASE, stid_range.base); + if (err) + goto out_free_l2t; + + t->mtus = mtutab.mtus; + t->nmtus = mtutab.size; + + TASK_INIT(&t->tid_release_task, 0 /* XXX? */, t3_process_tid_release_list, dev); + mtx_init(&t->tid_release_lock, "tid release", NULL, MTX_DEF); + t->dev = dev; + + TOE_DATA(dev) = t; + dev->recv = process_rx; + dev->neigh_update = t3_l2t_update; +#if 0 + offload_proc_dev_setup(dev); +#endif + /* Register netevent handler once */ + if (TAILQ_EMPTY(&adapter_list)) { +#if defined(CONFIG_CHELSIO_T3_MODULE) + if (prepare_arp_with_t3core()) + log(LOG_ERR, "Unable to set offload capabilities\n"); +#endif + } + add_adapter(adapter); + return 0; + +out_free_l2t: + t3_free_l2t(L2DATA(dev)); + L2DATA(dev) = NULL; +out_free: + free(t, M_DEVBUF); + return err; + +} + +void +cxgb_offload_deactivate(struct adapter *adapter) +{ + struct toedev *tdev = &adapter->tdev; + struct toe_data *t = TOE_DATA(tdev); + + remove_adapter(adapter); + if (TAILQ_EMPTY(&adapter_list)) { +#if defined(CONFIG_CHELSIO_T3_MODULE) + restore_arp_sans_t3core(); +#endif + } + free_tid_maps(&t->tid_maps); + TOE_DATA(tdev) = NULL; + t3_free_l2t(L2DATA(tdev)); + L2DATA(tdev) = NULL; + free(t, M_DEVBUF); +} + + +static inline void +register_tdev(struct toedev *tdev) +{ + static int unit; + + mtx_lock(&cxgb_db_lock); + snprintf(tdev->name, sizeof(tdev->name), "ofld_dev%d", unit++); + TAILQ_INSERT_TAIL(&ofld_dev_list, tdev, ofld_entry); + mtx_unlock(&cxgb_db_lock); +} + +static inline void +unregister_tdev(struct toedev *tdev) +{ + mtx_lock(&cxgb_db_lock); + TAILQ_REMOVE(&ofld_dev_list, tdev, ofld_entry); + mtx_unlock(&cxgb_db_lock); +} + +void +cxgb_adapter_ofld(struct adapter *adapter) +{ + struct toedev *tdev = &adapter->tdev; + + cxgb_set_dummy_ops(tdev); + tdev->send = t3_offload_tx; + tdev->ctl = cxgb_offload_ctl; + tdev->type = adapter->params.rev == 0 ? + T3A : T3B; + + register_tdev(tdev); +#if 0 + offload_proc_dev_init(tdev); +#endif +} + +void +cxgb_adapter_unofld(struct adapter *adapter) +{ + struct toedev *tdev = &adapter->tdev; +#if 0 + offload_proc_dev_cleanup(tdev); + offload_proc_dev_exit(tdev); +#endif + tdev->recv = NULL; + tdev->neigh_update = NULL; + + unregister_tdev(tdev); +} + +void +cxgb_offload_init(void) +{ + int i; + + if (inited) + return; + else + inited = 1; + + mtx_init(&cxgb_db_lock, "ofld db", NULL, MTX_DEF); + rw_init(&adapter_list_lock, "ofld adap list"); + TAILQ_INIT(&client_list); + TAILQ_INIT(&ofld_dev_list); + TAILQ_INIT(&adapter_list); + + for (i = 0; i < NUM_CPL_CMDS; ++i) + cpl_handlers[i] = do_bad_cpl; + + t3_register_cpl_handler(CPL_SMT_WRITE_RPL, do_smt_write_rpl); + t3_register_cpl_handler(CPL_L2T_WRITE_RPL, do_l2t_write_rpl); + t3_register_cpl_handler(CPL_PASS_OPEN_RPL, do_stid_rpl); + t3_register_cpl_handler(CPL_CLOSE_LISTSRV_RPL, do_stid_rpl); + t3_register_cpl_handler(CPL_PASS_ACCEPT_REQ, do_cr); + t3_register_cpl_handler(CPL_PASS_ESTABLISH, do_hwtid_rpl); + t3_register_cpl_handler(CPL_ABORT_RPL_RSS, do_hwtid_rpl); + t3_register_cpl_handler(CPL_ABORT_RPL, do_hwtid_rpl); + t3_register_cpl_handler(CPL_RX_URG_NOTIFY, do_hwtid_rpl); + t3_register_cpl_handler(CPL_RX_DATA, do_hwtid_rpl); + t3_register_cpl_handler(CPL_TX_DATA_ACK, do_hwtid_rpl); + t3_register_cpl_handler(CPL_TX_DMA_ACK, do_hwtid_rpl); + t3_register_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl); + t3_register_cpl_handler(CPL_PEER_CLOSE, do_hwtid_rpl); + t3_register_cpl_handler(CPL_CLOSE_CON_RPL, do_hwtid_rpl); + t3_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req_rss); + t3_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish); + t3_register_cpl_handler(CPL_SET_TCB_RPL, do_set_tcb_rpl); + t3_register_cpl_handler(CPL_RDMA_TERMINATE, do_term); + t3_register_cpl_handler(CPL_RDMA_EC_STATUS, do_hwtid_rpl); + t3_register_cpl_handler(CPL_TRACE_PKT, do_trace); + t3_register_cpl_handler(CPL_RX_DATA_DDP, do_hwtid_rpl); + t3_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_hwtid_rpl); + t3_register_cpl_handler(CPL_ISCSI_HDR, do_hwtid_rpl); +#if 0 + if (offload_proc_init()) + log(LOG_WARNING, "Unable to create /proc/net/cxgb3 dir\n"); +#endif +} + +void +cxgb_offload_exit(void) +{ +#if 0 + offload_proc_cleanup(); +#endif +} + +#if 0 +static int +offload_info_read_proc(char *buf, char **start, off_t offset, + int length, int *eof, void *data) +{ + struct toe_data *d = data; + struct tid_info *t = &d->tid_maps; + int len; + + len = sprintf(buf, "TID range: 0..%d, in use: %u\n" + "STID range: %d..%d, in use: %u\n" + "ATID range: %d..%d, in use: %u\n" + "MSS: %u\n", + t->ntids - 1, atomic_read(&t->tids_in_use), t->stid_base, + t->stid_base + t->nstids - 1, t->stids_in_use, + t->atid_base, t->atid_base + t->natids - 1, + t->atids_in_use, d->tx_max_chunk); + if (len > length) + len = length; + *eof = 1; + return len; +} + +static int +offload_info_proc_setup(struct proc_dir_entry *dir, + struct toe_data *d) +{ + struct proc_dir_entry *p; + + if (!dir) + return -EINVAL; + + p = create_proc_read_entry("info", 0, dir, offload_info_read_proc, d); + if (!p) + return -ENOMEM; + + p->owner = THIS_MODULE; + return 0; +} + + +static int +offload_devices_read_proc(char *buf, char **start, off_t offset, + int length, int *eof, void *data) +{ + int len; + struct toedev *dev; + struct net_device *ndev; + + len = sprintf(buf, "Device Interfaces\n"); + + mtx_lock(&cxgb_db_lock); + TAILQ_FOREACH(dev, &ofld_dev_list, ofld_entry) { + len += sprintf(buf + len, "%-16s", dev->name); + read_lock(&dev_base_lock); + for (ndev = dev_base; ndev; ndev = ndev->next) { + if (TOEDEV(ndev) == dev) + len += sprintf(buf + len, " %s", ndev->name); + } + read_unlock(&dev_base_lock); + len += sprintf(buf + len, "\n"); + if (len >= length) + break; + } + mtx_unlock(&cxgb_db_lock); + + if (len > length) + len = length; + *eof = 1; + return len; +} + +#endif + Property changes on: head/sys/dev/cxgb/cxgb_offload.c ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/dev/cxgb/cxgb_offload.h =================================================================== --- head/sys/dev/cxgb/cxgb_offload.h (nonexistent) +++ head/sys/dev/cxgb/cxgb_offload.h (revision 169978) @@ -0,0 +1,260 @@ + +/************************************************************************** + +Copyright (c) 2007, Chelsio Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Chelsio Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +$FreeBSD$ + +***************************************************************************/ + +#ifndef _CXGB_OFFLOAD_H +#define _CXGB_OFFLOAD_H + + +#include +#include + +#include +#include + +struct adapter; +struct cxgb_client; + +void cxgb_offload_init(void); +void cxgb_offload_exit(void); + +void cxgb_adapter_ofld(struct adapter *adapter); +void cxgb_adapter_unofld(struct adapter *adapter); +int cxgb_offload_activate(struct adapter *adapter); +void cxgb_offload_deactivate(struct adapter *adapter); +int cxgb_ofld_recv(struct toedev *dev, struct mbuf **m, int n); + +void cxgb_set_dummy_ops(struct toedev *dev); + + +/* + * Client registration. Users of T3 driver must register themselves. + * The T3 driver will call the add function of every client for each T3 + * adapter activated, passing up the toedev ptr. Each client fills out an + * array of callback functions to process CPL messages. + */ + +void cxgb_register_client(struct cxgb_client *client); +void cxgb_unregister_client(struct cxgb_client *client); +void cxgb_add_clients(struct toedev *tdev); +void cxgb_remove_clients(struct toedev *tdev); + +typedef int (*cxgb_cpl_handler_func)(struct toedev *dev, + struct mbuf *m, void *ctx); + +struct cxgb_client { + char *name; + void (*add) (struct toedev *); + void (*remove) (struct toedev *); + cxgb_cpl_handler_func *handlers; + int (*redirect)(void *ctx, struct rtentry *old, + struct rtentry *new, + struct l2t_entry *l2t); + TAILQ_ENTRY(cxgb_client) client_entry; +}; + +/* + * TID allocation services. + */ +int cxgb_alloc_atid(struct toedev *dev, struct cxgb_client *client, + void *ctx); +int cxgb_alloc_stid(struct toedev *dev, struct cxgb_client *client, + void *ctx); +void *cxgb_free_atid(struct toedev *dev, int atid); +void cxgb_free_stid(struct toedev *dev, int stid); +void cxgb_insert_tid(struct toedev *dev, struct cxgb_client *client, + void *ctx, + unsigned int tid); +void cxgb_queue_tid_release(struct toedev *dev, unsigned int tid); +void cxgb_remove_tid(struct toedev *dev, void *ctx, unsigned int tid); + +struct toe_tid_entry { + struct cxgb_client *client; + void *ctx; +}; + +/* CPL message priority levels */ +enum { + CPL_PRIORITY_DATA = 0, /* data messages */ + CPL_PRIORITY_SETUP = 1, /* connection setup messages */ + CPL_PRIORITY_TEARDOWN = 0, /* connection teardown messages */ + CPL_PRIORITY_LISTEN = 1, /* listen start/stop messages */ + CPL_PRIORITY_ACK = 1, /* RX ACK messages */ + CPL_PRIORITY_CONTROL = 1 /* offload control messages */ +}; + +/* Flags for return value of CPL message handlers */ +enum { + CPL_RET_BUF_DONE = 1, // buffer processing done, buffer may be freed + CPL_RET_BAD_MSG = 2, // bad CPL message (e.g., unknown opcode) + CPL_RET_UNKNOWN_TID = 4 // unexpected unknown TID +}; + +typedef int (*cpl_handler_func)(struct toedev *dev, struct mbuf *m); + +/* + * Returns a pointer to the first byte of the CPL header in an sk_buff that + * contains a CPL message. + */ +static inline void *cplhdr(struct mbuf *m) +{ + return m->m_data; +} + +void t3_register_cpl_handler(unsigned int opcode, cpl_handler_func h); + +union listen_entry { + struct toe_tid_entry toe_tid; + union listen_entry *next; +}; + +union active_open_entry { + struct toe_tid_entry toe_tid; + union active_open_entry *next; +}; + +/* + * Holds the size, base address, free list start, etc of the TID, server TID, + * and active-open TID tables for a offload device. + * The tables themselves are allocated dynamically. + */ +struct tid_info { + struct toe_tid_entry *tid_tab; + unsigned int ntids; + volatile int tids_in_use; + + union listen_entry *stid_tab; + unsigned int nstids; + unsigned int stid_base; + + union active_open_entry *atid_tab; + unsigned int natids; + unsigned int atid_base; + + /* + * The following members are accessed R/W so we put them in their own + * cache lines. + * + * XXX We could combine the atid fields above with the lock here since + * atids are use once (unlike other tids). OTOH the above fields are + * usually in cache due to tid_tab. + */ + struct mtx atid_lock /* ____cacheline_aligned_in_smp */; + union active_open_entry *afree; + unsigned int atids_in_use; + + struct mtx stid_lock /*____cacheline_aligned */; + union listen_entry *sfree; + unsigned int stids_in_use; +}; + +struct toe_data { +#ifdef notyet + struct list_head list_node; +#endif + struct toedev *dev; + unsigned int tx_max_chunk; /* max payload for TX_DATA */ + unsigned int max_wrs; /* max in-flight WRs per connection */ + unsigned int nmtus; + const unsigned short *mtus; + struct tid_info tid_maps; + + struct toe_tid_entry *tid_release_list; + struct mtx tid_release_lock; + struct task tid_release_task; +}; + +/* + * toedev -> toe_data accessor + */ +#define TOE_DATA(dev) (*(struct toe_data **)&(dev)->l4opt) + +/* + * Map an ATID or STID to their entries in the corresponding TID tables. + */ +static inline union active_open_entry *atid2entry(const struct tid_info *t, + unsigned int atid) +{ + return &t->atid_tab[atid - t->atid_base]; +} + + +static inline union listen_entry *stid2entry(const struct tid_info *t, + unsigned int stid) +{ + return &t->stid_tab[stid - t->stid_base]; +} + +/* + * Find the connection corresponding to a TID. + */ +static inline struct toe_tid_entry *lookup_tid(const struct tid_info *t, + unsigned int tid) +{ + return tid < t->ntids ? &(t->tid_tab[tid]) : NULL; +} + +/* + * Find the connection corresponding to a server TID. + */ +static inline struct toe_tid_entry *lookup_stid(const struct tid_info *t, + unsigned int tid) +{ + if (tid < t->stid_base || tid >= t->stid_base + t->nstids) + return NULL; + return &(stid2entry(t, tid)->toe_tid); +} + +/* + * Find the connection corresponding to an active-open TID. + */ +static inline struct toe_tid_entry *lookup_atid(const struct tid_info *t, + unsigned int tid) +{ + if (tid < t->atid_base || tid >= t->atid_base + t->natids) + return NULL; + return &(atid2entry(t, tid)->toe_tid); +} + +void *cxgb_alloc_mem(unsigned long size); +void cxgb_free_mem(void *addr); +void cxgb_neigh_update(struct rtentry *rt); +void cxgb_redirect(struct rtentry *old, struct rtentry *new); +int process_rx(struct toedev *dev, struct mbuf **m, int n); +int attach_toedev(struct toedev *dev); +void detach_toedev(struct toedev *dev); + + +#endif Property changes on: head/sys/dev/cxgb/cxgb_offload.h ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/dev/cxgb/cxgb_osdep.h =================================================================== --- head/sys/dev/cxgb/cxgb_osdep.h (revision 169977) +++ head/sys/dev/cxgb/cxgb_osdep.h (revision 169978) @@ -1,260 +1,283 @@ /************************************************************************** Copyright (c) 2007, Chelsio Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Chelsio Corporation nor the names of its + 2. Neither the name of the Chelsio Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. $FreeBSD$ ***************************************************************************/ #include #include #include #include #include #include #include #include #ifndef _CXGB_OSDEP_H_ #define _CXGB_OSDEP_H_ typedef struct adapter adapter_t; - struct sge_rspq; +#define PANIC_IF(exp) do { \ + if (exp) \ + panic("BUG: %s", exp); \ +} while (0) + + +#define m_get_priority(m) ((uintptr_t)(m)->m_pkthdr.rcvif) +#define m_set_priority(m, pri) ((m)->m_pkthdr.rcvif = (struct ifnet *)(pri)) + #if __FreeBSD_version > 700030 #define INTR_FILTERS #define FIRMWARE_LATEST #endif #if ((__FreeBSD_version > 602103) && (__FreeBSD_version < 700000)) #define FIRMWARE_LATEST #endif #if __FreeBSD_version > 700000 #define MSI_SUPPORTED #define TSO_SUPPORTED #define VLAN_SUPPORTED #define TASKQUEUE_CURRENT #endif +#define __read_mostly __attribute__((__section__(".data.read_mostly"))) + /* * Workaround for weird Chelsio issue */ #if __FreeBSD_version > 700029 #define PRIV_SUPPORTED #endif -#define CXGB_TX_CLEANUP_THRESHOLD 32 +#define CXGB_TX_CLEANUP_THRESHOLD 32 -#define LOG_WARNING 1 +#define LOG_WARNING 1 +#define LOG_ERR 2 #ifdef DEBUG_PRINT #define DPRINTF printf #else #define DPRINTF(...) #endif #define TX_MAX_SIZE (1 << 16) /* 64KB */ #define TX_MAX_SEGS 36 /* maximum supported by card */ #define TX_MAX_DESC 4 /* max descriptors per packet */ #define TX_START_MAX_DESC (TX_MAX_DESC << 1) /* maximum number of descriptors * call to start used per */ #define TX_CLEAN_MAX_DESC (TX_MAX_DESC << 2) /* maximum tx descriptors * to clean per iteration */ #if defined(__i386__) || defined(__amd64__) #define mb() __asm volatile("mfence":::"memory") #define rmb() __asm volatile("lfence":::"memory") #define wmb() __asm volatile("sfence" ::: "memory") #define smp_mb() mb() #define L1_CACHE_BYTES 32 static __inline void prefetch(void *x) { __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); } +extern void kdb_backtrace(void); + +#define WARN_ON(condition) do { \ + if (unlikely((condition)!=0)) { \ + log(LOG_WARNING, "BUG: warning at %s:%d/%s()\n", __FILE__, __LINE__, __FUNCTION__); \ + kdb_backtrace(); \ + } \ +} while (0) + + #else /* !i386 && !amd64 */ #define mb() #define rmb() #define wmb() #define smp_mb() #define prefetch(x) #define L1_CACHE_BYTES 32 #endif #define DBG_RX (1 << 0) static const int debug_flags = DBG_RX; #ifdef DEBUG_PRINT #define DBG(flag, msg) do { \ if ((flag & debug_flags)) \ printf msg; \ } while (0) #else #define DBG(...) #endif #define promisc_rx_mode(rm) ((rm)->port->ifp->if_flags & IFF_PROMISC) #define allmulti_rx_mode(rm) ((rm)->port->ifp->if_flags & IFF_ALLMULTI) #define CH_ERR(adap, fmt, ...)device_printf(adap->dev, fmt, ##__VA_ARGS__); #define CH_WARN(adap, fmt, ...) device_printf(adap->dev, fmt, ##__VA_ARGS__) #define CH_ALERT(adap, fmt, ...) device_printf(adap->dev, fmt, ##__VA_ARGS__) #define t3_os_sleep(x) DELAY((x) * 1000) +#define test_and_clear_bit(bit, p) atomic_cmpset_int((p), ((*(p)) | bit), ((*(p)) & ~bit)) + + #define max_t(type, a, b) (type)max((a), (b)) +#define net_device ifnet + + /* Standard PHY definitions */ #define BMCR_LOOPBACK BMCR_LOOP #define BMCR_ISOLATE BMCR_ISO #define BMCR_ANENABLE BMCR_AUTOEN #define BMCR_SPEED1000 BMCR_SPEED1 #define BMCR_SPEED100 BMCR_SPEED0 #define BMCR_ANRESTART BMCR_STARTNEG #define BMCR_FULLDPLX BMCR_FDX #define BMSR_LSTATUS BMSR_LINK #define BMSR_ANEGCOMPLETE BMSR_ACOMP #define MII_LPA MII_ANLPAR #define MII_ADVERTISE MII_ANAR #define MII_CTRL1000 MII_100T2CR #define ADVERTISE_PAUSE_CAP ANAR_FC #define ADVERTISE_PAUSE_ASYM 0x0800 #define ADVERTISE_1000HALF ANAR_X_HD #define ADVERTISE_1000FULL ANAR_X_FD #define ADVERTISE_10FULL ANAR_10_FD #define ADVERTISE_10HALF ANAR_10 #define ADVERTISE_100FULL ANAR_TX_FD #define ADVERTISE_100HALF ANAR_TX /* Standard PCI Extended Capaibilities definitions */ #define PCI_CAP_ID_VPD 0x03 #define PCI_VPD_ADDR 2 #define PCI_VPD_ADDR_F 0x8000 #define PCI_VPD_DATA 4 #define PCI_CAP_ID_EXP 0x10 #define PCI_EXP_DEVCTL 8 #define PCI_EXP_DEVCTL_PAYLOAD 0x00e0 #define PCI_EXP_LNKCTL 16 #define PCI_EXP_LNKSTA 18 /* * Linux compatibility macros */ /* Some simple translations */ #define __devinit #define udelay(x) DELAY(x) #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) #define le32_to_cpu(x) le32toh(x) #define cpu_to_le32(x) htole32(x) #define swab32(x) bswap32(x) #define simple_strtoul strtoul /* More types and endian definitions */ typedef uint8_t u8; typedef uint16_t u16; typedef uint32_t u32; typedef uint64_t u64; typedef uint8_t __u8; typedef uint16_t __u16; typedef uint32_t __u32; typedef uint8_t __be8; typedef uint16_t __be16; typedef uint32_t __be32; typedef uint64_t __be64; #if BYTE_ORDER == BIG_ENDIAN #define __BIG_ENDIAN_BITFIELD #elif BYTE_ORDER == LITTLE_ENDIAN #define __LITTLE_ENDIAN_BITFIELD #else #error "Must set BYTE_ORDER" #endif /* Indicates what features are supported by the interface. */ #define SUPPORTED_10baseT_Half (1 << 0) #define SUPPORTED_10baseT_Full (1 << 1) #define SUPPORTED_100baseT_Half (1 << 2) #define SUPPORTED_100baseT_Full (1 << 3) #define SUPPORTED_1000baseT_Half (1 << 4) #define SUPPORTED_1000baseT_Full (1 << 5) #define SUPPORTED_Autoneg (1 << 6) #define SUPPORTED_TP (1 << 7) #define SUPPORTED_AUI (1 << 8) #define SUPPORTED_MII (1 << 9) #define SUPPORTED_FIBRE (1 << 10) #define SUPPORTED_BNC (1 << 11) #define SUPPORTED_10000baseT_Full (1 << 12) #define SUPPORTED_Pause (1 << 13) #define SUPPORTED_Asym_Pause (1 << 14) /* Indicates what features are advertised by the interface. */ #define ADVERTISED_10baseT_Half (1 << 0) #define ADVERTISED_10baseT_Full (1 << 1) #define ADVERTISED_100baseT_Half (1 << 2) #define ADVERTISED_100baseT_Full (1 << 3) #define ADVERTISED_1000baseT_Half (1 << 4) #define ADVERTISED_1000baseT_Full (1 << 5) #define ADVERTISED_Autoneg (1 << 6) #define ADVERTISED_TP (1 << 7) #define ADVERTISED_AUI (1 << 8) #define ADVERTISED_MII (1 << 9) #define ADVERTISED_FIBRE (1 << 10) #define ADVERTISED_BNC (1 << 11) #define ADVERTISED_10000baseT_Full (1 << 12) #define ADVERTISED_Pause (1 << 13) #define ADVERTISED_Asym_Pause (1 << 14) /* Enable or disable autonegotiation. If this is set to enable, * the forced link modes above are completely ignored. */ #define AUTONEG_DISABLE 0x00 #define AUTONEG_ENABLE 0x01 #define SPEED_10 10 #define SPEED_100 100 #define SPEED_1000 1000 #define SPEED_10000 10000 #define DUPLEX_HALF 0 #define DUPLEX_FULL 1 #endif Index: head/sys/dev/cxgb/cxgb_sge.c =================================================================== --- head/sys/dev/cxgb/cxgb_sge.c (revision 169977) +++ head/sys/dev/cxgb/cxgb_sge.c (revision 169978) @@ -1,2300 +1,2665 @@ /************************************************************************** Copyright (c) 2007, Chelsio Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Chelsio Corporation nor the names of its + 2. Neither the name of the Chelsio Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ***************************************************************************/ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include #include uint32_t collapse_free = 0; uint32_t mb_free_vec_free = 0; int collapse_mbufs = 0; +static int recycle_enable = 1; + +/* + * XXX GC + */ +#define NET_XMIT_CN 2 +#define NET_XMIT_SUCCESS 0 + #define USE_GTS 0 #define SGE_RX_SM_BUF_SIZE 1536 #define SGE_RX_DROP_THRES 16 +#define SGE_RX_COPY_THRES 128 /* * Period of the Tx buffer reclaim timer. This timer does not need to run * frequently as Tx buffers are usually reclaimed by new Tx packets. */ #define TX_RECLAIM_PERIOD (hz >> 2) /* * work request size in bytes */ #define WR_LEN (WR_FLITS * 8) /* * Values for sge_txq.flags */ enum { TXQ_RUNNING = 1 << 0, /* fetch engine is running */ TXQ_LAST_PKT_DB = 1 << 1, /* last packet rang the doorbell */ }; struct tx_desc { uint64_t flit[TX_DESC_FLITS]; } __packed; struct rx_desc { uint32_t addr_lo; uint32_t len_gen; uint32_t gen2; uint32_t addr_hi; } __packed;; struct rsp_desc { /* response queue descriptor */ struct rss_header rss_hdr; uint32_t flags; uint32_t len_cq; uint8_t imm_data[47]; uint8_t intr_gen; } __packed; #define RX_SW_DESC_MAP_CREATED (1 << 0) #define TX_SW_DESC_MAP_CREATED (1 << 1) #define RX_SW_DESC_INUSE (1 << 3) #define TX_SW_DESC_MAPPED (1 << 4) #define RSPQ_NSOP_NEOP G_RSPD_SOP_EOP(0) #define RSPQ_EOP G_RSPD_SOP_EOP(F_RSPD_EOP) #define RSPQ_SOP G_RSPD_SOP_EOP(F_RSPD_SOP) #define RSPQ_SOP_EOP G_RSPD_SOP_EOP(F_RSPD_SOP|F_RSPD_EOP) struct tx_sw_desc { /* SW state per Tx descriptor */ struct mbuf *m; bus_dmamap_t map; int flags; }; struct rx_sw_desc { /* SW state per Rx descriptor */ void *cl; bus_dmamap_t map; int flags; }; struct txq_state { unsigned int compl; unsigned int gen; unsigned int pidx; }; struct refill_fl_cb_arg { int error; bus_dma_segment_t seg; int nseg; }; /* * Maps a number of flits to the number of Tx descriptors that can hold them. * The formula is * * desc = 1 + (flits - 2) / (WR_FLITS - 1). * * HW allows up to 4 descriptors to be combined into a WR. */ static uint8_t flit_desc_map[] = { 0, #if SGE_NUM_GENBITS == 1 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 #elif SGE_NUM_GENBITS == 2 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, #else # error "SGE_NUM_GENBITS must be 1 or 2" #endif }; static int lro_default = 0; int cxgb_debug = 0; static void t3_free_qset(adapter_t *sc, struct sge_qset *q); static void sge_timer_cb(void *arg); static void sge_timer_reclaim(void *arg, int ncount); static int free_tx_desc(adapter_t *sc, struct sge_txq *q, int n, struct mbuf **m_vec); /** * reclaim_completed_tx - reclaims completed Tx descriptors * @adapter: the adapter * @q: the Tx queue to reclaim completed descriptors from * * Reclaims Tx descriptors that the SGE has indicated it has processed, * and frees the associated buffers if possible. Called with the Tx * queue's lock held. */ static __inline int reclaim_completed_tx(adapter_t *adapter, struct sge_txq *q, int nbufs, struct mbuf **mvec) { int reclaimed, reclaim = desc_reclaimable(q); int n = 0; mtx_assert(&q->lock, MA_OWNED); - if (reclaim > 0) { n = free_tx_desc(adapter, q, min(reclaim, nbufs), mvec); reclaimed = min(reclaim, nbufs); q->cleaned += reclaimed; q->in_use -= reclaimed; } - return (n); } /** + * should_restart_tx - are there enough resources to restart a Tx queue? + * @q: the Tx queue + * + * Checks if there are enough descriptors to restart a suspended Tx queue. + */ +static __inline int +should_restart_tx(const struct sge_txq *q) +{ + unsigned int r = q->processed - q->cleaned; + + return q->in_use - r < (q->size >> 1); +} + +/** * t3_sge_init - initialize SGE * @adap: the adapter * @p: the SGE parameters * * Performs SGE initialization needed every time after a chip reset. * We do not initialize any of the queue sets here, instead the driver * top-level must request those individually. We also do not enable DMA * here, that should be done after the queues have been set up. */ void t3_sge_init(adapter_t *adap, struct sge_params *p) { u_int ctrl, ups; ups = 0; /* = ffs(pci_resource_len(adap->pdev, 2) >> 12); */ ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL | F_CQCRDTCTRL | V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS | V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING; #if SGE_NUM_GENBITS == 1 ctrl |= F_EGRGENCTRL; #endif if (adap->params.rev > 0) { if (!(adap->flags & (USING_MSIX | USING_MSI))) ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ; ctrl |= F_CQCRDTCTRL | F_AVOIDCQOVFL; } t3_write_reg(adap, A_SG_CONTROL, ctrl); t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) | V_LORCQDRBTHRSH(512)); t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10); t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) | V_TIMEOUT(200 * core_ticks_per_usec(adap))); t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH, 1000); t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256); t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000); t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256); t3_write_reg(adap, A_SG_OCO_BASE, V_BASE1(0xfff)); t3_write_reg(adap, A_SG_DRB_PRI_THRESH, 63 * 1024); } /** * sgl_len - calculates the size of an SGL of the given capacity * @n: the number of SGL entries * * Calculates the number of flits needed for a scatter/gather list that * can hold the given number of entries. */ static __inline unsigned int sgl_len(unsigned int n) { return ((3 * n) / 2 + (n & 1)); } /** * get_imm_packet - return the next ingress packet buffer from a response * @resp: the response descriptor containing the packet data * * Return a packet containing the immediate data of the given response. */ static __inline void get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m, void *cl) { int len; uint32_t flags = ntohl(resp->flags); uint8_t sopeop = G_RSPD_SOP_EOP(flags); /* * would be a firmware bug */ if (sopeop == RSPQ_NSOP_NEOP || sopeop == RSPQ_SOP) return; len = G_RSPD_LEN(ntohl(resp->len_cq)); switch (sopeop) { case RSPQ_SOP_EOP: m->m_len = m->m_pkthdr.len = len; memcpy(m->m_data, resp->imm_data, len); break; case RSPQ_EOP: memcpy(cl, resp->imm_data, len); m_iovappend(m, cl, MSIZE, len, 0); break; } } static __inline u_int flits_to_desc(u_int n) { return (flit_desc_map[n]); } void t3_sge_err_intr_handler(adapter_t *adapter) { unsigned int v, status; status = t3_read_reg(adapter, A_SG_INT_CAUSE); if (status & F_RSPQCREDITOVERFOW) CH_ALERT(adapter, "SGE response queue credit overflow\n"); if (status & F_RSPQDISABLED) { v = t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS); CH_ALERT(adapter, "packet delivered to disabled response queue (0x%x)\n", (v >> S_RSPQ0DISABLED) & 0xff); } t3_write_reg(adapter, A_SG_INT_CAUSE, status); if (status & (F_RSPQCREDITOVERFOW | F_RSPQDISABLED)) t3_fatal_err(adapter); } void t3_sge_prep(adapter_t *adap, struct sge_params *p) { int i; /* XXX Does ETHER_ALIGN need to be accounted for here? */ p->max_pkt_size = MJUM16BYTES - sizeof(struct cpl_rx_data); for (i = 0; i < SGE_QSETS; ++i) { struct qset_params *q = p->qset + i; q->polling = adap->params.rev > 0; if (adap->flags & USING_MSIX) q->coalesce_nsecs = 6000; else q->coalesce_nsecs = 3500; q->rspq_size = RSPQ_Q_SIZE; q->fl_size = FL_Q_SIZE; q->jumbo_size = JUMBO_Q_SIZE; q->txq_size[TXQ_ETH] = TX_ETH_Q_SIZE; q->txq_size[TXQ_OFLD] = 1024; q->txq_size[TXQ_CTRL] = 256; q->cong_thres = 0; } } int t3_sge_alloc(adapter_t *sc) { /* The parent tag. */ if (bus_dma_tag_create( NULL, /* parent */ 1, 0, /* algnmnt, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ BUS_SPACE_MAXSIZE_32BIT,/* maxsize */ BUS_SPACE_UNRESTRICTED, /* nsegments */ BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */ 0, /* flags */ NULL, NULL, /* lock, lockarg */ &sc->parent_dmat)) { device_printf(sc->dev, "Cannot allocate parent DMA tag\n"); return (ENOMEM); } /* * DMA tag for normal sized RX frames */ if (bus_dma_tag_create(sc->parent_dmat, MCLBYTES, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES, 1, MCLBYTES, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_dmat)) { device_printf(sc->dev, "Cannot allocate RX DMA tag\n"); return (ENOMEM); } /* * DMA tag for jumbo sized RX frames. */ if (bus_dma_tag_create(sc->parent_dmat, MJUMPAGESIZE, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, MJUMPAGESIZE, 1, MJUMPAGESIZE, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_jumbo_dmat)) { device_printf(sc->dev, "Cannot allocate RX jumbo DMA tag\n"); return (ENOMEM); } /* * DMA tag for TX frames. */ if (bus_dma_tag_create(sc->parent_dmat, 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, TX_MAX_SIZE, TX_MAX_SEGS, TX_MAX_SIZE, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->tx_dmat)) { device_printf(sc->dev, "Cannot allocate TX DMA tag\n"); return (ENOMEM); } return (0); } int t3_sge_free(struct adapter * sc) { if (sc->tx_dmat != NULL) bus_dma_tag_destroy(sc->tx_dmat); if (sc->rx_jumbo_dmat != NULL) bus_dma_tag_destroy(sc->rx_jumbo_dmat); if (sc->rx_dmat != NULL) bus_dma_tag_destroy(sc->rx_dmat); if (sc->parent_dmat != NULL) bus_dma_tag_destroy(sc->parent_dmat); return (0); } void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p) { qs->rspq.holdoff_tmr = max(p->coalesce_nsecs/100, 1U); qs->rspq.polling = 0 /* p->polling */; } static void refill_fl_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { struct refill_fl_cb_arg *cb_arg = arg; cb_arg->error = error; cb_arg->seg = segs[0]; cb_arg->nseg = nseg; } /** * refill_fl - refill an SGE free-buffer list * @sc: the controller softc * @q: the free-list to refill * @n: the number of new buffers to allocate * * (Re)populate an SGE free-buffer list with up to @n new packet buffers. * The caller must assure that @n does not exceed the queue's capacity. */ static void refill_fl(adapter_t *sc, struct sge_fl *q, int n) { struct rx_sw_desc *sd = &q->sdesc[q->pidx]; struct rx_desc *d = &q->desc[q->pidx]; struct refill_fl_cb_arg cb_arg; void *cl; int err; cb_arg.error = 0; while (n--) { /* * We only allocate a cluster, mbuf allocation happens after rx */ if ((cl = m_cljget(NULL, M_DONTWAIT, q->buf_size)) == NULL) { log(LOG_WARNING, "Failed to allocate cluster\n"); goto done; } if ((sd->flags & RX_SW_DESC_MAP_CREATED) == 0) { if ((err = bus_dmamap_create(q->entry_tag, 0, &sd->map))) { log(LOG_WARNING, "bus_dmamap_create failed %d\n", err); uma_zfree(q->zone, cl); goto done; } sd->flags |= RX_SW_DESC_MAP_CREATED; } err = bus_dmamap_load(q->entry_tag, sd->map, cl, q->buf_size, refill_fl_cb, &cb_arg, 0); if (err != 0 || cb_arg.error) { log(LOG_WARNING, "failure in refill_fl %d\n", cb_arg.error); /* * XXX free cluster */ return; } sd->flags |= RX_SW_DESC_INUSE; sd->cl = cl; d->addr_lo = htobe32(cb_arg.seg.ds_addr & 0xffffffff); d->addr_hi = htobe32(((uint64_t)cb_arg.seg.ds_addr >>32) & 0xffffffff); d->len_gen = htobe32(V_FLD_GEN1(q->gen)); d->gen2 = htobe32(V_FLD_GEN2(q->gen)); d++; sd++; if (++q->pidx == q->size) { q->pidx = 0; q->gen ^= 1; sd = q->sdesc; d = q->desc; } q->credits++; } done: t3_write_reg(sc, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); } /** * free_rx_bufs - free the Rx buffers on an SGE free list * @sc: the controle softc * @q: the SGE free list to clean up * * Release the buffers on an SGE free-buffer Rx queue. HW fetching from * this queue should be stopped before calling this function. */ static void free_rx_bufs(adapter_t *sc, struct sge_fl *q) { u_int cidx = q->cidx; while (q->credits--) { struct rx_sw_desc *d = &q->sdesc[cidx]; if (d->flags & RX_SW_DESC_INUSE) { bus_dmamap_unload(q->entry_tag, d->map); bus_dmamap_destroy(q->entry_tag, d->map); uma_zfree(q->zone, d->cl); } d->cl = NULL; if (++cidx == q->size) cidx = 0; } } static __inline void __refill_fl(adapter_t *adap, struct sge_fl *fl) { refill_fl(adap, fl, min(16U, fl->size - fl->credits)); } +/** + * recycle_rx_buf - recycle a receive buffer + * @adapter: the adapter + * @q: the SGE free list + * @idx: index of buffer to recycle + * + * Recycles the specified buffer on the given free list by adding it at + * the next available slot on the list. + */ static void +recycle_rx_buf(adapter_t *adap, struct sge_fl *q, unsigned int idx) +{ + struct rx_desc *from = &q->desc[idx]; + struct rx_desc *to = &q->desc[q->pidx]; + + q->sdesc[q->pidx] = q->sdesc[idx]; + to->addr_lo = from->addr_lo; // already big endian + to->addr_hi = from->addr_hi; // likewise + wmb(); + to->len_gen = htobe32(V_FLD_GEN1(q->gen)); + to->gen2 = htobe32(V_FLD_GEN2(q->gen)); + q->credits++; + + if (++q->pidx == q->size) { + q->pidx = 0; + q->gen ^= 1; + } + t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); +} + +static void alloc_ring_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { uint32_t *addr; addr = arg; *addr = segs[0].ds_addr; } static int alloc_ring(adapter_t *sc, size_t nelem, size_t elem_size, size_t sw_size, bus_addr_t *phys, void *desc, void *sdesc, bus_dma_tag_t *tag, bus_dmamap_t *map, bus_dma_tag_t parent_entry_tag, bus_dma_tag_t *entry_tag) { size_t len = nelem * elem_size; void *s = NULL; void *p = NULL; int err; if ((err = bus_dma_tag_create(sc->parent_dmat, PAGE_SIZE, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, 0, NULL, NULL, tag)) != 0) { device_printf(sc->dev, "Cannot allocate descriptor tag\n"); return (ENOMEM); } if ((err = bus_dmamem_alloc(*tag, (void **)&p, BUS_DMA_NOWAIT, map)) != 0) { device_printf(sc->dev, "Cannot allocate descriptor memory\n"); return (ENOMEM); } bus_dmamap_load(*tag, *map, p, len, alloc_ring_cb, phys, 0); bzero(p, len); *(void **)desc = p; if (sw_size) { len = nelem * sw_size; s = malloc(len, M_DEVBUF, M_WAITOK); bzero(s, len); *(void **)sdesc = s; } if (parent_entry_tag == NULL) return (0); if ((err = bus_dma_tag_create(parent_entry_tag, 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, TX_MAX_SIZE, TX_MAX_SEGS, TX_MAX_SIZE, BUS_DMA_ALLOCNOW, NULL, NULL, entry_tag)) != 0) { device_printf(sc->dev, "Cannot allocate descriptor entry tag\n"); return (ENOMEM); } return (0); } static void sge_slow_intr_handler(void *arg, int ncount) { adapter_t *sc = arg; t3_slow_intr_handler(sc); } static void sge_timer_cb(void *arg) { adapter_t *sc = arg; struct sge_qset *qs; struct sge_txq *txq; int i, j; int reclaim_eth, reclaim_ofl, refill_rx; for (i = 0; i < sc->params.nports; i++) for (j = 0; j < sc->port[i].nqsets; j++) { qs = &sc->sge.qs[i + j]; txq = &qs->txq[0]; reclaim_eth = txq[TXQ_ETH].processed - txq[TXQ_ETH].cleaned; reclaim_ofl = txq[TXQ_OFLD].processed - txq[TXQ_OFLD].cleaned; refill_rx = ((qs->fl[0].credits < qs->fl[0].size) || (qs->fl[1].credits < qs->fl[1].size)); if (reclaim_eth || reclaim_ofl || refill_rx) { taskqueue_enqueue(sc->tq, &sc->timer_reclaim_task); goto done; } } done: callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); } /* * This is meant to be a catch-all function to keep sge state private * to sge.c * */ int t3_sge_init_sw(adapter_t *sc) { callout_init(&sc->sge_timer_ch, CALLOUT_MPSAFE); callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); TASK_INIT(&sc->timer_reclaim_task, 0, sge_timer_reclaim, sc); TASK_INIT(&sc->slow_intr_task, 0, sge_slow_intr_handler, sc); return (0); } void t3_sge_deinit_sw(adapter_t *sc) { callout_drain(&sc->sge_timer_ch); if (sc->tq) { taskqueue_drain(sc->tq, &sc->timer_reclaim_task); taskqueue_drain(sc->tq, &sc->slow_intr_task); } } /** * refill_rspq - replenish an SGE response queue * @adapter: the adapter * @q: the response queue to replenish * @credits: how many new responses to make available * * Replenishes a response queue by making the supplied number of responses * available to HW. */ static __inline void refill_rspq(adapter_t *sc, const struct sge_rspq *q, u_int credits) { /* mbufs are allocated on demand when a rspq entry is processed. */ t3_write_reg(sc, A_SG_RSPQ_CREDIT_RETURN, V_RSPQ(q->cntxt_id) | V_CREDITS(credits)); } static void sge_timer_reclaim(void *arg, int ncount) { adapter_t *sc = arg; int i, nqsets = 0; struct sge_qset *qs; struct sge_txq *txq; struct mtx *lock; struct mbuf *m_vec[TX_CLEAN_MAX_DESC]; int n, reclaimable; /* * XXX assuming these quantities are allowed to change during operation */ for (i = 0; i < sc->params.nports; i++) nqsets += sc->port[i].nqsets; for (i = 0; i < nqsets; i++) { qs = &sc->sge.qs[i]; txq = &qs->txq[TXQ_ETH]; reclaimable = desc_reclaimable(txq); if (reclaimable > 0) { mtx_lock(&txq->lock); n = reclaim_completed_tx(sc, txq, TX_CLEAN_MAX_DESC, m_vec); mtx_unlock(&txq->lock); - + for (i = 0; i < n; i++) { m_freem_vec(m_vec[i]); } } txq = &qs->txq[TXQ_OFLD]; reclaimable = desc_reclaimable(txq); if (reclaimable > 0) { mtx_lock(&txq->lock); n = reclaim_completed_tx(sc, txq, TX_CLEAN_MAX_DESC, m_vec); mtx_unlock(&txq->lock); for (i = 0; i < n; i++) { m_freem_vec(m_vec[i]); } } lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock : &sc->sge.qs[0].rspq.lock; if (mtx_trylock(lock)) { /* XXX currently assume that we are *NOT* polling */ uint32_t status = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS); if (qs->fl[0].credits < qs->fl[0].size - 16) __refill_fl(sc, &qs->fl[0]); if (qs->fl[1].credits < qs->fl[1].size - 16) __refill_fl(sc, &qs->fl[1]); if (status & (1 << qs->rspq.cntxt_id)) { if (qs->rspq.credits) { refill_rspq(sc, &qs->rspq, 1); qs->rspq.credits--; t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, 1 << qs->rspq.cntxt_id); } } mtx_unlock(lock); } } } /** * init_qset_cntxt - initialize an SGE queue set context info * @qs: the queue set * @id: the queue set id * * Initializes the TIDs and context ids for the queues of a queue set. */ static void init_qset_cntxt(struct sge_qset *qs, u_int id) { qs->rspq.cntxt_id = id; qs->fl[0].cntxt_id = 2 * id; qs->fl[1].cntxt_id = 2 * id + 1; qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id; qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id; qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id; qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id; qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id; } static void txq_prod(struct sge_txq *txq, unsigned int ndesc, struct txq_state *txqs) { txq->in_use += ndesc; /* * XXX we don't handle stopping of queue * presumably start handles this when we bump against the end */ txqs->gen = txq->gen; txq->unacked += ndesc; txqs->compl = (txq->unacked & 8) << (S_WR_COMPL - 3); txq->unacked &= 7; txqs->pidx = txq->pidx; txq->pidx += ndesc; if (txq->pidx >= txq->size) { txq->pidx -= txq->size; txq->gen ^= 1; } } /** * calc_tx_descs - calculate the number of Tx descriptors for a packet * @m: the packet mbufs * @nsegs: the number of segments * * Returns the number of Tx descriptors needed for the given Ethernet * packet. Ethernet packets require addition of WR and CPL headers. */ static __inline unsigned int calc_tx_descs(const struct mbuf *m, int nsegs) { unsigned int flits; if (m->m_pkthdr.len <= WR_LEN - sizeof(struct cpl_tx_pkt)) return 1; flits = sgl_len(nsegs) + 2; #ifdef TSO_SUPPORTED if (m->m_pkthdr.csum_flags & (CSUM_TSO)) flits++; #endif return flits_to_desc(flits); } static unsigned int busdma_map_mbufs(struct mbuf **m, struct sge_txq *txq, struct tx_sw_desc *stx, bus_dma_segment_t *segs, int *nsegs) { struct mbuf *m0; int err, pktlen; m0 = *m; pktlen = m0->m_pkthdr.len; err = bus_dmamap_load_mvec_sg(txq->entry_tag, stx->map, m0, segs, nsegs, 0); #ifdef DEBUG if (err) { int n = 0; struct mbuf *mtmp = m0; while(mtmp) { n++; mtmp = mtmp->m_next; } printf("map_mbufs: bus_dmamap_load_mbuf_sg failed with %d - pkthdr.len==%d nmbufs=%d\n", err, m0->m_pkthdr.len, n); } #endif if (err == EFBIG) { /* Too many segments, try to defrag */ m0 = m_defrag(m0, M_NOWAIT); if (m0 == NULL) { m_freem(*m); *m = NULL; return (ENOBUFS); } *m = m0; err = bus_dmamap_load_mbuf_sg(txq->entry_tag, stx->map, m0, segs, nsegs, 0); } if (err == ENOMEM) { return (err); } if (err) { if (cxgb_debug) printf("map failure err=%d pktlen=%d\n", err, pktlen); m_freem_vec(m0); *m = NULL; return (err); } bus_dmamap_sync(txq->entry_tag, stx->map, BUS_DMASYNC_PREWRITE); stx->flags |= TX_SW_DESC_MAPPED; return (0); } /** * make_sgl - populate a scatter/gather list for a packet * @sgp: the SGL to populate * @segs: the packet dma segments * @nsegs: the number of segments * * Generates a scatter/gather list for the buffers that make up a packet * and returns the SGL size in 8-byte words. The caller must size the SGL * appropriately. */ static __inline void make_sgl(struct sg_ent *sgp, bus_dma_segment_t *segs, int nsegs) { int i, idx; for (idx = 0, i = 0; i < nsegs; i++, idx ^= 1) { if (i && idx == 0) ++sgp; sgp->len[idx] = htobe32(segs[i].ds_len); sgp->addr[idx] = htobe64(segs[i].ds_addr); } if (idx) sgp->len[idx] = 0; } /** * check_ring_tx_db - check and potentially ring a Tx queue's doorbell * @adap: the adapter * @q: the Tx queue * * Ring the doorbel if a Tx queue is asleep. There is a natural race, * where the HW is going to sleep just after we checked, however, * then the interrupt handler will detect the outstanding TX packet * and ring the doorbell for us. * * When GTS is disabled we unconditionally ring the doorbell. */ static __inline void check_ring_tx_db(adapter_t *adap, struct sge_txq *q) { #if USE_GTS clear_bit(TXQ_LAST_PKT_DB, &q->flags); if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) { set_bit(TXQ_LAST_PKT_DB, &q->flags); #ifdef T3_TRACE T3_TRACE1(adap->tb[q->cntxt_id & 7], "doorbell Tx, cntxt %d", q->cntxt_id); #endif t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); } #else wmb(); /* write descriptors before telling HW */ t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); #endif } static __inline void wr_gen2(struct tx_desc *d, unsigned int gen) { #if SGE_NUM_GENBITS == 2 d->flit[TX_DESC_FLITS - 1] = htobe64(gen); #endif } + + +/** + * write_wr_hdr_sgl - write a WR header and, optionally, SGL + * @ndesc: number of Tx descriptors spanned by the SGL + * @txd: first Tx descriptor to be written + * @txqs: txq state (generation and producer index) + * @txq: the SGE Tx queue + * @sgl: the SGL + * @flits: number of flits to the start of the SGL in the first descriptor + * @sgl_flits: the SGL size in flits + * @wr_hi: top 32 bits of WR header based on WR type (big endian) + * @wr_lo: low 32 bits of WR header based on WR type (big endian) + * + * Write a work request header and an associated SGL. If the SGL is + * small enough to fit into one Tx descriptor it has already been written + * and we just need to write the WR header. Otherwise we distribute the + * SGL across the number of descriptors it spans. + */ + +static void +write_wr_hdr_sgl(unsigned int ndesc, struct tx_desc *txd, struct txq_state *txqs, + const struct sge_txq *txq, const struct sg_ent *sgl, unsigned int flits, + unsigned int sgl_flits, unsigned int wr_hi, unsigned int wr_lo) +{ + + struct work_request_hdr *wrp = (struct work_request_hdr *)txd; + struct tx_sw_desc *txsd = &txq->sdesc[txqs->pidx]; + + if (__predict_true(ndesc == 1)) { + wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) | + V_WR_SGLSFLT(flits)) | wr_hi; + wmb(); + wrp->wr_lo = htonl(V_WR_LEN(flits + sgl_flits) | + V_WR_GEN(txqs->gen)) | wr_lo; + /* XXX gen? */ + wr_gen2(txd, txqs->gen); + } else { + unsigned int ogen = txqs->gen; + const uint64_t *fp = (const uint64_t *)sgl; + struct work_request_hdr *wp = wrp; + + wrp->wr_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) | + V_WR_SGLSFLT(flits)) | wr_hi; + + while (sgl_flits) { + unsigned int avail = WR_FLITS - flits; + + if (avail > sgl_flits) + avail = sgl_flits; + memcpy(&txd->flit[flits], fp, avail * sizeof(*fp)); + sgl_flits -= avail; + ndesc--; + if (!sgl_flits) + break; + + fp += avail; + txd++; + txsd++; + if (++txqs->pidx == txq->size) { + txqs->pidx = 0; + txqs->gen ^= 1; + txd = txq->desc; + txsd = txq->sdesc; + } + + /* + * when the head of the mbuf chain + * is freed all clusters will be freed + * with it + */ + txsd->m = NULL; + wrp = (struct work_request_hdr *)txd; + wrp->wr_hi = htonl(V_WR_DATATYPE(1) | + V_WR_SGLSFLT(1)) | wr_hi; + wrp->wr_lo = htonl(V_WR_LEN(min(WR_FLITS, + sgl_flits + 1)) | + V_WR_GEN(txqs->gen)) | wr_lo; + wr_gen2(txd, txqs->gen); + flits = 1; + } + wrp->wr_hi |= htonl(F_WR_EOP); + wmb(); + wp->wr_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo; + wr_gen2((struct tx_desc *)wp, ogen); + } +} + + /* sizeof(*eh) + sizeof(*vhdr) + sizeof(*ip) + sizeof(*tcp) */ #define TCPPKTHDRSIZE (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + 20 + 20) int t3_encap(struct port_info *p, struct mbuf **m) { adapter_t *sc; struct mbuf *m0; struct sge_qset *qs; struct sge_txq *txq; struct tx_sw_desc *stx; struct txq_state txqs; unsigned int nsegs, ndesc, flits, cntrl, mlen; int err, tso_info = 0; struct work_request_hdr *wrp; struct tx_sw_desc *txsd; struct sg_ent *sgp, sgl[TX_MAX_SEGS / 2 + 1]; bus_dma_segment_t segs[TX_MAX_SEGS]; uint32_t wr_hi, wr_lo, sgl_flits; struct tx_desc *txd; struct cpl_tx_pkt *cpl; DPRINTF("t3_encap "); m0 = *m; sc = p->adapter; qs = &sc->sge.qs[p->first_qset]; txq = &qs->txq[TXQ_ETH]; stx = &txq->sdesc[txq->pidx]; txd = &txq->desc[txq->pidx]; cpl = (struct cpl_tx_pkt *)txd; mlen = m0->m_pkthdr.len; cpl->len = htonl(mlen | 0x80000000); DPRINTF("mlen=%d\n", mlen); /* * XXX handle checksum, TSO, and VLAN here * */ cntrl = V_TXPKT_INTF(p->port); /* * XXX need to add VLAN support for 6.x */ #ifdef VLAN_SUPPORTED if (m0->m_flags & M_VLANTAG) cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); if (m0->m_pkthdr.csum_flags & (CSUM_TSO)) tso_info = V_LSO_MSS(m0->m_pkthdr.tso_segsz); #endif if (tso_info) { int eth_type; struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *) cpl; struct ip *ip; struct tcphdr *tcp; uint8_t *pkthdr, tmp[TCPPKTHDRSIZE]; /* is this too large for the stack? */ txd->flit[2] = 0; cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO); hdr->cntrl = htonl(cntrl); if (__predict_false(m0->m_len < TCPPKTHDRSIZE)) { pkthdr = &tmp[0]; m_copydata(m0, 0, TCPPKTHDRSIZE, pkthdr); } else { pkthdr = m0->m_data; } if (__predict_false(m0->m_flags & M_VLANTAG)) { eth_type = CPL_ETH_II_VLAN; ip = (struct ip *)(pkthdr + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); } else { eth_type = CPL_ETH_II; ip = (struct ip *)(pkthdr + ETHER_HDR_LEN); } tcp = (struct tcphdr *)((uint8_t *)ip + sizeof(*ip)); tso_info |= V_LSO_ETH_TYPE(eth_type) | V_LSO_IPHDR_WORDS(ip->ip_hl) | V_LSO_TCPHDR_WORDS(tcp->th_off); hdr->lso_info = htonl(tso_info); flits = 3; } else { cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT); cpl->cntrl = htonl(cntrl); if (mlen <= WR_LEN - sizeof(*cpl)) { txq_prod(txq, 1, &txqs); txq->sdesc[txqs.pidx].m = m0; + m_set_priority(m0, txqs.pidx); if (m0->m_len == m0->m_pkthdr.len) memcpy(&txd->flit[2], m0->m_data, mlen); else m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]); flits = (mlen + 7) / 8 + 2; cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) | V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | F_WR_SOP | F_WR_EOP | txqs.compl); wmb(); cpl->wr.wr_lo = htonl(V_WR_LEN(flits) | V_WR_GEN(txqs.gen) | V_WR_TID(txq->token)); wr_gen2(txd, txqs.gen); check_ring_tx_db(sc, txq); return (0); } flits = 2; } wrp = (struct work_request_hdr *)txd; if ((err = busdma_map_mbufs(m, txq, stx, segs, &nsegs)) != 0) { return (err); } m0 = *m; ndesc = calc_tx_descs(m0, nsegs); - sgp = (ndesc == 1) ? (struct sg_ent *)&txd->flit[flits] : &sgl[0]; + sgp = (ndesc == 1) ? (struct sg_ent *)&txd->flit[flits] : sgl; make_sgl(sgp, segs, nsegs); sgl_flits = sgl_len(nsegs); DPRINTF("make_sgl success nsegs==%d ndesc==%d\n", nsegs, ndesc); txq_prod(txq, ndesc, &txqs); txsd = &txq->sdesc[txqs.pidx]; wr_hi = htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl); wr_lo = htonl(V_WR_TID(txq->token)); txsd->m = m0; - - if (__predict_true(ndesc == 1)) { - wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) | - V_WR_SGLSFLT(flits)) | wr_hi; - wmb(); - wrp->wr_lo = htonl(V_WR_LEN(flits + sgl_flits) | - V_WR_GEN(txqs.gen)) | wr_lo; - /* XXX gen? */ - wr_gen2(txd, txqs.gen); - } else { - unsigned int ogen = txqs.gen; - const uint64_t *fp = (const uint64_t *)sgl; - struct work_request_hdr *wp = wrp; - - /* XXX - CHECK ME */ - wrp->wr_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) | - V_WR_SGLSFLT(flits)) | wr_hi; - - while (sgl_flits) { - unsigned int avail = WR_FLITS - flits; + m_set_priority(m0, txqs.pidx); - if (avail > sgl_flits) - avail = sgl_flits; - memcpy(&txd->flit[flits], fp, avail * sizeof(*fp)); - sgl_flits -= avail; - ndesc--; - if (!sgl_flits) - break; - - fp += avail; - txd++; - txsd++; - if (++txqs.pidx == txq->size) { - txqs.pidx = 0; - txqs.gen ^= 1; - txd = txq->desc; - txsd = txq->sdesc; - } - - /* - * when the head of the mbuf chain - * is freed all clusters will be freed - * with it - */ - txsd->m = NULL; - wrp = (struct work_request_hdr *)txd; - wrp->wr_hi = htonl(V_WR_DATATYPE(1) | - V_WR_SGLSFLT(1)) | wr_hi; - wrp->wr_lo = htonl(V_WR_LEN(min(WR_FLITS, - sgl_flits + 1)) | - V_WR_GEN(txqs.gen)) | wr_lo; - wr_gen2(txd, txqs.gen); - flits = 1; - } -#ifdef WHY - skb->priority = pidx; -#endif - wrp->wr_hi |= htonl(F_WR_EOP); - wmb(); - wp->wr_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo; - wr_gen2((struct tx_desc *)wp, ogen); - } + write_wr_hdr_sgl(ndesc, txd, &txqs, txq, sgl, flits, sgl_flits, wr_hi, wr_lo); check_ring_tx_db(p->adapter, txq); return (0); } /** * write_imm - write a packet into a Tx descriptor as immediate data * @d: the Tx descriptor to write * @m: the packet * @len: the length of packet data to write as immediate data * @gen: the generation bit value to write * * Writes a packet as immediate data into a Tx descriptor. The packet * contains a work request at its beginning. We must write the packet * carefully so the SGE doesn't read accidentally before it's written in * its entirety. */ -static __inline void write_imm(struct tx_desc *d, struct mbuf *m, - unsigned int len, unsigned int gen) +static __inline void +write_imm(struct tx_desc *d, struct mbuf *m, + unsigned int len, unsigned int gen) { - struct work_request_hdr *from = (struct work_request_hdr *)m->m_data; + struct work_request_hdr *from = mtod(m, struct work_request_hdr *); struct work_request_hdr *to = (struct work_request_hdr *)d; memcpy(&to[1], &from[1], len - sizeof(*from)); to->wr_hi = from->wr_hi | htonl(F_WR_SOP | F_WR_EOP | V_WR_BCNTLFLT(len & 7)); wmb(); to->wr_lo = from->wr_lo | htonl(V_WR_GEN(gen) | V_WR_LEN((len + 7) / 8)); wr_gen2(d, gen); m_freem(m); } /** * check_desc_avail - check descriptor availability on a send queue * @adap: the adapter * @q: the TX queue * @m: the packet needing the descriptors * @ndesc: the number of Tx descriptors needed * @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL) * * Checks if the requested number of Tx descriptors is available on an * SGE send queue. If the queue is already suspended or not enough * descriptors are available the packet is queued for later transmission. * Must be called with the Tx queue locked. * * Returns 0 if enough descriptors are available, 1 if there aren't * enough descriptors and the packet has been queued, and 2 if the caller * needs to retry because there weren't enough descriptors at the * beginning of the call but some freed up in the mean time. */ static __inline int check_desc_avail(adapter_t *adap, struct sge_txq *q, - struct mbuf *m, unsigned int ndesc, - unsigned int qid) + struct mbuf *m, unsigned int ndesc, + unsigned int qid) { /* * XXX We currently only use this for checking the control queue * the control queue is only used for binding qsets which happens * at init time so we are guaranteed enough descriptors */ -#if 0 - if (__predict_false(!skb_queue_empty(&q->sendq))) { -addq_exit: __skb_queue_tail(&q->sendq, skb); + if (__predict_false(!mbufq_empty(&q->sendq))) { +addq_exit: mbufq_tail(&q->sendq, m); return 1; } if (__predict_false(q->size - q->in_use < ndesc)) { struct sge_qset *qs = txq_to_qset(q, qid); - set_bit(qid, &qs->txq_stopped); - smp_mb__after_clear_bit(); + setbit(&qs->txq_stopped, qid); + smp_mb(); if (should_restart_tx(q) && test_and_clear_bit(qid, &qs->txq_stopped)) return 2; q->stops++; goto addq_exit; } -#endif return 0; } /** * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs * @q: the SGE control Tx queue * * This is a variant of reclaim_completed_tx() that is used for Tx queues * that send only immediate data (presently just the control queues) and - * thus do not have any sk_buffs to release. + * thus do not have any mbufs */ static __inline void reclaim_completed_tx_imm(struct sge_txq *q) { unsigned int reclaim = q->processed - q->cleaned; mtx_assert(&q->lock, MA_OWNED); q->in_use -= reclaim; q->cleaned += reclaim; } static __inline int immediate(const struct mbuf *m) { return m->m_len <= WR_LEN && m->m_pkthdr.len <= WR_LEN ; } /** * ctrl_xmit - send a packet through an SGE control Tx queue * @adap: the adapter * @q: the control queue * @m: the packet * * Send a packet through an SGE control Tx queue. Packets sent through * a control queue must fit entirely as immediate data in a single Tx * descriptor and have no page fragments. */ static int ctrl_xmit(adapter_t *adap, struct sge_txq *q, struct mbuf *m) { int ret; struct work_request_hdr *wrp = (struct work_request_hdr *)m->m_data; if (__predict_false(!immediate(m))) { m_freem(m); return 0; } wrp->wr_hi |= htonl(F_WR_SOP | F_WR_EOP); wrp->wr_lo = htonl(V_WR_TID(q->token)); mtx_lock(&q->lock); again: reclaim_completed_tx_imm(q); ret = check_desc_avail(adap, q, m, 1, TXQ_CTRL); if (__predict_false(ret)) { if (ret == 1) { mtx_unlock(&q->lock); return (-1); } goto again; } write_imm(&q->desc[q->pidx], m, m->m_len, q->gen); q->in_use++; if (++q->pidx >= q->size) { q->pidx = 0; q->gen ^= 1; } mtx_unlock(&q->lock); wmb(); t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); return (0); } -#ifdef RESTART_CTRLQ + /** * restart_ctrlq - restart a suspended control queue * @qs: the queue set cotaining the control queue * * Resumes transmission on a suspended Tx control queue. */ static void -restart_ctrlq(unsigned long data) +restart_ctrlq(void *data, int npending) { struct mbuf *m; struct sge_qset *qs = (struct sge_qset *)data; struct sge_txq *q = &qs->txq[TXQ_CTRL]; adapter_t *adap = qs->port->adapter; mtx_lock(&q->lock); again: reclaim_completed_tx_imm(q); - + while (q->in_use < q->size && - (skb = __skb_dequeue(&q->sendq)) != NULL) { + (m = mbufq_dequeue(&q->sendq)) != NULL) { - write_imm(&q->desc[q->pidx], skb, skb->len, q->gen); + write_imm(&q->desc[q->pidx], m, m->m_len, q->gen); if (++q->pidx >= q->size) { q->pidx = 0; q->gen ^= 1; } q->in_use++; } - if (!skb_queue_empty(&q->sendq)) { - set_bit(TXQ_CTRL, &qs->txq_stopped); - smp_mb__after_clear_bit(); + if (!mbufq_empty(&q->sendq)) { + setbit(&qs->txq_stopped, TXQ_CTRL); + smp_mb(); if (should_restart_tx(q) && test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) goto again; q->stops++; } - mtx_unlock(&q->lock); t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); } -#endif + /* * Send a management message through control queue 0 */ int t3_mgmt_tx(struct adapter *adap, struct mbuf *m) { return ctrl_xmit(adap, &adap->sge.qs[0].txq[TXQ_CTRL], m); } /** - * t3_sge_alloc_qset - initialize an SGE queue set - * @sc: the controller softc - * @id: the queue set id - * @nports: how many Ethernet ports will be using this queue set - * @irq_vec_idx: the IRQ vector index for response queue interrupts - * @p: configuration parameters for this queue set - * @ntxq: number of Tx queues for the queue set - * @pi: port info for queue set - * - * Allocate resources and initialize an SGE queue set. A queue set - * comprises a response queue, two Rx free-buffer queues, and up to 3 - * Tx queues. The Tx queues are assigned roles in the order Ethernet - * queue, offload queue, and control queue. - */ -int -t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx, - const struct qset_params *p, int ntxq, struct port_info *pi) -{ - struct sge_qset *q = &sc->sge.qs[id]; - int i, ret = 0; - - init_qset_cntxt(q, id); - - if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc), - sizeof(struct rx_sw_desc), &q->fl[0].phys_addr, - &q->fl[0].desc, &q->fl[0].sdesc, - &q->fl[0].desc_tag, &q->fl[0].desc_map, - sc->rx_dmat, &q->fl[0].entry_tag)) != 0) { - printf("error %d from alloc ring fl0\n", ret); - goto err; - } - - if ((ret = alloc_ring(sc, p->jumbo_size, sizeof(struct rx_desc), - sizeof(struct rx_sw_desc), &q->fl[1].phys_addr, - &q->fl[1].desc, &q->fl[1].sdesc, - &q->fl[1].desc_tag, &q->fl[1].desc_map, - sc->rx_jumbo_dmat, &q->fl[1].entry_tag)) != 0) { - printf("error %d from alloc ring fl1\n", ret); - goto err; - } - - if ((ret = alloc_ring(sc, p->rspq_size, sizeof(struct rsp_desc), 0, - &q->rspq.phys_addr, &q->rspq.desc, NULL, - &q->rspq.desc_tag, &q->rspq.desc_map, - NULL, NULL)) != 0) { - printf("error %d from alloc ring rspq\n", ret); - goto err; - } - - for (i = 0; i < ntxq; ++i) { - /* - * The control queue always uses immediate data so does not - * need to keep track of any mbufs. - * XXX Placeholder for future TOE support. - */ - size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc); - - if ((ret = alloc_ring(sc, p->txq_size[i], - sizeof(struct tx_desc), sz, - &q->txq[i].phys_addr, &q->txq[i].desc, - &q->txq[i].sdesc, &q->txq[i].desc_tag, - &q->txq[i].desc_map, - sc->tx_dmat, &q->txq[i].entry_tag)) != 0) { - printf("error %d from alloc ring tx %i\n", ret, i); - goto err; - } - q->txq[i].gen = 1; - q->txq[i].size = p->txq_size[i]; - mtx_init(&q->txq[i].lock, "t3 txq lock", NULL, MTX_DEF); - } - - q->fl[0].gen = q->fl[1].gen = 1; - q->fl[0].size = p->fl_size; - q->fl[1].size = p->jumbo_size; - - q->rspq.gen = 1; - q->rspq.size = p->rspq_size; - mtx_init(&q->rspq.lock, "t3 rspq lock", NULL, MTX_DEF); - - q->txq[TXQ_ETH].stop_thres = nports * - flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3); - - q->fl[0].buf_size = MCLBYTES; - q->fl[0].zone = zone_clust; - q->fl[0].type = EXT_CLUSTER; - q->fl[1].buf_size = MJUMPAGESIZE; - q->fl[1].zone = zone_jumbop; - q->fl[1].type = EXT_JUMBOP; - - q->lro.enabled = lro_default; - - mtx_lock(&sc->sge.reg_lock); - ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx, - q->rspq.phys_addr, q->rspq.size, - q->fl[0].buf_size, 1, 0); - if (ret) { - printf("error %d from t3_sge_init_rspcntxt\n", ret); - goto err_unlock; - } - - for (i = 0; i < SGE_RXQ_PER_SET; ++i) { - ret = -t3_sge_init_flcntxt(sc, q->fl[i].cntxt_id, 0, - q->fl[i].phys_addr, q->fl[i].size, - q->fl[i].buf_size, p->cong_thres, 1, - 0); - if (ret) { - printf("error %d from t3_sge_init_flcntxt for index i=%d\n", ret, i); - goto err_unlock; - } - } - - ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_ETH].cntxt_id, USE_GTS, - SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr, - q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token, - 1, 0); - if (ret) { - printf("error %d from t3_sge_init_ecntxt\n", ret); - goto err_unlock; - } - - if (ntxq > 1) { - ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_OFLD].cntxt_id, - USE_GTS, SGE_CNTXT_OFLD, id, - q->txq[TXQ_OFLD].phys_addr, - q->txq[TXQ_OFLD].size, 0, 1, 0); - if (ret) { - printf("error %d from t3_sge_init_ecntxt\n", ret); - goto err_unlock; - } - } - - if (ntxq > 2) { - ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_CTRL].cntxt_id, 0, - SGE_CNTXT_CTRL, id, - q->txq[TXQ_CTRL].phys_addr, - q->txq[TXQ_CTRL].size, - q->txq[TXQ_CTRL].token, 1, 0); - if (ret) { - printf("error %d from t3_sge_init_ecntxt\n", ret); - goto err_unlock; - } - } - - mtx_unlock(&sc->sge.reg_lock); - t3_update_qset_coalesce(q, p); - q->port = pi; - - refill_fl(sc, &q->fl[0], q->fl[0].size); - refill_fl(sc, &q->fl[1], q->fl[1].size); - refill_rspq(sc, &q->rspq, q->rspq.size - 1); - - t3_write_reg(sc, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) | - V_NEWTIMER(q->rspq.holdoff_tmr)); - - return (0); - -err_unlock: - mtx_unlock(&sc->sge.reg_lock); -err: - t3_free_qset(sc, q); - - return (ret); -} - - -/** * free_qset - free the resources of an SGE queue set * @sc: the controller owning the queue set * @q: the queue set * * Release the HW and SW resources associated with an SGE queue set, such * as HW contexts, packet buffers, and descriptor rings. Traffic to the * queue set must be quiesced prior to calling this. */ static void t3_free_qset(adapter_t *sc, struct sge_qset *q) { int i; for (i = 0; i < SGE_RXQ_PER_SET; ++i) { if (q->fl[i].desc) { mtx_lock(&sc->sge.reg_lock); t3_sge_disable_fl(sc, q->fl[i].cntxt_id); mtx_unlock(&sc->sge.reg_lock); bus_dmamap_unload(q->fl[i].desc_tag, q->fl[i].desc_map); bus_dmamem_free(q->fl[i].desc_tag, q->fl[i].desc, q->fl[i].desc_map); bus_dma_tag_destroy(q->fl[i].desc_tag); bus_dma_tag_destroy(q->fl[i].entry_tag); } if (q->fl[i].sdesc) { free_rx_bufs(sc, &q->fl[i]); free(q->fl[i].sdesc, M_DEVBUF); } } for (i = 0; i < SGE_TXQ_PER_SET; ++i) { if (q->txq[i].desc) { mtx_lock(&sc->sge.reg_lock); t3_sge_enable_ecntxt(sc, q->txq[i].cntxt_id, 0); mtx_unlock(&sc->sge.reg_lock); bus_dmamap_unload(q->txq[i].desc_tag, q->txq[i].desc_map); bus_dmamem_free(q->txq[i].desc_tag, q->txq[i].desc, q->txq[i].desc_map); bus_dma_tag_destroy(q->txq[i].desc_tag); bus_dma_tag_destroy(q->txq[i].entry_tag); } if (q->txq[i].sdesc) { free(q->txq[i].sdesc, M_DEVBUF); } if (mtx_initialized(&q->txq[i].lock)) { mtx_destroy(&q->txq[i].lock); } } if (q->rspq.desc) { mtx_lock(&sc->sge.reg_lock); t3_sge_disable_rspcntxt(sc, q->rspq.cntxt_id); mtx_unlock(&sc->sge.reg_lock); bus_dmamap_unload(q->rspq.desc_tag, q->rspq.desc_map); bus_dmamem_free(q->rspq.desc_tag, q->rspq.desc, q->rspq.desc_map); bus_dma_tag_destroy(q->rspq.desc_tag); } if (mtx_initialized(&q->rspq.lock)) mtx_destroy(&q->rspq.lock); bzero(q, sizeof(*q)); } /** * t3_free_sge_resources - free SGE resources * @sc: the adapter softc * * Frees resources used by the SGE queue sets. */ void t3_free_sge_resources(adapter_t *sc) { int i; for (i = 0; i < SGE_QSETS; ++i) t3_free_qset(sc, &sc->sge.qs[i]); } /** * t3_sge_start - enable SGE * @sc: the controller softc * * Enables the SGE for DMAs. This is the last step in starting packet * transfers. */ void t3_sge_start(adapter_t *sc) { t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE); } +/** + * t3_sge_stop - disable SGE operation + * @sc: the adapter + * + * Disables the DMA engine. This can be called in emeregencies (e.g., + * from error interrupts) or from normal process context. In the latter + * case it also disables any pending queue restart tasklets. Note that + * if it is called in interrupt context it cannot disable the restart + * tasklets as it cannot wait, however the tasklets will have no effect + * since the doorbells are disabled and the driver will call this again + * later from process context, at which time the tasklets will be stopped + * if they are still running. + */ +void +t3_sge_stop(adapter_t *sc) +{ + int i; + t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, 0); + for (i = 0; i < SGE_QSETS; ++i) { + struct sge_qset *qs = &sc->sge.qs[i]; + + taskqueue_drain(sc->tq, &qs->txq[TXQ_OFLD].qresume_tsk); + taskqueue_drain(sc->tq, &qs->txq[TXQ_CTRL].qresume_tsk); + } +} + + /** * free_tx_desc - reclaims Tx descriptors and their buffers * @adapter: the adapter * @q: the Tx queue to reclaim descriptors from * @n: the number of descriptors to reclaim * * Reclaims Tx descriptors from an SGE Tx queue and frees the associated * Tx buffers. Called with the Tx queue lock held. */ int free_tx_desc(adapter_t *sc, struct sge_txq *q, int n, struct mbuf **m_vec) { struct tx_sw_desc *d; unsigned int cidx = q->cidx; int nbufs = 0; #ifdef T3_TRACE T3_TRACE2(sc->tb[q->cntxt_id & 7], "reclaiming %u Tx descriptors at cidx %u", n, cidx); #endif d = &q->sdesc[cidx]; while (n-- > 0) { DPRINTF("cidx=%d d=%p\n", cidx, d); if (d->m) { if (d->flags & TX_SW_DESC_MAPPED) { bus_dmamap_unload(q->entry_tag, d->map); bus_dmamap_destroy(q->entry_tag, d->map); d->flags &= ~TX_SW_DESC_MAPPED; } - m_vec[nbufs] = d->m; - d->m = NULL; - nbufs++; + if (m_get_priority(d->m) == cidx) { + m_vec[nbufs] = d->m; + d->m = NULL; + nbufs++; + } else { + printf("pri=%d cidx=%d\n", m_get_priority(d->m), cidx); + } } ++d; if (++cidx == q->size) { cidx = 0; d = q->sdesc; } } q->cidx = cidx; return (nbufs); } /** * is_new_response - check if a response is newly written * @r: the response descriptor * @q: the response queue * * Returns true if a response descriptor contains a yet unprocessed * response. */ static __inline int is_new_response(const struct rsp_desc *r, const struct sge_rspq *q) { return (r->intr_gen & F_RSPD_GEN2) == q->gen; } #define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS) #define RSPD_CTRL_MASK (RSPD_GTS_MASK | \ V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \ V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \ V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR)) /* How long to delay the next interrupt in case of memory shortage, in 0.1us. */ #define NOMEM_INTR_DELAY 2500 -static __inline void -deliver_partial_bundle(struct t3cdev *tdev, struct sge_rspq *q) +/** + * write_ofld_wr - write an offload work request + * @adap: the adapter + * @m: the packet to send + * @q: the Tx queue + * @pidx: index of the first Tx descriptor to write + * @gen: the generation value to use + * @ndesc: number of descriptors the packet will occupy + * + * Write an offload work request to send the supplied packet. The packet + * data already carry the work request with most fields populated. + */ +static void +write_ofld_wr(adapter_t *adap, struct mbuf *m, + struct sge_txq *q, unsigned int pidx, + unsigned int gen, unsigned int ndesc, + bus_dma_segment_t *segs, unsigned int nsegs) { - ; + unsigned int sgl_flits, flits; + struct work_request_hdr *from; + struct sg_ent *sgp, sgl[TX_MAX_SEGS / 2 + 1]; + struct tx_desc *d = &q->desc[pidx]; + struct txq_state txqs; + + if (immediate(m)) { + q->sdesc[pidx].m = NULL; + write_imm(d, m, m->m_len, gen); + return; + } + + /* Only TX_DATA builds SGLs */ + + from = mtod(m, struct work_request_hdr *); + memcpy(&d->flit[1], &from[1], + (uint8_t *)m->m_pkthdr.header - mtod(m, uint8_t *) - sizeof(*from)); + + flits = ((uint8_t *)m->m_pkthdr.header - mtod(m, uint8_t *)) / 8; + sgp = (ndesc == 1) ? (struct sg_ent *)&d->flit[flits] : sgl; + + make_sgl(sgp, segs, nsegs); + sgl_flits = sgl_len(nsegs); + + txqs.gen = q->gen; + txqs.pidx = q->pidx; + txqs.compl = (q->unacked & 8) << (S_WR_COMPL - 3); + write_wr_hdr_sgl(ndesc, d, &txqs, q, sgl, flits, sgl_flits, + from->wr_hi, from->wr_lo); } -static __inline void -rx_offload(struct t3cdev *tdev, struct sge_rspq *rq, - struct mbuf *m) +/** + * calc_tx_descs_ofld - calculate # of Tx descriptors for an offload packet + * @m: the packet + * + * Returns the number of Tx descriptors needed for the given offload + * packet. These packets are already fully constructed. + */ +static __inline unsigned int +calc_tx_descs_ofld(struct mbuf *m, unsigned int nsegs) { -#ifdef notyet - if (rq->polling) { - rq->offload_skbs[rq->offload_skbs_idx++] = skb; - if (rq->offload_skbs_idx == RX_BUNDLE_SIZE) { - cxgb_ofld_recv(tdev, rq->offload_skbs, RX_BUNDLE_SIZE); - rq->offload_skbs_idx = 0; - rq->offload_bundles++; + unsigned int flits, cnt = 0; + + + if (m->m_len <= WR_LEN) + return 1; /* packet fits as immediate data */ + + if (m->m_flags & M_IOVEC) + cnt = mtomv(m)->mv_count; + + flits = ((uint8_t *)m->m_pkthdr.header - mtod(m, uint8_t *)) / 8; /* headers */ + + return flits_to_desc(flits + sgl_len(cnt)); +} + +/** + * ofld_xmit - send a packet through an offload queue + * @adap: the adapter + * @q: the Tx offload queue + * @m: the packet + * + * Send an offload packet through an SGE offload queue. + */ +static int +ofld_xmit(adapter_t *adap, struct sge_txq *q, struct mbuf *m) +{ + int ret; + unsigned int pidx, gen, nsegs; + unsigned int ndesc; + struct mbuf *m_vec[TX_CLEAN_MAX_DESC]; + bus_dma_segment_t segs[TX_MAX_SEGS]; + int i, cleaned; + struct tx_sw_desc *stx = &q->sdesc[q->pidx]; + + mtx_lock(&q->lock); + if ((ret = busdma_map_mbufs(&m, q, stx, segs, &nsegs)) != 0) { + mtx_unlock(&q->lock); + return (ret); + } + ndesc = calc_tx_descs_ofld(m, nsegs); +again: cleaned = reclaim_completed_tx(adap, q, TX_CLEAN_MAX_DESC, m_vec); + + ret = check_desc_avail(adap, q, m, ndesc, TXQ_OFLD); + if (__predict_false(ret)) { + if (ret == 1) { + m_set_priority(m, ndesc); /* save for restart */ + mtx_unlock(&q->lock); + return NET_XMIT_CN; } - } else + goto again; + } + + gen = q->gen; + q->in_use += ndesc; + pidx = q->pidx; + q->pidx += ndesc; + if (q->pidx >= q->size) { + q->pidx -= q->size; + q->gen ^= 1; + } +#ifdef T3_TRACE + T3_TRACE5(adap->tb[q->cntxt_id & 7], + "ofld_xmit: ndesc %u, pidx %u, len %u, main %u, frags %u", + ndesc, pidx, skb->len, skb->len - skb->data_len, + skb_shinfo(skb)->nr_frags); #endif - { - /* XXX */ - panic("implement offload enqueue\n"); + mtx_unlock(&q->lock); + + write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs); + check_ring_tx_db(adap, q); + + for (i = 0; i < cleaned; i++) { + m_freem_vec(m_vec[i]); } + return NET_XMIT_SUCCESS; +} +/** + * restart_offloadq - restart a suspended offload queue + * @qs: the queue set cotaining the offload queue + * + * Resumes transmission on a suspended Tx offload queue. + */ +static void +restart_offloadq(void *data, int npending) +{ + + struct mbuf *m; + struct sge_qset *qs = data; + struct sge_txq *q = &qs->txq[TXQ_OFLD]; + adapter_t *adap = qs->port->adapter; + struct mbuf *m_vec[TX_CLEAN_MAX_DESC]; + bus_dma_segment_t segs[TX_MAX_SEGS]; + int nsegs, i, cleaned; + struct tx_sw_desc *stx = &q->sdesc[q->pidx]; + + mtx_lock(&q->lock); +again: cleaned = reclaim_completed_tx(adap, q, TX_CLEAN_MAX_DESC, m_vec); + + while ((m = mbufq_peek(&q->sendq)) != NULL) { + unsigned int gen, pidx; + unsigned int ndesc = m_get_priority(m); + + if (__predict_false(q->size - q->in_use < ndesc)) { + setbit(&qs->txq_stopped, TXQ_OFLD); + smp_mb(); + + if (should_restart_tx(q) && + test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) + goto again; + q->stops++; + break; + } + + gen = q->gen; + q->in_use += ndesc; + pidx = q->pidx; + q->pidx += ndesc; + if (q->pidx >= q->size) { + q->pidx -= q->size; + q->gen ^= 1; + } + + (void)mbufq_dequeue(&q->sendq); + busdma_map_mbufs(&m, q, stx, segs, &nsegs); + mtx_unlock(&q->lock); + write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs); + mtx_lock(&q->lock); + } + mtx_unlock(&q->lock); + +#if USE_GTS + set_bit(TXQ_RUNNING, &q->flags); + set_bit(TXQ_LAST_PKT_DB, &q->flags); +#endif + t3_write_reg(adap, A_SG_KDOORBELL, + F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); + + for (i = 0; i < cleaned; i++) { + m_freem_vec(m_vec[i]); + } } +/** + * queue_set - return the queue set a packet should use + * @m: the packet + * + * Maps a packet to the SGE queue set it should use. The desired queue + * set is carried in bits 1-3 in the packet's priority. + */ +static __inline int +queue_set(const struct mbuf *m) +{ + return m_get_priority(m) >> 1; +} + +/** + * is_ctrl_pkt - return whether an offload packet is a control packet + * @m: the packet + * + * Determines whether an offload packet should use an OFLD or a CTRL + * Tx queue. This is indicated by bit 0 in the packet's priority. + */ +static __inline int +is_ctrl_pkt(const struct mbuf *m) +{ + return m_get_priority(m) & 1; +} + +/** + * t3_offload_tx - send an offload packet + * @tdev: the offload device to send to + * @m: the packet + * + * Sends an offload packet. We use the packet priority to select the + * appropriate Tx queue as follows: bit 0 indicates whether the packet + * should be sent as regular or control, bits 1-3 select the queue set. + */ +int +t3_offload_tx(struct toedev *tdev, struct mbuf *m) +{ + adapter_t *adap = tdev2adap(tdev); + struct sge_qset *qs = &adap->sge.qs[queue_set(m)]; + + if (__predict_false(is_ctrl_pkt(m))) + return ctrl_xmit(adap, &qs->txq[TXQ_CTRL], m); + + return ofld_xmit(adap, &qs->txq[TXQ_OFLD], m); +} + +/** + * deliver_partial_bundle - deliver a (partial) bundle of Rx offload pkts + * @tdev: the offload device that will be receiving the packets + * @q: the SGE response queue that assembled the bundle + * @m: the partial bundle + * @n: the number of packets in the bundle + * + * Delivers a (partial) bundle of Rx offload packets to an offload device. + */ +static __inline void +deliver_partial_bundle(struct toedev *tdev, + struct sge_rspq *q, + struct mbuf *mbufs[], int n) +{ + if (n) { + q->offload_bundles++; + cxgb_ofld_recv(tdev, mbufs, n); + } +} + +static __inline int +rx_offload(struct toedev *tdev, struct sge_rspq *rq, + struct mbuf *m, struct mbuf *rx_gather[], + unsigned int gather_idx) +{ + rq->offload_pkts++; + m->m_pkthdr.header = mtod(m, void *); + + rx_gather[gather_idx++] = m; + if (gather_idx == RX_BUNDLE_SIZE) { + cxgb_ofld_recv(tdev, rx_gather, RX_BUNDLE_SIZE); + gather_idx = 0; + rq->offload_bundles++; + } + return (gather_idx); +} + static void restart_tx(struct sge_qset *qs) { - ; + struct adapter *sc = qs->port->adapter; + + if (isset(&qs->txq_stopped, TXQ_OFLD) && + should_restart_tx(&qs->txq[TXQ_OFLD]) && + test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) { + qs->txq[TXQ_OFLD].restarts++; + taskqueue_enqueue(sc->tq, &qs->txq[TXQ_OFLD].qresume_tsk); + } + if (isset(&qs->txq_stopped, TXQ_CTRL) && + should_restart_tx(&qs->txq[TXQ_CTRL]) && + test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) { + qs->txq[TXQ_CTRL].restarts++; + taskqueue_enqueue(sc->tq, &qs->txq[TXQ_CTRL].qresume_tsk); + } } +/** + * t3_sge_alloc_qset - initialize an SGE queue set + * @sc: the controller softc + * @id: the queue set id + * @nports: how many Ethernet ports will be using this queue set + * @irq_vec_idx: the IRQ vector index for response queue interrupts + * @p: configuration parameters for this queue set + * @ntxq: number of Tx queues for the queue set + * @pi: port info for queue set + * + * Allocate resources and initialize an SGE queue set. A queue set + * comprises a response queue, two Rx free-buffer queues, and up to 3 + * Tx queues. The Tx queues are assigned roles in the order Ethernet + * queue, offload queue, and control queue. + */ +int +t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx, + const struct qset_params *p, int ntxq, struct port_info *pi) +{ + struct sge_qset *q = &sc->sge.qs[id]; + int i, ret = 0; + + init_qset_cntxt(q, id); + + if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc), + sizeof(struct rx_sw_desc), &q->fl[0].phys_addr, + &q->fl[0].desc, &q->fl[0].sdesc, + &q->fl[0].desc_tag, &q->fl[0].desc_map, + sc->rx_dmat, &q->fl[0].entry_tag)) != 0) { + printf("error %d from alloc ring fl0\n", ret); + goto err; + } + + if ((ret = alloc_ring(sc, p->jumbo_size, sizeof(struct rx_desc), + sizeof(struct rx_sw_desc), &q->fl[1].phys_addr, + &q->fl[1].desc, &q->fl[1].sdesc, + &q->fl[1].desc_tag, &q->fl[1].desc_map, + sc->rx_jumbo_dmat, &q->fl[1].entry_tag)) != 0) { + printf("error %d from alloc ring fl1\n", ret); + goto err; + } + + if ((ret = alloc_ring(sc, p->rspq_size, sizeof(struct rsp_desc), 0, + &q->rspq.phys_addr, &q->rspq.desc, NULL, + &q->rspq.desc_tag, &q->rspq.desc_map, + NULL, NULL)) != 0) { + printf("error %d from alloc ring rspq\n", ret); + goto err; + } + + for (i = 0; i < ntxq; ++i) { + /* + * The control queue always uses immediate data so does not + * need to keep track of any mbufs. + * XXX Placeholder for future TOE support. + */ + size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc); + + if ((ret = alloc_ring(sc, p->txq_size[i], + sizeof(struct tx_desc), sz, + &q->txq[i].phys_addr, &q->txq[i].desc, + &q->txq[i].sdesc, &q->txq[i].desc_tag, + &q->txq[i].desc_map, + sc->tx_dmat, &q->txq[i].entry_tag)) != 0) { + printf("error %d from alloc ring tx %i\n", ret, i); + goto err; + } + mbufq_init(&q->txq[i].sendq); + q->txq[i].gen = 1; + q->txq[i].size = p->txq_size[i]; + mtx_init(&q->txq[i].lock, "t3 txq lock", NULL, MTX_DEF); + } + + TASK_INIT(&q->txq[TXQ_OFLD].qresume_tsk, 0, restart_offloadq, q); + TASK_INIT(&q->txq[TXQ_CTRL].qresume_tsk, 0, restart_ctrlq, q); + + q->fl[0].gen = q->fl[1].gen = 1; + q->fl[0].size = p->fl_size; + q->fl[1].size = p->jumbo_size; + + q->rspq.gen = 1; + q->rspq.size = p->rspq_size; + mtx_init(&q->rspq.lock, "t3 rspq lock", NULL, MTX_DEF); + + q->txq[TXQ_ETH].stop_thres = nports * + flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3); + + q->fl[0].buf_size = MCLBYTES; + q->fl[0].zone = zone_clust; + q->fl[0].type = EXT_CLUSTER; + q->fl[1].buf_size = MJUMPAGESIZE; + q->fl[1].zone = zone_jumbop; + q->fl[1].type = EXT_JUMBOP; + + q->lro.enabled = lro_default; + + mtx_lock(&sc->sge.reg_lock); + ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx, + q->rspq.phys_addr, q->rspq.size, + q->fl[0].buf_size, 1, 0); + if (ret) { + printf("error %d from t3_sge_init_rspcntxt\n", ret); + goto err_unlock; + } + + for (i = 0; i < SGE_RXQ_PER_SET; ++i) { + ret = -t3_sge_init_flcntxt(sc, q->fl[i].cntxt_id, 0, + q->fl[i].phys_addr, q->fl[i].size, + q->fl[i].buf_size, p->cong_thres, 1, + 0); + if (ret) { + printf("error %d from t3_sge_init_flcntxt for index i=%d\n", ret, i); + goto err_unlock; + } + } + + ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_ETH].cntxt_id, USE_GTS, + SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr, + q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token, + 1, 0); + if (ret) { + printf("error %d from t3_sge_init_ecntxt\n", ret); + goto err_unlock; + } + + if (ntxq > 1) { + ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_OFLD].cntxt_id, + USE_GTS, SGE_CNTXT_OFLD, id, + q->txq[TXQ_OFLD].phys_addr, + q->txq[TXQ_OFLD].size, 0, 1, 0); + if (ret) { + printf("error %d from t3_sge_init_ecntxt\n", ret); + goto err_unlock; + } + } + + if (ntxq > 2) { + ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_CTRL].cntxt_id, 0, + SGE_CNTXT_CTRL, id, + q->txq[TXQ_CTRL].phys_addr, + q->txq[TXQ_CTRL].size, + q->txq[TXQ_CTRL].token, 1, 0); + if (ret) { + printf("error %d from t3_sge_init_ecntxt\n", ret); + goto err_unlock; + } + } + + mtx_unlock(&sc->sge.reg_lock); + t3_update_qset_coalesce(q, p); + q->port = pi; + + refill_fl(sc, &q->fl[0], q->fl[0].size); + refill_fl(sc, &q->fl[1], q->fl[1].size); + refill_rspq(sc, &q->rspq, q->rspq.size - 1); + + t3_write_reg(sc, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) | + V_NEWTIMER(q->rspq.holdoff_tmr)); + + return (0); + +err_unlock: + mtx_unlock(&sc->sge.reg_lock); +err: + t3_free_qset(sc, q); + + return (ret); +} + void t3_rx_eth(struct port_info *pi, struct sge_rspq *rq, struct mbuf *m, int ethpad) { struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(m->m_data + ethpad); struct ifnet *ifp = pi->ifp; DPRINTF("rx_eth m=%p m->m_data=%p p->iff=%d\n", m, m->m_data, cpl->iff); if (&pi->adapter->port[cpl->iff] != pi) panic("bad port index %d m->m_data=%p\n", cpl->iff, m->m_data); if ((ifp->if_capenable & IFCAP_RXCSUM) && !cpl->fragment && cpl->csum_valid && cpl->csum == 0xffff) { m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID); rspq_to_qset(rq)->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++; m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID|CSUM_DATA_VALID|CSUM_PSEUDO_HDR); m->m_pkthdr.csum_data = 0xffff; } /* * XXX need to add VLAN support for 6.x */ #ifdef VLAN_SUPPORTED if (__predict_false(cpl->vlan_valid)) { m->m_pkthdr.ether_vtag = ntohs(cpl->vlan); m->m_flags |= M_VLANTAG; } #endif m->m_pkthdr.rcvif = ifp; m->m_pkthdr.header = m->m_data + sizeof(*cpl) + ethpad; m_explode(m); /* * adjust after conversion to mbuf chain */ m_adj(m, sizeof(*cpl) + ethpad); (*ifp->if_input)(ifp, m); } /** * get_packet - return the next ingress packet buffer from a free list * @adap: the adapter that received the packet * @drop_thres: # of remaining buffers before we start dropping packets * @qs: the qset that the SGE free list holding the packet belongs to * @mh: the mbuf header, contains a pointer to the head and tail of the mbuf chain * @r: response descriptor * * Get the next packet from a free list and complete setup of the * sk_buff. If the packet is small we make a copy and recycle the * original buffer, otherwise we use the original buffer itself. If a * positive drop threshold is supplied packets are dropped and their * buffers recycled if (a) the number of remaining buffers is under the * threshold and the packet is too big to copy, or (b) the packet should * be copied but there is no memory for the copy. */ - static int get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs, struct mbuf *m, struct rsp_desc *r) { unsigned int len_cq = ntohl(r->len_cq); struct sge_fl *fl = (len_cq & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0]; struct rx_sw_desc *sd = &fl->sdesc[fl->cidx]; uint32_t len = G_RSPD_LEN(len_cq); uint32_t flags = ntohl(r->flags); uint8_t sopeop = G_RSPD_SOP_EOP(flags); + void *cl; int ret = 0; prefetch(sd->cl); fl->credits--; bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(fl->entry_tag, sd->map); - + if (recycle_enable && len <= SGE_RX_COPY_THRES && sopeop == RSPQ_SOP_EOP) { + cl = mtod(m, void *); + memcpy(cl, sd->cl, len); + recycle_rx_buf(adap, fl, fl->cidx); + } else { + cl = sd->cl; + bus_dmamap_unload(fl->entry_tag, sd->map); + } switch(sopeop) { case RSPQ_SOP_EOP: DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m)); - m_cljset(m, sd->cl, fl->type); + if (cl == sd->cl) + m_cljset(m, cl, fl->type); m->m_len = m->m_pkthdr.len = len; ret = 1; goto done; break; case RSPQ_NSOP_NEOP: DBG(DBG_RX, ("get_packet: NO_SOP-NO_EOP m %p\n", m)); ret = 0; break; case RSPQ_SOP: DBG(DBG_RX, ("get_packet: SOP m %p\n", m)); m_iovinit(m); ret = 0; break; case RSPQ_EOP: DBG(DBG_RX, ("get_packet: EOP m %p\n", m)); ret = 1; break; } - m_iovappend(m, sd->cl, fl->buf_size, len, 0); + m_iovappend(m, cl, fl->buf_size, len, 0); done: if (++fl->cidx == fl->size) fl->cidx = 0; return (ret); } /** * handle_rsp_cntrl_info - handles control information in a response * @qs: the queue set corresponding to the response * @flags: the response control flags * * Handles the control information of an SGE response, such as GTS * indications and completion credits for the queue set's Tx queues. * HW coalesces credits, we don't do any extra SW coalescing. */ static __inline void handle_rsp_cntrl_info(struct sge_qset *qs, uint32_t flags) { unsigned int credits; #if USE_GTS if (flags & F_RSPD_TXQ0_GTS) clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags); #endif credits = G_RSPD_TXQ0_CR(flags); if (credits) { qs->txq[TXQ_ETH].processed += credits; if (desc_reclaimable(&qs->txq[TXQ_ETH]) > TX_START_MAX_DESC) taskqueue_enqueue(qs->port->adapter->tq, &qs->port->adapter->timer_reclaim_task); } credits = G_RSPD_TXQ2_CR(flags); if (credits) qs->txq[TXQ_CTRL].processed += credits; # if USE_GTS if (flags & F_RSPD_TXQ1_GTS) clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags); # endif credits = G_RSPD_TXQ1_CR(flags); if (credits) qs->txq[TXQ_OFLD].processed += credits; } static void check_ring_db(adapter_t *adap, struct sge_qset *qs, unsigned int sleeping) { ; } -/* - * This is an awful hack to bind the ithread to CPU 1 - * to work around lack of ithread affinity - */ -static void -bind_ithread(int cpu) -{ -#if 0 - KASSERT(cpu < mp_ncpus, ("invalid cpu identifier")); - if (mp_ncpus > 1) { - mtx_lock_spin(&sched_lock); - sched_bind(curthread, cpu); - mtx_unlock_spin(&sched_lock); - } -#endif -} - /** * process_responses - process responses from an SGE response queue * @adap: the adapter * @qs: the queue set to which the response queue belongs * @budget: how many responses can be processed in this round * * Process responses from an SGE response queue up to the supplied budget. * Responses include received packets as well as credits and other events * for the queues that belong to the response queue's queue set. * A negative budget is effectively unlimited. * * Additionally choose the interrupt holdoff time for the next interrupt * on this queue. If the system is under memory shortage use a fairly * long delay to help recovery. */ static int process_responses(adapter_t *adap, struct sge_qset *qs, int budget) { struct sge_rspq *rspq = &qs->rspq; struct rsp_desc *r = &rspq->desc[rspq->cidx]; int budget_left = budget; unsigned int sleeping = 0; int lro = qs->lro.enabled; - - static uint8_t pinned[MAXCPU]; - + struct mbuf *offload_mbufs[RX_BUNDLE_SIZE]; + int ngathered = 0; #ifdef DEBUG static int last_holdoff = 0; if (rspq->holdoff_tmr != last_holdoff) { printf("next_holdoff=%d\n", rspq->holdoff_tmr); last_holdoff = rspq->holdoff_tmr; } -#endif - if (pinned[qs->rspq.cntxt_id * adap->params.nports] == 0) { - /* - * Assumes that cntxt_id < mp_ncpus - */ - bind_ithread(qs->rspq.cntxt_id); - pinned[qs->rspq.cntxt_id * adap->params.nports] = 1; - } +#endif rspq->next_holdoff = rspq->holdoff_tmr; while (__predict_true(budget_left && is_new_response(r, rspq))) { int eth, eop = 0, ethpad = 0; uint32_t flags = ntohl(r->flags); uint32_t rss_csum = *(const uint32_t *)r; uint32_t rss_hash = r->rss_hdr.rss_hash_val; eth = (r->rss_hdr.opcode == CPL_RX_PKT); if (__predict_false(flags & F_RSPD_ASYNC_NOTIF)) { /* XXX */ printf("async notification\n"); } else if (flags & F_RSPD_IMM_DATA_VALID) { struct mbuf *m = NULL; if (cxgb_debug) printf("IMM DATA VALID\n"); if (rspq->m == NULL) rspq->m = m_gethdr(M_NOWAIT, MT_DATA); else m = m_gethdr(M_NOWAIT, MT_DATA); if (rspq->m == NULL || m == NULL) { rspq->next_holdoff = NOMEM_INTR_DELAY; budget_left--; break; } get_imm_packet(adap, r, rspq->m, m); eop = 1; rspq->imm_data++; } else if (r->len_cq) { int drop_thresh = eth ? SGE_RX_DROP_THRES : 0; if (rspq->m == NULL) rspq->m = m_gethdr(M_NOWAIT, MT_DATA); if (rspq->m == NULL) { log(LOG_WARNING, "failed to get mbuf for packet\n"); break; } ethpad = 2; eop = get_packet(adap, drop_thresh, qs, rspq->m, r); } else { DPRINTF("pure response\n"); rspq->pure_rsps++; } if (flags & RSPD_CTRL_MASK) { sleeping |= flags & RSPD_GTS_MASK; handle_rsp_cntrl_info(qs, flags); } r++; if (__predict_false(++rspq->cidx == rspq->size)) { rspq->cidx = 0; rspq->gen ^= 1; r = rspq->desc; } prefetch(r); if (++rspq->credits >= (rspq->size / 4)) { refill_rspq(adap, rspq, rspq->credits); rspq->credits = 0; } if (eop) { prefetch(rspq->m->m_data); prefetch(rspq->m->m_data + L1_CACHE_BYTES); if (eth) { t3_rx_eth_lro(adap, rspq, rspq->m, ethpad, rss_hash, rss_csum, lro); rspq->m = NULL; } else { -#ifdef notyet - if (__predict_false(r->rss_hdr.opcode == CPL_TRACE_PKT)) - m_adj(m, 2); - - rx_offload(&adap->tdev, rspq, m); -#endif + rspq->m->m_pkthdr.csum_data = rss_csum; + /* + * XXX size mismatch + */ + m_set_priority(rspq->m, rss_hash); + + ngathered = rx_offload(&adap->tdev, rspq, rspq->m, + offload_mbufs, ngathered); } #ifdef notyet taskqueue_enqueue(adap->tq, &adap->timer_reclaim_task); #else __refill_fl(adap, &qs->fl[0]); __refill_fl(adap, &qs->fl[1]); #endif } --budget_left; } - t3_sge_lro_flush_all(adap, qs); - deliver_partial_bundle(&adap->tdev, rspq); + + deliver_partial_bundle(&adap->tdev, rspq, offload_mbufs, ngathered); + t3_lro_flush(adap, qs, &qs->lro); + if (sleeping) check_ring_db(adap, qs, sleeping); smp_mb(); /* commit Tx queue processed updates */ if (__predict_false(qs->txq_stopped != 0)) restart_tx(qs); budget -= budget_left; return (budget); } /* * A helper function that processes responses and issues GTS. */ static __inline int process_responses_gts(adapter_t *adap, struct sge_rspq *rq) { int work; static int last_holdoff = 0; work = process_responses(adap, rspq_to_qset(rq), -1); if (cxgb_debug && (rq->next_holdoff != last_holdoff)) { printf("next_holdoff=%d\n", rq->next_holdoff); last_holdoff = rq->next_holdoff; } t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) | V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx)); return work; } /* * Interrupt handler for legacy INTx interrupts for T3B-based cards. * Handles data events from SGE response queues as well as error and other * async events as they all use the same interrupt pin. We use one SGE * response queue per port in this mode and protect all response queues with * queue 0's lock. */ void t3b_intr(void *data) { uint32_t map; adapter_t *adap = data; struct sge_rspq *q0 = &adap->sge.qs[0].rspq; struct sge_rspq *q1 = &adap->sge.qs[1].rspq; t3_write_reg(adap, A_PL_CLI, 0); map = t3_read_reg(adap, A_SG_DATA_INTR); if (!map) return; if (__predict_false(map & F_ERRINTR)) taskqueue_enqueue(adap->tq, &adap->slow_intr_task); mtx_lock(&q0->lock); if (__predict_true(map & 1)) process_responses_gts(adap, q0); if (map & 2) process_responses_gts(adap, q1); mtx_unlock(&q0->lock); } /* * The MSI interrupt handler. This needs to handle data events from SGE * response queues as well as error and other async events as they all use * the same MSI vector. We use one SGE response queue per port in this mode * and protect all response queues with queue 0's lock. */ void t3_intr_msi(void *data) { adapter_t *adap = data; struct sge_rspq *q0 = &adap->sge.qs[0].rspq; struct sge_rspq *q1 = &adap->sge.qs[1].rspq; int new_packets = 0; mtx_lock(&q0->lock); if (process_responses_gts(adap, q0)) { new_packets = 1; } if (adap->params.nports == 2 && process_responses_gts(adap, q1)) { new_packets = 1; } mtx_unlock(&q0->lock); if (new_packets == 0) taskqueue_enqueue(adap->tq, &adap->slow_intr_task); } void t3_intr_msix(void *data) { struct sge_qset *qs = data; adapter_t *adap = qs->port->adapter; struct sge_rspq *rspq = &qs->rspq; mtx_lock(&rspq->lock); - if (process_responses_gts(adap, rspq) == 0) { -#ifdef notyet + if (process_responses_gts(adap, rspq) == 0) rspq->unhandled_irqs++; -#endif - } mtx_unlock(&rspq->lock); } /* * broken by recent mbuf changes */ static int t3_lro_enable(SYSCTL_HANDLER_ARGS) { adapter_t *sc; int i, j, enabled, err, nqsets = 0; #ifndef LRO_WORKING return (0); #endif sc = arg1; enabled = sc->sge.qs[0].lro.enabled; err = sysctl_handle_int(oidp, &enabled, arg2, req); if (err != 0) { return (err); } if (enabled == sc->sge.qs[0].lro.enabled) return (0); for (i = 0; i < sc->params.nports; i++) for (j = 0; j < sc->port[i].nqsets; j++) nqsets++; for (i = 0; i < nqsets; i++) { sc->sge.qs[i].lro.enabled = enabled; } return (0); } static int t3_set_coalesce_nsecs(SYSCTL_HANDLER_ARGS) { adapter_t *sc = arg1; struct qset_params *qsp = &sc->params.sge.qset[0]; int coalesce_nsecs; struct sge_qset *qs; int i, j, err, nqsets = 0; struct mtx *lock; coalesce_nsecs = qsp->coalesce_nsecs; err = sysctl_handle_int(oidp, &coalesce_nsecs, arg2, req); if (err != 0) { return (err); } if (coalesce_nsecs == qsp->coalesce_nsecs) return (0); for (i = 0; i < sc->params.nports; i++) for (j = 0; j < sc->port[i].nqsets; j++) nqsets++; coalesce_nsecs = max(100, coalesce_nsecs); for (i = 0; i < nqsets; i++) { qs = &sc->sge.qs[i]; qsp = &sc->params.sge.qset[i]; qsp->coalesce_nsecs = coalesce_nsecs; lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock : &sc->sge.qs[0].rspq.lock; mtx_lock(lock); t3_update_qset_coalesce(qs, qsp); t3_write_reg(sc, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) | V_NEWTIMER(qs->rspq.holdoff_tmr)); mtx_unlock(lock); } return (0); } void t3_add_sysctls(adapter_t *sc) { struct sysctl_ctx_list *ctx; struct sysctl_oid_list *children; ctx = device_get_sysctl_ctx(sc->dev); children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); /* random information */ SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "enable_lro", CTLTYPE_INT|CTLFLAG_RW, sc, 0, t3_lro_enable, "I", "enable large receive offload"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal", CTLTYPE_INT|CTLFLAG_RW, sc, 0, t3_set_coalesce_nsecs, "I", "interrupt coalescing timer (ns)"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "enable_debug", CTLFLAG_RW, &cxgb_debug, 0, "enable verbose debugging output"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "collapse_free", CTLFLAG_RD, &collapse_free, 0, "frees during collapse"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "mb_free_vec_free", CTLFLAG_RD, &mb_free_vec_free, 0, "frees during mb_free_vec"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "collapse_mbufs", CTLFLAG_RW, &collapse_mbufs, 0, "collapse mbuf chains into iovecs"); } /** * t3_get_desc - dump an SGE descriptor for debugging purposes * @qs: the queue set * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx) * @idx: the descriptor index in the queue * @data: where to dump the descriptor contents * * Dumps the contents of a HW descriptor of an SGE queue. Returns the * size of the descriptor. */ int t3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx, unsigned char *data) { if (qnum >= 6) return (EINVAL); if (qnum < 3) { if (!qs->txq[qnum].desc || idx >= qs->txq[qnum].size) return -EINVAL; memcpy(data, &qs->txq[qnum].desc[idx], sizeof(struct tx_desc)); return sizeof(struct tx_desc); } if (qnum == 3) { if (!qs->rspq.desc || idx >= qs->rspq.size) return (EINVAL); memcpy(data, &qs->rspq.desc[idx], sizeof(struct rsp_desc)); return sizeof(struct rsp_desc); } qnum -= 4; if (!qs->fl[qnum].desc || idx >= qs->fl[qnum].size) return (EINVAL); memcpy(data, &qs->fl[qnum].desc[idx], sizeof(struct rx_desc)); return sizeof(struct rx_desc); } Index: head/sys/dev/cxgb/sys/mbufq.h =================================================================== --- head/sys/dev/cxgb/sys/mbufq.h (nonexistent) +++ head/sys/dev/cxgb/sys/mbufq.h (revision 169978) @@ -0,0 +1,86 @@ +/************************************************************************** + +Copyright (c) 2007, Chelsio Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Neither the name of the Chelsio Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +$FreeBSD$ + +***************************************************************************/ + +struct mbuf_head { + struct mbuf *head; + struct mbuf *tail; + uint32_t qlen; + struct mtx lock; +}; + +static __inline void +mbufq_init(struct mbuf_head *l) +{ + l->head = l->tail = NULL; +} + +static __inline int +mbufq_empty(struct mbuf_head *l) +{ + return (l->head == NULL); +} + +static __inline int +mbufq_len(struct mbuf_head *l) +{ + return (l->qlen); +} + + +static __inline void +mbufq_tail(struct mbuf_head *l, struct mbuf *m) +{ + l->qlen++; + l->tail->m_nextpkt = m; + l->tail = m; +} + +static __inline struct mbuf * +mbufq_dequeue(struct mbuf_head *l) +{ + struct mbuf *m; + + m = l->head; + if (m) { + if (m == l->tail) + l->tail = NULL; + l->head = m->m_nextpkt; + l->qlen--; + } + + return (m); +} + +static __inline struct mbuf * +mbufq_peek(struct mbuf_head *l) +{ + return (l->head); +} Property changes on: head/sys/dev/cxgb/sys/mbufq.h ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/dev/cxgb/t3fw-4.0.0.bin.gz.uu =================================================================== --- head/sys/dev/cxgb/t3fw-4.0.0.bin.gz.uu (nonexistent) +++ head/sys/dev/cxgb/t3fw-4.0.0.bin.gz.uu (revision 169978) @@ -0,0 +1,483 @@ +/************************************************************************** + +Copyright (c) 2007, Chelsio Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Neither the name of the Chelsio Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +$FreeBSD$ + +***************************************************************************/ + +begin 644 t3fw-4.0.0.bin.gz +M'XL("'+5*D8``W0S9GB&6L +M%MMNT"(J;1%:JX9P4>NE"EYJ;S/*_IZU`];VG9G?>[[?^9WO?.>\T;7WVFNO +MR[/6>B[_9ZVU2<5XV`4,D\H9AE!A&):*/87U87@6QL?0W8%Q,*P78T,-1+&I +M4Z<+&"?LES@O2<7IO9\/A[L*2ZFF,>SB*.S;45C_**QF%+9Y%(9CM!J_ +M-(J#8\VJ.`);3(68QEE-XQXRC1MO&C-4EG'6 +MP`L-(0V421R')8Z3)HY3);*ON#@F9SI#F4X2QS:HY$S'`U%ZU`95)D3:0YE3 +M**$]C#D5SIRBK5^K@N9Q4C$%;VP05,V1X?#_S5$*_C1%T!B2SWOBK5$7X=6; +M$_HQ5'M+&--"6_%$/!4+NI^%-PG>WV1;IT^HUJFRU-F:96."VQYO^U,J)LM) +M;IA=9HV;R%LVYCN-D;0Q?CGS01R1L7@"'YK:H-(8D]BJWZ='<\U!;"W>4,:[ +M025CO(04^^_??__^5_Q`3KO^;8;3HZ3,C\2_?!U[-Q`N8,%;_F6NAGMO+'.B#>)9*I9&3L8%3+8"W2%D7<#D/ZE`>"!$0?PNND-0 +M7@.82+$4=I$"$D0'X_N$&9`'*7-@)`,\:?0'<)+$$=I+T%(A7@*ND-8 +M!'&4M@C"4HC/1W<(!1!':05#<0?:@`K', +MT8LJ[+F;D`YC_DH#Y/TRP$O--R&.[D=C[PYBF(1AD(T*@I`$(0/#(KH9!NO> +M\_&-R?#\,&J4!J,(6:SH?2J&+67IP(09+-/_A,K$0.!@`@S]Q\00E($\;$$2 +M*HZ"P$MB'S$2`I=MBX<%L[VR8CT8HD)`,I\P#-,#H9?/H-I^POA,1BP0I&+M +M*9\+KQB4>)//,(-<)AT>CT`8+A;+'+E9$^A';"I;MR`C&9I#.=C.,C>QK8'Q +MR8+@#%`M4JDL3`8S"#5##@PS8E`&1]3&,JU*U!-4.X8Q-SB8#)7=#&$]P]Q` +M:?^(93Z&,6VF&>9']'P/QO<,QOD'S,V)KECF6%JNR!C`$C+T5 +MPE*H9P/D`7D_!G6T\OJ`G#[4-+?=:4BV91B2WX;@A+`M`]J6(`RBL@RB>C*` +M9J@)Z.4P/QF2O=!EYD=X+X4X9DC>#^$C"NY0]J,:2)<9DC^$M,,9?3#N?0!J +M>J&=/G;F,Z!/QX`_CO7&WL,$<+\#M`C\\(-[%-#MA'`=LJII=NQ:C2KLA3,0 +M$"^!/GD!=-`+4R'88YGC1DBS`E^F0C-J%?8'&#WN82ASD/D'`_SXTJ3X1T/QF_" +M=;B#GGT<]+&J586-A?2'#D(^H,$,NE9O56':U2IL-(Q_]#(5%@ETAP*-L5`DPAL`+(SOUL4"$%`!Q]TT22P!P^#;>"U(1I&VP.TH+O]-__^9_Q` +M)GH#]Y8VX$70=\=6_T^I^+]___W[W_`'_'XE8!G_V>^?I#\1E_%?J3>@E_^P +M94@___3K]/\K`FN3ANX04D#_O?S`\__987B^$61"MGHH_CC<,AYX_@1N20\\ +M(U"A^LVS]#?/@M\\\W[SC/_FF8/PP%!\..V7WTNEG#^5XB^75EBV_>7XMVEW)Y2'J\T**@T.+B4 +MSR\5"$J%PE*"*!7].UJZH+`3[W:*H/T>)Z_!&?0W9_`!)_\#I^"@4WC(27SH +M%#7^:UI`)OZHCCI3*8]BN@GZ/[]'F$X=Y2Q1;`@+9>[A5ZREX=PX8IUUXPC" +M&B@_6X%U@&\\2`S5-_LEK/-7SY78E[]Z_@G[ZI?G_Y%?D`*S,3__#Q?[_\UO +MD32((\-3PBM.2=!*'(W1:@P?A3\]"GL*+172F.P&&7I#S&47#I4W1"-OD+13 +M\"'YSG:!+@>C"GCZW7=7;R?<&;H<#E4@T._^B7A&L7BGR)WA7NO.U.5PJ0*I +MX@;&3\(4`TSX``,1^0`3.L!L4-&I]%+EP*!B8)#*#&2X!U>6)EPQ\`\4Y[&# +MOTC*HZWR@1\S57]5XZG8HW12Z,#WZZSKK,M&)88/?!V4QT5)&+TT?.!ZPEP,_B]?\+5RX`K]>F(.!MU$]:$;6Z\*8@\\__9. +MU^@(GD$30VFB==I@?3R^*AH%?C"03VYG:IXR'=Z*#T=P7IU4'!"W&HB!!-;_VH[0_G5#YK +M5U**^`PJ,W+@B'E_%7UF32>>RBDW[??TM-M/N1)5)$UODI++9;O6Z,[G4`3Q +M]CIX;]*=SXX9.&A?KZ5&H_1[37255H*0.>-Y]B+Z8OOPAY<"NXH>[>F,&ZBK75SF+5&6E@@&\JH0^ +MKQS85K71M4$B2)(X9%09/_U'@>493+(/M\3C:Y8'4:/$5+SFG>5!^E$2?;QZ +M]?+Q^HDJ"\&A"$S\TWB!8(H]8ZUR8%/LP"9+SB2LQH&U%SQX!6+T^SWZW4K] +M`4^>=?GV"18UKW!,T6@Z2WA,U-#F>ETYD,=C9*ZE#D/56Y[ET0/OEKT)77&] +M0;B2@ATR`8-31!1<+6_QCM-Z%4G6R41U`EUMMO"X2+*#_ML&G.R4F`K,-_ME`!.-Z/"5DJVWP +M8;UQF2\I):2TH^:49)&4.&V-,R[;$!HZ,#,56Q@Y,/UH1LS`,\!'T()E=*:Y +M#T?TCT4-AEP<5;BGHCUWCV-?@UNW3?QV+K=X;_4>G1K7$TF$&]?%XY\_GYLJ +M7#!K_@:8]2G`,*F(86;0]:?K6S^@9V5YZ==YO&G\U;B]248OI=2XD<`$@NEI +MQ2VZN1B46[#A(_WHJB7;_MTMW/\]2Q/ +M_5%W.]N^_CFZB5X7.:"!>:F\4-XK8-"85_N>@(K +M2`T;D"T_-MUA,ZEY[\X5?BRBI\8.D*5]U;U9ADV77)=Y?EG05+ST/'VS^ANR +M.BF8D4G\>-,]N'@NZE1D1PWIEKWGIDJR&SX2G!%19=FG))`\G-A1,YR(Y`I( +M)%@2TYB7;,QSN+VHHX&I;D^_&F3NX)A/L_SW"Z$ENY=`;T33*F"@8,6.!1"X?3CZ^^K&?)C5C3@?V3_ +M3D-M54S_UB4'8-XQ4ZV'?MW>_I,%=$T+^8.L[B^Z_3ET$:$W8YEG^]3AE?V9,?T;!F+S1 +M])UJ3U5ECL%8%DJ?T)?)Z1JHM:Q*7R83LVS#6R.3#"#.&<#-95(88E899@M: +M1=0!4!:Y&@*-1W3_W*.TA1500VVFK@\G&W'#-C0CPHNC'/[JC'Q_;D]#8X5_ +ME9M7U&T^+88)(9F`0J@&PQG5_SQVBV<"U;((6Q6)FF!K$1S$J?NUT)Z,7-K1 +MW.!VTV_G!IF:.:86S*S&0S[&S:B:E(`.$:\+Z!"!7A5L4./S=_#F;^>2,`-DQ_>-TYZN6E,$T)5O.>VB__=938"]8:['W+=WJ35W*@^ZX$H7CZH9VURG' +MY77]K@ZJ)&S#EZ[.==?,UU4;OBJ4T0_1O5Y&(IGYEV7CZ(?X;PM)-\$?1_*Y +MDX^/KG(==M>/:7V$O8]OS41WR$/)%^>)JJK,9;(J5U6%KM*J[`^*:PD=Y^L!WUR';LYY@//&@['LNE*$WPVVMG-G5:6J+,+3'F +M%C5=$GG].)VE:XF+N'Y:WZ+1MT1&7O]*UQ*M;XFE6D8JKW?2FZF6>.7U#N)# +M7+`"IUJT@!]LS/0`OY`_(GZA78!/;,PXX!N=*AB,P'P7;_XFX!NP!*V%]Q6` +MZ&.D#PTJ,OSZ3O)#V;8Z8VV.CB!7(]-RW%B;;6\Z8IA;Q>J$3PUS/4T%=N]' +M.K5*F)%J4DT\QF#\21QBDG'6=6J[UU!#%3'Q^W\IJ\?C=POC$^3@Q"B+HZJ5-"M],WZ9)2.=VGO+YZ +M8R@]-1X4T?NN>GJ=8+9,D(*[O)1J8IZ5E\7AW1@O8*1"!T?0/Y[OEX+J(8J3 +MB/(D?IV4LDIII[9@D'=UO.3&!!$C)=Q2B1L7N_%B@PFZR>`Z=;S8I!#J%(KK +MRP6S<1ZH)CI+!=I*,%4FF(K3>7`I;2K6\]RRZJ/0(A05C$\*86=VFFYL):`?!BGG#:.XN9^[/X.Q)P(# +M?^_32MDY5864OPI/"?YC@2Q?2K>#R2=@^OO'IW5/M=W;,L2$+R`F_!6F)KEC +M<1AOX,8%[\(TEPPS)-@E>JF0F'Y\VV?)R%CR9N`I(F7B%BY+.H[%E'TB- +MZC@3P6L])O;@)L0#[1'7[II+JI9<@LI>TI=X[)M?T(<3].UC^X"=E=>^)P_+ +M=K3IF@$?2UF5]X2Q&=CY,8/:4C1/=&TL\*_=N\.B5DGJ66Z43/+^)"R>(&&$ +M(D8D$4PF'%RO0R"8O(.1_!T`)KZB'?%PNB^([ESWK)56ZX$?`"B"@"$EDJ@'/S)*8*`+CF`^$=TPS;X$AOIMPT^5_$IO1Y$\';U<0\P[IKW84+&5(95 +M*,HC[,E[Z=Y`S=!S(+!:6JSP*('(/8M9&MMS%47*PHB]>\UV;D%"OFY/&]E( +MBD1/52;DS3/$QPOKR/)Y,%#T;:%#0"4H;8-&L*?48:4K'*:#R.,BO3!9'\X' +MD`].0_H9H'DM3,W*3HBL<(07C]C!@/&$,@%ZSJD,;;!EP-L*MJ-^C>D"XE5IVOH6Y_AJP`>\-OO?^8#`S6G[/H:!7B15G+::"5XB@8$T +M1%][9KXK:/XF/!4OTZNU]F>YE%JS$C0A_AI0`,VW>EAUC5`D\2PK#YJPO-&K +M-D..E07CZ=OVYE7`\;0K;QR=0[@FN&+W%XC!30L/`84NKA.8:W/6'(&\4\V@ +MSL]/,JC->:^*=HT%796HCJ7"Q?;]4_GU#RKD8]7\Y81H.<$73!8."\%R1W1B +M.#\]PT*IXUW*Y?F)FG#%NTK'"$WD7?Z[)/YZ:EQX&,9%56A6<[B)$P3+"8U< +M$2,#).'0MJF!NGX#ND&Z+6J5P1CEC!&)E@#*Z\ +M)@?U"Y$-D7$:D;W^\\1PI38\/#Y\1/9\>MG&Z_2LBG['PJP%KANE`Z".Z0S^ +M-(YD*H>7,IY.XC\_?L0U;O9(_FS\W8B-%[.C^$BU`QS%%=\."G:`,LQKU^[/KW;OK]`>R*4V8+E[K=5`)8<>7%B84Z+;[30G!XC:2))\R@H0^F\)R +MBGXWA^4Y+>![>BWTAFY(`-*31'_A2.9Q@K/&TZFBG/'Y(RLNYD%ISW$`[QTEXHEA41/#8IB'W':\0K+D%'@$#H"`'N5FR2\_F3!X`TW$'6(<* +M%RQ?!_$K#CDP4*9LAT83S@=&L=W;"_)J#)>9P^4@K^&AOJ4IO":#/#3ZVW5F +MN9R^;93+TG;_F/>J[=Z'G1)]N-0^:RN=D3O?%5K1[[Z>N\"QT#W@NN%8H)=+ +M7=+")=NNQO$2R!XBHJ/=\2X$RI>!:XC#\GV +M>'0=V:``!&Z![G1V]+>C;/^X'=!YQ!)V3@/9Q&Z!OB1GR78P19F6DFS[S=7T +M>Z'?6DQJ))$BOXS7)A.WX:9GY'"MNA7S;:(_J>IFD6'3]YM^W/1#Y1WN`%YU +M&[E()=E=2PV'58:R;%^27AL,+:40'*1;V0F%B84YY3Z*(]LN5+/S*AZ>U],U +MCNWT+"I<:ALT(/-"S`ZLR0E`3D53^(\HO#M%Y1-T,ZRT5W@PL)IT(_I;;M2W +M7./I20LW1Z9O#\*2'9BS@(A1++XJ"L0#5[H]WTJ7T&OI7C2?=R2O<"1%XP4# +M4N%L#GTSJ'@\GY%*EN`ZN524G62L'"0+QP=E3W`8P#-$,^N7!OGQ$#\.?B:8 +MEZBK]P`(D@[\P&Z]6N5*RE;Q7%($-6?CE)H7]"F^<7*Q7N"054\552>951.A +M[I!)24$#4JCMS7D"O3J6T"M2,1RN/+#7>U#-GB-ZEC2"-T-,JI5$!%_]N^ZFS?)GW?)JJ=M7?6[-.L+; +M&/^2U+[F!QO35BAWR/84"JZ2NAEGSB$_R4L+,"5P-`?1U6&%6UMN8#UF,2X(=!W@[#&GU:AQL.R&8?CSG%TPK +ML#V(:?.M.H*W[`,8G[BTYLC(JT^;:JN6G`8Q'#36>NCO[/0/T5>?=(P'<&U? +MRS&&!X.X1EU-`H'>'7#1I,A%P[I-M=D>I;W]=--1B]I<-/^<2C(PUJ2.]?56 +M1G8^ZF,ZZ.XK3JLT6OYD54^0KFE7]5>2Y_ +M_JI=D"_[@"8 +M&.'SML&C8!&0/A#.L@T>A.Y7*,"2$HT"W?YA2(]=T^W/MCN_25#S/'35$<%\ +M&;)D3>QBBZ?`BM:VMWF6'WM4>>4?>6,*1]-W8J[\6+V^REELV+2Q?!/BOQ*R +M*BEX0.8IXRU&3-[4B[BTE%V0SM;O5NG1@I"3G3.$(ODIW*BTO4;;X-\Z)?;D +MK)@K'8'NTS<#*#?"$_;+`(PH?+4HS!%`N47S]A1V]:[P<;S7'1'Y(RKG>5Y% +MOMJNNH"C5IY0/!^<-1BEZOFV0048.%91"X.0`A7N!L<73.;?SJGR%2G"$-N] +M78'7@GL/8%=^VWWL^BK2G4UO4&BG00#>_HFO&\L +MR>88)Y,/`^/OO(W3WV9%[1'CK+L?7I^ +MV=^PHSQ>EE+0^AP&$`EL1D-0MRS$@<=@QYX\J-N>-->DSX1X7; +M1(T*-5LI[:@19NW$D)!I/%Y2@E6GC8_6RC$J/BI!^YA^5!B,!B&<+-Q)ZBJO +MZ^)CUUP>(:Y)QM1O7\9<>>QH.Q78,:C--#3A8@:WL$N)HHNCI*$6>?XBW]"J)GC^ +M^;WN>D=O_OD&AZMWQ:$@XWF.\1*['MG-KD?>^P!TZ#E5U)7)T5?B\]6Q5Y)$ +M3XOV46^OFT;WB@[A!H(C_'A\2,B4$Y3(]&E!F!498IX2KS+'#$4O`2U% +MB!8GRL&+E:,R;#QP96F!5KFNP/90L8TIH6\*AW8(#MO0"060*4H5//]=T#J. +M!((#ZCWMV4HDB$BNYC"'Z/J8R_<*MH1>_JQ811$9U!8QG1G2QJ%[$[>0)G44 +M%Y1-;9SJ,?H6EYO$'1Q/#$B16MV"DGH%8SB0%'MY0`3FLXT3>[D?WL50[%0(;,P"3"F2'7KX8<_FB1XM0.$W@EK%/8<.1\$`DQ,V)OMQA +MKHE!C+,E7)BKT&T)8Z\*]AJJJXF.OGPR4#=(;>Y6K@/JY12:8BZWB*,5(7Z. +MV(\JPUJFB[F)&)!S@`%2PUJJKM(+&" +M`1G[I!Z.:`(1[@`GKC:4+3:RV,C][BGH7A"#WA3O-&_1A!0IS%O4[%7EV,'> +M8W,-8@<'FA:S34=\I +M8`*"Q\@`S)JW:8XB&R`6/QG,]B3ZB<`(B@3$I,$A%;MU*DFAE[.XKBE:$&P#IY@ +MV(03DG`V(>+RJXA>%'FR<(NKKI@J2N#Z86Q(^E;(X`0<&`5LTI9UD)R[D3M` +M0EKLY3>XP"E^#OLF(2$T(4&1D!"6&(\[$@I-+AV`>EV"./3R0[K1[%8`UJT? +M[;'7G$&0_T/9>W6FVIR#,]XY`B_:=;5@0$^XI:7K@9=I;_1EP\%*4QG&^SW8 +M"IE9'4ULQ^V]3U)0T"'C.@26S&S)3)$E)SLA/OK$VY__P:")T6FB5[E!^K'C +MAT(OA_VKU8^.I<"K#LJB"5YQZRGZIO*RFFZ.ALO%?&G$Y1#1=-&!J>[Z*F^> +MJKK!];?J#V(N*]T'08-5O1_LQBL/>1JC+X^(NLQ%(/1#$MRH_4BO,:_2K4:T +MR/T*BQGXTY":1)'?_;+?#C*K'\U=D@E]O4.GAEZZ/N)R?-2E`;,JK#AAY*5O +MXU4BK1K`OM`T2[M@"F0Z5DZM_!FMRQU++]B?__Z2+K!W45@3K_X*YHCS-WZ"'ONNA[SLY=/WQ@_]^[Y]2V>D^C.9!]_D?02\XQ,^D^!XI%D<( +MKTZ`V7H[TZM3D2,OM9)NV2ZWN23GMJ"+* +MWA>`.MA?I$(;\PTF%;_\@^H#IK\1"5H^VHTMR3:VJ8QEV3;FSU^F:M$065#S +MM#=!&V9)X(><8?<"9FL7E$`7OU^P":YY\S=`MM$Z36Q:M5&G>G'!>GB,8S/T +M1EXZ:QZMUN]A2GI-H]7&T7$-?M,V]3O'@XRC-<9M:H/V1=O@3]`M>CWQ$REP +M3>`?X@IW`=59<6)IAAT#=YRLP;AP;=T)R-'-^6[`08CGM7D6_F]#G3EGZG +MGZ$T3%3236!81*T!0_9Y]*7949=F@]_Z=?+"YV.-+]U+/\9NBM<7D#'(EGTK +M"CP%KOE)>BA;-"U$,DU$/B=Z*IG7)B57XY6CHB[-:7"@/0@@R5KXA$/E>1@< +M4]?HXH1MC"4AN#`I]M)?5IZ>QETEHK^,NO2(QUM9#SUQOX_@,3`/4DH-A0E& +M%=]SP$$9_T:X/K"H2)V6+[^42#*R6D9S.[MA#3X@,O^0C3`FO)1?4K-OS+C-TI*(U8#.U-D;/`M>"I`KYJL_CZ/>-$Y7^Y*^34X*8?#GMK)#2?2;H +M3?]X>\_+^4E%4AN3&%AD[DXMER&`#\JD08P7KZ +M2.RE1XJ2]EE"BJ;Q^D7GVM,[<;W_)[K]S6O+J\-2L1AW#;5;BF4X7'(Z`[M9 +M4!WJVFQ/?4$'B,3/??/:=G[V_3A?-R5W<[$BK7K+^P.6W1)RIW;0.P^YIYZ29+\GR2&W?E]6DU.[\;W@Q7CZ'(YWL:>V +M<(J-V>9M2URM="MLS#*T=Z]7A>E4X8IO/@9OP>Z<8E2)>)\+#7U:X4=CQ8VD +MX1OMGCJ"F*;7AIVHR7^BF,J=4K\#6\]#^HJ1-.*&!'YQ@N0@N>+1$:S?"5(P +M@+9<:@-G26CU*8FXCH3J`G7L:80*[35+\Y\HI(JGU._%;O&P@5TFF93(NV(%390A.2BZ.*JKW].;6 +M._8WN-WU;^?*'5YHU56/#M;$XQJ"`^R#EK=W#A.!Q+.*]:5[K`:P1?39I#RI +MPUH95A'NE@GLL\I'T)L%@FE!.\:[%/12$_0@52R>;F]_(VU6@5FE6'T93\5' +M+'@"]/5SU4JDRSF6O/!\99%6IY(4R"A5B->MTTIV.^H=E#;$TLL]V.AE3+U\ +MRT7N;G]]FZF+,5WDIV#E\R?#[-Z@:W*U9I4DZIO%YE*N2162[S25\DD8`F>Y +M-I6#+7@B-_>[=Y"3%)M/Z37!J\N"*-6+]K69HCK27K-NY#-,*K(JC.R6[;I&[<\QM@6O`-O+66H$Y[5I8=&(PMLAW6//]N:!L^Z6 +M$PYR37YX96BU$GRV7G,X8,OT8F7:/F7A[8+0(GEEZ#O7H&P&.(.AA;(BN4-1 +MK;2H^`7*'8Q%R^])GC/82]0B7"A<."UMWYO1WUB!7<1M(N-G'`-=8J0W"!IE +MT(^H;Q;L;H0+V!/*M2"!?<4RLU9BT8:D8I). +M"4O/3:.>0-NIV^8-8&Y]\Q:?D5B164(4%H2`@VWAPA?/%9DIQBU(H:VHQ: +MDKHIT6OM`L'O*"U1]/W2@C^8 +M3X%_+)TI%D\3^,>9M.&VP3\F+_R(L# +M>DU,U,7K@2-?X`3MO@[6>2!2>/67,U_Y='G]?SJ;`2ZUX`I.(5\HUT=7*S][ +M%OG5:$RXUXOD]*S\T+,U;ABG`J5)SIDSV!YU\4L$N1IENQMU^[.+E<*=(MV! +M;-N]"RPV"8Z]>"/L8H=XABS\XD"A*D\C]N'X69)X1J1/P'65=UH?6E(%4RW> +M=6*;>>6%%\[6GQRK)S@`M(2[QHM$4TXV4/?NQ>>`:Q/UBYL5B8Q(W:\K.MI/2:`"+EM!VBN'M;37CJ#8)=I3^8;H.B! +M83\0/*V:`EZ,'-DX-AZXUGYQF*EF"7O``.N)@6?7'>D;4Z*U=O%>BTQ(K-\B/[5^V$D2LYE\HNF;-[H4UI +MR3,!0.C;E0`B*>(?%I7(H%(`P!(T$$:\='^_E4,@G2B9GL:40*7UC2+1-*C-7O_IZ@_6TDUD)5H* +M"LAWVQ_-5L&JO@A+N[+UWOWRV-\1C0%N!^B[(+,GL(6X8/WI`.X)@%:S2F14 +MA8D'A)8^+5"^K\WXC1:`K"_5_LE>(-+"$LD0EML(Q]X,X-CHBS$VQ@F@R7"8 +M"9!U\F^K/T`$!;9!5QU'0/A?]Q$9'Z`!T9>`0'LK*P.!==,E&U\Y\?S\4DU@ +MEQ/U(P+1+9C>FLGBU\U$^?`R"-UT3L5]%(>1.EXWM/;%%>MJ9*=KUFP2&(A[ +MH@J.88N,GF54*>@OQ6VX40MF=!NHA5\VM3E3[I]-$%#&96FS\A,)#JBN5,SX +M^0NYUJ&3E=C@4/7X][2U=2)HNS]GC7K7BDY6\J?!2-6?H%-!(1B&SE8>S;5& +M7CC#GJX\I*^MBK[PU9)M:,W5U.&)NM!A/S4:0!0Y*".O"]"1A!DX>R0A#<'- +M]:_$7OBILZ(=G&L"@-JI:F?LA2,'*\VW/=HRCOFZTOR#)VW6 +MRH,3Z,T`I,@]X\7B*?;VIV(OT#$7FLP_3%HX):;^&;09XD1'*HEA^62?`E>Z +M)NK"O@-\&FDD0*'(NEV(5MG6L\N&^/ME.C"L?DESQU\M'Y&,3KAJ`,+ +M6J?72K:XZ\&0AE`WN7DRK\-\DT_=X6X9,%]GS'?X,?2]#HMI9+$ +M7*BB'N):5"&%5LM#?.2L6BNUMD'O\,!6!XY$6NGU8)EM@V[::ALL?=<*XVL; +M?`TX+?+"VJ/.F`L9R@N9QL#"5R8Z*BMD<$O._:.RE4-'9=U[0$];:D,LV\3` +M.T=K`O#&-FCHE$1<6`HPAFZ*OK"4TL1@;>@XGKX#'=4B3N#ZTYG1%Q8(+X[: +MTX\.6N:UYW8TU.E/8RO=W*)35>WHK-87@7.6?;0W#YVOY$Z/NO"G?SJ,@?,N +M"01V\MG[>ZASF'G#O(S8[@&Q/NE]YX-KPU)]?RTWU6+EFHD?3:!3XG&+E@CQ +M`(.W'<\HRU%OV@9:G#.X,*`]`-]\@L/D8=E +MM2?`W=?-X+Y3L1J<_9@+X^S.OU8Y#V[R3C>4<=*2]Y^UGEMJ9$_M\K\(G-K] +M7=0%H_*"D6H9MBT!U1VM`(8Z^ROE3==8K`++,LR22C[@'7R,O(.`/A]R#O;8 +MF/>C+TB4%T+HS9B79YX;L^M$X;/']57W?SEQ!ROR(L.S/V9 +M/3"7!AZ:`O@"^MJUU)\$+D5]"GXE7_IOB&#/<:@4H`L1_%/,7ZNAO7J06FTP +M03S1NL5(8##V=/OP^C@GG-7Q*G3>D@>6H+D!#('>*J.(N!6;QZS:;#TY"S36 +MDHUV>_+3\TN?'%8IZN1LP`P&3:SR_/<` +M9'90@!A(\`,28O4),2MV1;?N1UEU1,9.G2XA1I<033MU";$=S@`M^CT_038; +MAL:)C6NUP8+!$\$LR2TKDT$0Z>),6BB5WK%K74!.KF/ +M@:_69D^>9U3C)#G]^)%4["N8E`4;06?N!UP.J#RM_9/(\QLMM55F0I!^;::N +MUF-W/AUU_N`F9\SY;(N:1Z\+G"&GLZO*JM=7;X*9-]5Z*&VPJ;G+A_2&9P2&Y,3?7[5T7KE^95#1]@""]E`\?!"=K&_ZF8!.I[; +M9CPM7NV6YG;#E+O]HOMKZ?#XRT/;RP9HX5#"?1YX&Z5V+!T:+^.RP+[UG'L; +M;(PHL#?#KOG>*P[P23#M/*W2IP9K"-5K6V$,OAIQ_H68\\^!VZI>L`6>OZ'^ +MC(24/:R;&NC$GX>P,7#L9[+MYRRU.?N?3D='+B/0>MSZT-+URO-/'ZQT.=-F +M/?,;??]*[/FI,>=_]U_4]R9HP2_;[C=V9),S1,;3V;K4X/GKP\VI8209"&>84$.M#5L!Z##[DZ)PB7G" +M'.:U0#EHJDO&/R&`WBSY`?I=8X&^W"R//I^L/!])'S':@RMON6Y2J43%=^CD +M]NV"5&2>?^!1J8J5:%G]H.'."+2(59N-OB?8EFTA.(B6>^/3N@_;!INP)EYD +M'V.8RT[Y&YEU?DP?2?6QJN.MX7EOK[B9V^[H:'#K3F-OYW+S3U6T&_^,D[MQ +M(ZB.P36@-V@O*'".SX"^U^A#_D#?M[JY,5#5CC9D7][(A"J1JKE?)>V^Z:"+ +MFQL<+GK%ST&Z9HZN!8,ZZ?5#M3)_Z)1H6,)-M=FF+J5I6W;KBT@9@WUC9;:C +M?DAN@_7L9T,K]/?.816ZE;.BE^#(KGS#T7,_S!<1XC>]DKIXA5>45/%V][@ +MIH^;BJ@MV!KO`D,-GP`;\R&^['U<_]%/K5MLS!YTO$:-?_X%@#43@+5307\] +MQ$_%U,WH)_>>F +MCPU;,!OC@#X2]EDEB:,QL.``K*AP7BKV5VAX9%_QT9K8OJ*(/F?`CJ/%@ELX +M@)Q?%@L^JNS-^RCWDX:Z\H]6NH.H_2'4`3%2U$120%(?0&:]2-AT9T?>*BQ;X9<$`9M!&<\(; +MGH1%RH2W/.%]?VI:D7Y)5.9,+!70YZO6ZRJE:;-6''RDRTF?`E,H/!/PIXIC +M^IZ/[IN-]J2FQ/AOIO\X]%&,^`%_:OBS&+JFP*KL>R*]1=/P#^$]$=1Z*O%+ +M05F[!K"'1K9#8]%D2SX161+813IEW_A_FHZ4E,-*O[ZF"$1@/MU$MP,H!Y4V +MAYEC&TSIE("?170'_"R;C5DPA/O^4#_#H36H)'I`?8T&K>3@HUOJZ@'^A>@R +MN'D*K]N2P==E<;%*P[5Y8?7AQF&]P%*/->*Y+'F#Y);.]ML*4C^N+`Q.DU +ML:?K\4,!!SQ65YDQY(`_%$]D;#NQW;2R\^6SJ2?'Z=GOVH)WL"YN@;[_GF7; +MKWWPX87&I!X+I53Z=RSPJHOV<( +M*#*.X`C&C@\.GF)/G4Y!/3F3^/]!+NR+3+\\!.Q"[N^'#L,Z<2NY@]KJH(!4 +MM6:(5#ICV#E_';0`6CL'GAX?2-EN8^H`Q:S?JH?>Z5"!(3_]8453>Y"?%/]^ +MK.XC`3U+F,+ZZ8X`&0M/1:9_/>2GB]%W(;_RTQ,TT;;!+&@UY4$_'7W(,&Q+ +M64::PQ0-G5WH?5M76Q73^R:[X748(`?MM3L;*C;0WRE[EX'L4F48R*W$CYO* +M.*R9\1BL`M;2>([7@\IFSU1@.X9%QF25ZJPR@S44JMR*ZMV)7A>@0_IHW2'<>$QXBRW]!#_"]/_TXU]'M\@/V%`UY.Y<> +M^(;UM&K!<^3P,$U%J?.?G07]//AHKM:BDIA!M!D+.'0#]0-F;8C)R:V?X?53 +M3KZIE+O%014P5"E_SC]>6##K83HCTVY,E43VZHU:KB$U)%=ET/(!I;I5+GO` +M:V'WP`WE=JT]A%;1ITQV26`46;S*?9D4I\C">Q\2/R8KUA2HQ7^2;#.0$T7H +MO(N&/($;$SCV]K^&]H(T&G:?H6?I"8SWXGB"F++B?1MY'#^YPWCO7APX(_=Y +M-ND!GDTJ"%SSU,0YP4ID'MO*U18-?OP6FDG`]\*7QI/DE+0O#MF8P[3SL,FC +M]JC<&F$)6D19=>3K5&P+[VF<-)#!C0+16'GP`&FO64A!@SF3@A?B"_\.!Z6+?E0U"H&<30]LZ4WQ+ZSX1+=(ZLTQ^N*U<#=$`?6&>0 +M0QR>:V,*0(HRA$-.D]?&-'1*#H-,<6R,4?_93ZA((HRG>*(L7Z/LZL6NT`#RI]L/Z]QJM\JC0>ME@/=1N0M(W_7LQ4;QZ,W*GKWDX^!` +MQ$3W[$++]_Z`6XU@S[!;7=!>00>0E+O][=R0W%,@=^YVM+H?CX,/"_RV"^%S +MEL_D7=:#X])FO4>O/^.DPJ6IF/7S%\^I*B-\="%PGGCQK#G,G[IJCF_IK#FX +MG;75F\VU&!CKD)!IYG#IN53[YF<#]KJ&M==-A58QNXJB[&F-Z5EK4(F*52&- +M0K-7:_!JZ/4AC23=*ZH:G__Z#J8B5N1(=L4L1U\G&7U)RIXW(WH."WX2":X0 +M1;&N:"ISD7"`LP8=?1MSME[9L]2B(MV1U2/!2ZR.$C`"ZG:.[C!O!4(4E.YV +MMK)GOGU]+)59%=LS;TD.)-HLMSVZ<)G]UHMZN?2T4Z<*JW,+V4W-U\KA=4%! +MC+AX6EJ3]^`BL?`5\5/)/+=4^#3N_O/PUIQE'I;V[-C?X/>_*'N>C^V9C?3? +MK_%[--+CYWZ#WPNB37(IU;Q(I^*'.,@]@>;YQ5:J9='*']"W4M$]CQK!ZK?) +M]K3IFM%*Q"&1KB6;/6/X:RS_KHTI[)0`AA).LPW.KG]:V:.75,B$$0IW3$3/ +MTL"@4>%B&+.8GD6`?6R#)RLCZ`_UX:'LV;RE!1%Z>:AM,([V1O0`C(V)Z8DJ +MC$(*JB1SK[]HY-"WJSUA0]^NEOWS;UP!.]UYM%W9(QC9(QSRY=CU +M>'3NHN>![Z_!G?OE:\L@74>([K08'$WVI`;Z%.@>](Q^+ZK[[Y2:5WF3IMW? +M!9;0(KOON&X%3@,:M6&!XX"VP6EH,9#.*HR)[1GCDE5+[>W_X4\]N$B_2%D< +MHULM6/SSXSWM9P*[B>Q>8@]M;%.=68H2%)!`OST=W +M?^[J=5^H[*-IHY;_P/01M?C]^3.'RT%5^Y,,X0K:.V=PKT&N,,OEMGO706<$ +MWL$;7]*_/L%^6/H&B/!2<5P_6X.VCAIC13LYZRHOQG0R<'#&/#[ +MZ.Z=([N_.U;MJ!?NY`KS..7UAE/XVQDCHKJW1G%J.Z+,=V;(KI+3>.2FR-(]GWG"C)]Q]+330*\7H$_[\0=WLN(."T1U40;"GA+\4GFHY?68#=)J +MV9=+0[N/YH7EA^IKL/B)/:(Z*E^:$OQ\ODRTBRH(-X3S#"U1R"&R8F\<#RM0 +MZ)-5AD=49!V5$CR=/D7XQYE^1S5,W3][_YBW)IS;O[Q87A3N"&L8\#I<857A +MZ5=KANP:VK5$8QG9_0=H)X63QZY,SV%^'MG]/'U4Y\5;=Y.5.$G.-/X-)]%8 +MQ70GQW9O@L%!*&3X"]'`?$5U3QO9_11[\#13UX3_=5\PO$*3*>]V1G>7*+M+ +MQ'LXI@.9`A='()A)?8J#7)1\KFE[:&@^Z8MT[3CNP5H +M&3LWJCN+?`P8S-#\O;[YCFDROFKSXPUUYN8?]2UWUNP,TC?_`)%5.PW'JL$Z +MP!-%<(A=..:(%#JX^I;OX242\[F9`H9C83\2%%\<%:]]<65CN*7Y!UWSCZ*] +MI,1-AKA)H4[16IZ*?4\<)`64(BWC.ETB/(L+A3-UO\/9CN91!DWP2B_K?6EB +MT[ZP!W@_N$U@:LY9\C40>PJD(+I[HMW9JNP65ZUW.2TJ8N,&,3KBX5"A58-2 +MBXI?4:8OXULV$O'L^0M3,[C=*E-+=F!M,%#EMC9]-`A_5+0(A'I*ZKY#4Y3DU +M7480GO*N0-U$5#=/[O]'J'\'[0COQK11TC#_]K-)^I>68C4YY5+,F>=6ZN9B +MB_-F2JIE>)60>X\P]V)!C%`R(.,-$/J.189;%-%*BAQC\WN];KC5Y0D:2>J6 +MKJ+7T+%8?WJ1Z`0IJ!OKZ!,*I\"=<)"NOEUU^M.+O>4`[,I3A]=$%/X+T)4P +M__E0O\^1O'/,B:(YS!F:UH^^8QC]?64R,!DU^@?SZ!]%^TC!`"D9($/,BE0, +M3"7Q<2#*+TH*9*9G1?O?.K;$=`J/];\%0[]J\V1J](_&6DR_FZV=E;I!^ +M]`\06;7S]Q#!U#S$Y"F1P<#D8[^'=&1];F<*W!S=#T.3!\H3\..JG=&!OQ$` +M;Z$4]F)D"!;_;6*I+8#K4?)9@0I(4WS(0/N*C[`Z:#-PYG^T]^1">AI +MZ/_3]"@>):361EO^SKR6M3K:7TZ:97O]^:?RVF/\4U=M?M3165#34%?>OC*7 +MF]=1WKYJY_-Y'4"(KP:;$"EQ<\M/Z4^S?\+@?*;0P0%6HX8URS#]V@#]AO.( +M_@Y:/QG''HN4^+GB1H[ATI`W?4K200]#=D2?SZKPJ^D,L-'`]#K$]#PWH2M! +MZT+.P+I0@/X8_TC2(-O[(5HS(::PA_*SC;79]"@[_4ZT_U6%7[;G(]258\RJ +MS8\%>H+V5G)Y>1UP_U_2EV#DH-#M\58^9<4BNJX+GI(+'025.;4K692!C>RZ +M1CXCVU6GJ\T2_E&DVY85Z%>@3$17+Y2!D@\4B.KJ)J?+=C?J]V<1+XKT![)8 +M&TBO)?6RJ*XS([N^V@UUY>PR&.J3N&X!.K#KB'Q[+5]3FX6/%@WO+00;",RN +M2@WWQV=+0[N.:<&%_5!@V#-HV&_6[QD414R>O^')^24S%ZQ/2<5FHCW7QY#/ +MJJ_5F^K7$\3OC?5._>[;`O;;K#>//V:L+S'5;P@\KCQN-NBQ@V.T,SAO/+T3 +M&#+]DVEA?K&-:81JXA^Y8F/VQQ-V&[/;1!#"K\;;F/>6OB(**E)8'N/$WUEO +MON-C8FDX\">:"*%&BZ>R5%V]7,UN"9''E@O5]'U='*.=$17 +MP;'_*)4INO)3L5'^FM37BTZ20%%4ESNFZWWR,!7=]?[>MLI3-B:>':_8KIDQ7;KTC?Z7C&9P-[M;=]3O8>Y:I#--5\D1ZM$VWD_0X/>M- +M9;BIC`LPU<:4C>P:93R0@18^#JP=QB^3Y5TZ760S19@BNQ)&=A6&=A5!AKC1 +M)B1AP(DUF$B4)')-H.)C%M2"9.?,86:%=:V-[)II>.0*\0@GAL:X59C4=WO<8;P,6/4>9XW/0"EL+]FLVSFME0@FDUI2\F@H.#=+^U]>=\K.V9K +MG]]E>GZW]LXUTYWKVHL&TT6CZ>@B[=&_V!@3&@<]8340W#G,,_*N*48*>$>WK-ION'KEEN8123>U*[:41]O9/C]0" +M0&<>#=PF-.^CO13D*M'EH%Q:5/8A9#V(SI\\TGZ<'C::[S<446[6E942L +M[[J9C9M:1APYW909US(D4/IFN:B.PW9>WR)O>H]^/T`.U8)V&PS#VP8"&W,PVA,Y+Y+H5VRZ"Y99)<\IDL>U26GD^CG1_K.A_L\%"%(0-A@ +M3(_JC9-XL(8OB)U*[5NQ]>\\;=%K$S+4KA_W_K3GYSI&4Y81EYWE +M'1._K$"[+%^S+$];5FTIJ[*4>;P6^F>E[W:$[W:,[W:4[^-(WVUR$372=XL4 +M!^\X4W=NEZ^J7KRS^R]W:+W=IOZPS;+MDV/:-=MOE81NB +M]/6P.BXTPM>87ARZ6"!"`^T9U->&IN_CV9C^3HG<]W6D;[;2]SZ;\3':*S@X +M@=JBA*?,>+MJ*5T_PK]SVDH_7U`3[2NJE$;Y&A?KII^K?_]` +M\%_P-_X2IO0YWCK)RY>*_D(*#E*ZWP]N=U!E'%]-A"]S)9N^BQ0>HKR[]I2% +MN`02MX`J\U#HSQMEQ)55Q]GSX^T%$;Z22-\8H^>>L5D,AH:,4K1^@71*L+<1 +MQ8_8F-OHG)S9'&_9JK5LU9BWQ@V/E=RW5.G+C/!M +MB/2],=+W>I1O:;1O"9T48KDQW([/L:WMAJ=YB[J5_C^&/*RPCPV:`XS>PXC07K7 +MMZJIW@1L_G?&?/I1-#>C<./I1X84Q\-X"MXY9_#(G$$/JNKW=^-_,,T9_-/0 +M/G^H;W&6-7[*LVTOQ$7.>.]W\4\^FS6KZ?%ECW"/64Y)@IX@2D9[G]@PZQ=L +MI/3]V9$1G_GLB>1W9]4OG[=;:ZV,6P_4,R%3&GKN'%O/@Q5T6CECV/)CUF-K.6G2I]?B:=*I:[E?Q`[7)ARN +M[?8#M<6CVB[_NC9:NIZ['E^/:OW2RGD"4HL*&_!%H:OO'_J>SH +M8YJXXN_>NZO8WO5:H!WE\WK79+^!$R9^+`;/MC<>G>M5`_ +M!NJ:IKGW>[_/=[][O_>N=[_?"_!7["7]G2F)UQ@T@.&85;("T4H1Q,1KD((` +M.2#MF+W!9&01YG0@0_WG0$HOF2FQ$>!>&TOA3U-[T[@'^GTA*A->.,.'0+MM +M?^J!-`[*1B#SE+P_D;4"(Y(SU'LF]7>.*!N>'?6#UM?;EL(QN;5VYW)-SH#< +M5K>K'J"GY8X,9LMIZO54]3J;9V9/LG,^3Q]RYYY(=YS(,/2Q(8'KG$^N*>DQ +M0/Q9E!&R'E6%%9%[L5Q@NF"^.UV]E*9>-.:8#;GZ'XI:^]^;;^WJ9YUL:-2F +M7LQ0O^=OEIPZU1,F/./@.7MS +M+G0?%?J/'9NSSW'^^H7!/@=AHF4%!1*A,5TZHF'.+`.38>FX:$Q?+X,0@^JI&`EI,=D +M!#T$(2HGWL=XF!?9&`935/H(TJ1^49[1M3R\(B2K'Q+)K>6[RKK*T2,RA_E$ +M931)G=TEA,S4PQ2G8%P&$S*X+XOZX\1`L]J2I+:@,3E9;2']6F92=8M>RTQZ +M%9N5,'T/.ZS(%WE`^0@H4?7%/+'+U&G6_%#U/NJ'F:J?OXG9V[C]!ANAB%?V +M):T9;^E-[$P1_XZP$3I+^0F$:)ZW\2+B;Q&W>^/+(6[(T'[VP&C?#2Y"\;ZS<13^++4OK3UTW;3_#!]^TG\SU6K>"MI']R5I8G^A"`O_>&IK +M2N^-O$1(]J4FM31#+6:1/.7:7(3.5,M/C1B'#/L'880R1JA,M<0?>3-+Z0=A +MFN=L!P:Y6S(O0DXD1OMA%1X0@4"C:HR@C2S\U^'O$%I%+F6(=-!%(3<-9!I" +M&ZR6":Z8EX4D%D4'&E;C-<([VYT)VYRV]YVE7K!7ZAWOQH?DPX6?N*`UF4H@ +M@75$&HG#I)5_M&%IP>@1W",?+3SF@CG)4,/YEF#FCXQWA`\.=U[NNB*E$[3\ +M$9.T(#Q%>:2H9V[./%WNO%E\`@U?3D8:V0Y"1E0B&\F0G6Q5B(+:V1?"?,@> +MYGG1+.91J`HW.@W2@@AI-K[T%T(>8@R0S$A,@7`)H4`(GQ7B%H7$\*3_694) +M9@S_"'0UU(`8<\9ED:_#9N/T@9GH)7WJ,VU?.2/_,Z7/8?MB +MJ[(W7G;,_EC^X^>I0691%A#I,U4@BTY!\8]%D:U*X_^K<691'$_A/VE#FL9S +M9@4L2M)3>3S,]TQX)5B5JKB*6=.,QQ7^"O^XMF-Q;9^P]^<_KX$Q*#)H@APP08E!]V4P-?<,.>`X)E&=],.)Z(%$P_MX:KT! +M1?UD3<#LXK+LXLKLXJKLXL6Q^GZQN!_[[P=N*IQ_56""-.VFFUGY1Q?<4<_S.L!\*SW +M;VAJ"`JOU`N5BRKJ7EV(A>R*MQLV-`OU_C5">>-F02X1<$EI84DI=@E+*VN% +M0HR+A>`F8:U_0\/6@M5-#;XU_J:&=9O\FYL*?,&-I<[UP8U^9[37N<6Y=LLJ +M5X%K3IZPG/0W$+):U]:M`L9%!1B3KY`OR*2!R<=$`U"P[/+J?P&C)[&/F'$` +!```` +` +end Property changes on: head/sys/dev/cxgb/t3fw-4.0.0.bin.gz.uu ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/modules/cxgb/Makefile =================================================================== --- head/sys/modules/cxgb/Makefile (revision 169977) +++ head/sys/modules/cxgb/Makefile (revision 169978) @@ -1,25 +1,27 @@ # $FreeBSD$ CXGB = ${.CURDIR}/../../dev/cxgb .PATH: ${CXGB} ${CXGB}/common ${CXGB}/sys KMOD= if_cxgb SRCS= cxgb_mc5.c cxgb_vsc8211.c cxgb_ael1002.c cxgb_mv88e1xxx.c SRCS+= cxgb_xgmac.c cxgb_t3_hw.c cxgb_main.c cxgb_sge.c cxgb_lro.c -SRCS+= device_if.h bus_if.h pci_if.h +SRCS+= cxgb_offload.c cxgb_l2t.c +SRCS+= device_if.h bus_if.h pci_if.h opt_zero.h SRCS+= uipc_mvec.c -CFLAGS+= -DCONFIG_CHELSIO_T3_CORE -g -DDEFAULT_JUMBO -DCONFIG_DEFINED +CFLAGS+= -DCONFIG_CHELSIO_T3_CORE -g -DDEFAULT_JUMBO -DCONFIG_DEFINED +#CFLAGS+= -DINVARIANT_SUPPORT -DINVARIANTS -DDEBUG .if ${MACHINE_ARCH} != "ia64" # ld is broken on ia64 -t3fw-3.2.bin: ${CXGB}/t3fw-3.2.bin.gz.uu - uudecode -p < ${CXGB}/t3fw-3.2.bin.gz.uu \ +t3fw-4.0.0.bin: ${CXGB}/t3fw-4.0.0.bin.gz.uu + uudecode -p < ${CXGB}/t3fw-4.0.0.bin.gz.uu \ | gzip -dc > ${.TARGET} -FIRMWS= t3fw-3.2.bin:t3fw32 -CLEANFILES+= t3fw-3.2.bin +FIRMWS= t3fw-4.0.0.bin:t3fw400 +CLEANFILES+= t3fw-4.0.0.bin .endif .include