Index: head/sys/conf/options =================================================================== --- head/sys/conf/options (revision 167053) +++ head/sys/conf/options (revision 167054) @@ -1,762 +1,763 @@ # $FreeBSD$ # # On the handling of kernel options # # All kernel options should be listed in NOTES, with suitable # descriptions. Negative options (options that make some code not # compile) should be commented out; LINT (generated from NOTES) should # compile as much code as possible. Try to structure option-using # code so that a single option only switch code on, or only switch # code off, to make it possible to have a full compile-test. If # necessary, you can check for COMPILING_LINT to get maximum code # coverage. # # All new options shall also be listed in either "conf/options" or # "conf/options.". Options that affect a single source-file # .[c|s] should be directed into "opt_.h", while options # that affect multiple files should either go in "opt_global.h" if # this is a kernel-wide option (used just about everywhere), or in # "opt_.h" if it affects only some files. # Note that the effect of listing only an option without a # header-file-name in conf/options (and cousins) is that the last # convention is followed. # # This handling scheme is not yet fully implemented. # # # Format of this file: # Option name filename # # If filename is missing, the default is # opt_.h AAC_DEBUG opt_aac.h AHC_ALLOW_MEMIO opt_aic7xxx.h AHC_TMODE_ENABLE opt_aic7xxx.h AHC_DUMP_EEPROM opt_aic7xxx.h AHC_DEBUG opt_aic7xxx.h AHC_DEBUG_OPTS opt_aic7xxx.h AHC_REG_PRETTY_PRINT opt_aic7xxx.h AHD_DEBUG opt_aic79xx.h AHD_DEBUG_OPTS opt_aic79xx.h AHD_TMODE_ENABLE opt_aic79xx.h AHD_REG_PRETTY_PRINT opt_aic79xx.h ADW_ALLOW_MEMIO opt_adw.h TWA_DEBUG opt_twa.h TWA_FLASH_FIRMWARE opt_twa.h # Debugging options. DDB DDB_NUMSYM opt_ddb.h GDB KDB opt_global.h KDB_TRACE opt_kdb.h KDB_UNATTENDED opt_kdb.h SYSCTL_DEBUG opt_sysctl.h NO_SYSCTL_DESCR opt_global.h # Miscellaneous options. ADAPTIVE_GIANT opt_adaptive_mutexes.h NO_ADAPTIVE_MUTEXES opt_adaptive_mutexes.h ALQ AUDIT opt_global.h CODA_COMPAT_5 opt_coda.h COMPAT_43 opt_compat.h COMPAT_43TTY opt_compat.h COMPAT_FREEBSD4 opt_compat.h COMPAT_FREEBSD5 opt_compat.h COMPAT_FREEBSD6 opt_compat.h COMPILING_LINT opt_global.h CY_PCI_FASTINTR DIRECTIO FULL_PREEMPTION opt_sched.h IPI_PREEMPTION opt_sched.h GEOM_AES opt_geom.h GEOM_BDE opt_geom.h GEOM_BSD opt_geom.h GEOM_CACHE opt_geom.h GEOM_CONCAT opt_geom.h GEOM_ELI opt_geom.h GEOM_FOX opt_geom.h GEOM_GATE opt_geom.h GEOM_JOURNAL opt_geom.h GEOM_LABEL opt_geom.h GEOM_MBR opt_geom.h GEOM_MIRROR opt_geom.h GEOM_NOP opt_geom.h GEOM_PART_APM opt_geom.h GEOM_PART_GPT opt_geom.h GEOM_PC98 opt_geom.h GEOM_RAID3 opt_geom.h GEOM_SHSEC opt_geom.h GEOM_STRIPE opt_geom.h GEOM_SUNLABEL opt_geom.h GEOM_UZIP opt_geom.h GEOM_VOL opt_geom.h GEOM_ZERO opt_geom.h KSE opt_global.h KSTACK_MAX_PAGES KSTACK_PAGES KTRACE KTRACE_REQUEST_POOL opt_ktrace.h LIBICONV MAC MAC_ALWAYS_LABEL_MBUF opt_mac.h MAC_BIBA opt_dontuse.h MAC_BSDEXTENDED opt_dontuse.h MAC_IFOFF opt_dontuse.h MAC_LOMAC opt_dontuse.h MAC_MLS opt_dontuse.h MAC_NONE opt_dontuse.h MAC_PARTITION opt_dontuse.h MAC_PORTACL opt_dontuse.h MAC_SEEOTHERUIDS opt_dontuse.h MAC_STATIC opt_mac.h MAC_STUB opt_dontuse.h MAC_TEST opt_dontuse.h MD_ROOT opt_md.h MD_ROOT_SIZE opt_md.h MFI_DEBUG opt_mfi.h MFI_DECODE_LOG opt_mfi.h MPROF_BUFFERS opt_mprof.h MPROF_HASH_SIZE opt_mprof.h MUTEX_WAKE_ALL NSWBUF_MIN opt_swap.h PANIC_REBOOT_WAIT_TIME opt_panic.h PPC_DEBUG opt_ppc.h PPC_PROBE_CHIPSET opt_ppc.h PPS_SYNC opt_ntp.h PREEMPTION opt_sched.h QUOTA SCHED_4BSD opt_sched.h SCHED_CORE opt_sched.h SCHED_ULE opt_sched.h SHOW_BUSYBUFS SLEEPQUEUE_PROFILING SLHCI_DEBUG opt_slhci.h SPX_HACK SUIDDIR MSGMNB opt_sysvipc.h MSGMNI opt_sysvipc.h MSGSEG opt_sysvipc.h MSGSSZ opt_sysvipc.h MSGTQL opt_sysvipc.h SEMMAP opt_sysvipc.h SEMMNI opt_sysvipc.h SEMMNS opt_sysvipc.h SEMMNU opt_sysvipc.h SEMMSL opt_sysvipc.h SEMOPM opt_sysvipc.h SEMUME opt_sysvipc.h SHMALL opt_sysvipc.h SHMMAX opt_sysvipc.h SHMMAXPGS opt_sysvipc.h SHMMIN opt_sysvipc.h SHMMNI opt_sysvipc.h SHMSEG opt_sysvipc.h SYSVMSG opt_sysvipc.h SYSVSEM opt_sysvipc.h SYSVSHM opt_sysvipc.h SW_WATCHDOG opt_watchdog.h TURNSTILE_PROFILING TTYHOG opt_tty.h VFS_AIO VERBOSE_SYSINIT opt_global.h WLCACHE opt_wavelan.h WLDEBUG opt_wavelan.h # POSIX kernel options P1003_1B_MQUEUE opt_posix.h P1003_1B_SEMAPHORES opt_posix.h _KPOSIX_PRIORITY_SCHEDULING opt_posix.h # Do we want the config file compiled into the kernel? INCLUDE_CONFIG_FILE opt_config.h # Options for static filesystems. These should only be used at config # time, since the corresponding lkms cannot work if there are any static # dependencies. Unusability is enforced by hiding the defines for the # options in a never-included header. CD9660 opt_dontuse.h CODA opt_dontuse.h EXT2FS opt_dontuse.h FDESCFS opt_dontuse.h HPFS opt_dontuse.h MSDOSFS opt_dontuse.h NTFS opt_dontuse.h NULLFS opt_dontuse.h NWFS opt_dontuse.h PORTALFS opt_dontuse.h PROCFS opt_dontuse.h PSEUDOFS opt_dontuse.h REISERFS opt_dontuse.h SMBFS opt_dontuse.h UDF opt_dontuse.h UMAPFS opt_dontuse.h UNIONFS opt_dontuse.h # Pseudofs debugging PSEUDOFS_TRACE opt_pseudofs.h # Broken - ffs_snapshot() dependency from ufs_lookup() :-( FFS opt_ffs_broken_fixme.h # These static filesystems have one slightly bogus static dependency in # sys/i386/i386/autoconf.c. If any of these filesystems are # statically compiled into the kernel, code for mounting them as root # filesystems will be enabled - but look below. NFSCLIENT opt_nfs.h NFSSERVER opt_nfs.h NFS4CLIENT opt_nfs.h # filesystems and libiconv bridge CD9660_ICONV opt_dontuse.h MSDOSFS_ICONV opt_dontuse.h NTFS_ICONV opt_dontuse.h UDF_ICONV opt_dontuse.h # If you are following the conditions in the copyright, # you can enable soft-updates which will speed up a lot of thigs # and make the system safer from crashes at the same time. # otherwise a STUB module will be compiled in. SOFTUPDATES opt_ffs.h # On small, embedded systems, it can be useful to turn off support for # snapshots. It saves about 30-40k for a feature that would be lightly # used, if it is used at all. NO_FFS_SNAPSHOT opt_ffs.h # Enabling this option turns on support for Access Control Lists in UFS, # which can be used to support high security configurations. Depends on # UFS_EXTATTR. UFS_ACL opt_ufs.h # Enabling this option turns on support for extended attributes in UFS-based # filesystems, which can be used to support high security configurations # as well as new filesystem features. UFS_EXTATTR opt_ufs.h UFS_EXTATTR_AUTOSTART opt_ufs.h # Enable fast hash lookups for large directories on UFS-based filesystems. UFS_DIRHASH opt_ufs.h # Enable gjournal-based UFS journal. UFS_GJOURNAL opt_ufs.h # The below sentence is not in English, and neither is this one. # We plan to remove the static dependences above, with a # _ROOT option to control if it usable as root. This list # allows these options to be present in config files already (though # they won't make any difference yet). NFS_ROOT opt_nfsroot.h # SMB/CIFS requester NETSMB opt_netsmb.h # Options used only in subr_param.c. HZ opt_param.h MAXFILES opt_param.h NBUF opt_param.h NSFBUFS opt_param.h VM_BCACHE_SIZE_MAX opt_param.h VM_SWZONE_SIZE_MAX opt_param.h MAXUSERS DFLDSIZ opt_param.h MAXDSIZ opt_param.h MAXSSIZ opt_param.h # Generic SCSI options. CAM_MAX_HIGHPOWER opt_cam.h CAMDEBUG opt_cam.h CAM_DEBUG_DELAY opt_cam.h CAM_DEBUG_BUS opt_cam.h CAM_DEBUG_TARGET opt_cam.h CAM_DEBUG_LUN opt_cam.h CAM_DEBUG_FLAGS opt_cam.h SCSI_DELAY opt_scsi.h SCSI_NO_SENSE_STRINGS opt_scsi.h SCSI_NO_OP_STRINGS opt_scsi.h # Options used only in cam/scsi/scsi_cd.c CHANGER_MIN_BUSY_SECONDS opt_cd.h CHANGER_MAX_BUSY_SECONDS opt_cd.h # Options used only in cam/scsi/scsi_sa.c. SA_IO_TIMEOUT opt_sa.h SA_SPACE_TIMEOUT opt_sa.h SA_REWIND_TIMEOUT opt_sa.h SA_ERASE_TIMEOUT opt_sa.h SA_1FM_AT_EOD opt_sa.h # Options used only in cam/scsi/scsi_pt.c SCSI_PT_DEFAULT_TIMEOUT opt_pt.h # Options used only in cam/scsi/scsi_ses.c SES_ENABLE_PASSTHROUGH opt_ses.h # Options used in dev/sym/ (Symbios SCSI driver). SYM_SETUP_LP_PROBE_MAP opt_sym.h #-Low Priority Probe Map (bits) # Allows the ncr to take precedence # 1 (1<<0) -> 810a, 860 # 2 (1<<1) -> 825a, 875, 885, 895 # 4 (1<<2) -> 895a, 896, 1510d SYM_SETUP_SCSI_DIFF opt_sym.h #-HVD support for 825a, 875, 885 # disabled:0 (default), enabled:1 SYM_SETUP_PCI_PARITY opt_sym.h #-PCI parity checking # disabled:0, enabled:1 (default) SYM_SETUP_MAX_LUN opt_sym.h #-Number of LUNs supported # default:8, range:[1..64] # Options used only in pci/ncr.c SCSI_NCR_DEBUG opt_ncr.h SCSI_NCR_MAX_SYNC opt_ncr.h SCSI_NCR_MAX_WIDE opt_ncr.h SCSI_NCR_MYADDR opt_ncr.h # Options used only in dev/isp/* ISP_TARGET_MODE opt_isp.h ISP_FW_CRASH_DUMP opt_isp.h ISP_DEFAULT_ROLES opt_isp.h # Options used in the 'ata' ATA/ATAPI driver ATA_STATIC_ID opt_ata.h ATA_NOPCI opt_ata.h # Net stuff. ACCEPT_FILTER_DATA ACCEPT_FILTER_HTTP ALTQ opt_global.h ALTQ_CBQ opt_altq.h ALTQ_RED opt_altq.h ALTQ_RIO opt_altq.h ALTQ_HFSC opt_altq.h ALTQ_CDNR opt_altq.h ALTQ_PRIQ opt_altq.h ALTQ_NOPCC opt_altq.h ALTQ_DEBUG opt_altq.h BOOTP opt_bootp.h BOOTP_COMPAT opt_bootp.h BOOTP_NFSROOT opt_bootp.h BOOTP_NFSV3 opt_bootp.h BOOTP_WIRED_TO opt_bootp.h DEVICE_POLLING DEV_ENC opt_enc.h DEV_PF opt_pf.h DEV_PFLOG opt_pf.h DEV_PFSYNC opt_pf.h PF_MPSAFE_UGID opt_pf.h ETHER_II opt_ef.h ETHER_8023 opt_ef.h ETHER_8022 opt_ef.h ETHER_SNAP opt_ef.h MROUTING opt_mrouting.h INET opt_inet.h INET6 opt_inet6.h IPSEC opt_ipsec.h IPSEC_ESP opt_ipsec.h IPSEC_DEBUG opt_ipsec.h IPSEC_FILTERGIF opt_ipsec.h FAST_IPSEC opt_ipsec.h IPDIVERT DUMMYNET opt_ipdn.h IPFILTER opt_ipfilter.h IPFILTER_LOG opt_ipfilter.h IPFILTER_LOOKUP opt_ipfilter.h IPFILTER_DEFAULT_BLOCK opt_ipfilter.h IPFIREWALL opt_ipfw.h IPFIREWALL_VERBOSE opt_ipfw.h IPFIREWALL_VERBOSE_LIMIT opt_ipfw.h IPFIREWALL_DEFAULT_TO_ACCEPT opt_ipfw.h IPFIREWALL_FORWARD opt_ipfw.h IPFIREWALL_NAT opt_ipfw.h IPSTEALTH IPX IPXIP opt_ipx.h LIBMBPOOL LIBMCHAIN LIBALIAS MBUF_STRESS_TEST NCP NETATALK opt_atalk.h NET_WITH_GIANT opt_net.h PPP_BSDCOMP opt_ppp.h PPP_DEFLATE opt_ppp.h PPP_FILTER opt_ppp.h SLIP_IFF_OPTS opt_slip.h TCPDEBUG TCP_SIGNATURE opt_inet.h TCP_SACK_DEBUG opt_tcp_sack.h TCP_DROP_SYNFIN opt_tcp_input.h DEV_VLAN opt_vlan.h VLAN_ARRAY opt_vlan.h XBONEHACK # # SCTP # SCTP opt_sctp.h SCTP_DEBUG opt_sctp.h SCTP_HIGH_SPEED opt_sctp.h SCTP_LOG_MAXBURST opt_sctp.h SCTP_LOG_RWND opt_sctp.h SCTP_STAT_LOGGING opt_sctp.h SCTP_CWND_LOGGING opt_sctp.h SCTP_CWND_MONITOR opt_sctp.h SCTP_BLK_LOGGING opt_sctp.h SCTP_STR_LOGGING opt_sctp.h SCTP_FR_LOGGING opt_sctp.h SCTP_MAP_LOGGING opt_sctp.h SCTP_SACK_LOGGING opt_sctp.h SCTP_LOCK_LOGGING opt_sctp.h SCTP_RTTVAR_LOGGING opt_sctp.h SCTP_SB_LOGGING opt_sctp.h SCTP_WITH_NO_CSUM opt_sctp.h SCTP_EARLYFR_LOGGING opt_sctp.h SCTP_NAGLE_LOGGING opt_sctp.h SCTP_WAKE_LOGGING opt_sctp.h SCTP_RECV_RWND_LOGGING opt_sctp.h SCTP_SACK_RWND_LOGGING opt_sctp.h SCTP_MBUF_LOGGING opt_sctp.h # Netgraph(4). Use option NETGRAPH to enable the base netgraph code. # Each netgraph node type can be either be compiled into the kernel # or loaded dynamically. To get the former, include the corresponding # option below. Each type has its own man page, e.g. ng_async(4). NETGRAPH NETGRAPH_DEBUG opt_netgraph.h NETGRAPH_ASYNC opt_netgraph.h NETGRAPH_ATMLLC opt_netgraph.h NETGRAPH_ATM_ATMPIF opt_netgraph.h NETGRAPH_BLUETOOTH opt_netgraph.h NETGRAPH_BLUETOOTH_BT3C opt_netgraph.h NETGRAPH_BLUETOOTH_H4 opt_netgraph.h NETGRAPH_BLUETOOTH_HCI opt_netgraph.h NETGRAPH_BLUETOOTH_L2CAP opt_netgraph.h NETGRAPH_BLUETOOTH_SOCKET opt_netgraph.h NETGRAPH_BLUETOOTH_UBT opt_netgraph.h NETGRAPH_BLUETOOTH_UBTBCMFW opt_netgraph.h NETGRAPH_BPF opt_netgraph.h NETGRAPH_BRIDGE opt_netgraph.h NETGRAPH_CISCO opt_netgraph.h NETGRAPH_DEFLATE opt_netgraph.h NETGRAPH_DEVICE opt_netgraph.h NETGRAPH_ECHO opt_netgraph.h NETGRAPH_EIFACE opt_netgraph.h NETGRAPH_ETHER opt_netgraph.h NETGRAPH_FEC opt_netgraph.h NETGRAPH_FRAME_RELAY opt_netgraph.h NETGRAPH_GIF opt_netgraph.h NETGRAPH_GIF_DEMUX opt_netgraph.h NETGRAPH_HOLE opt_netgraph.h NETGRAPH_IFACE opt_netgraph.h NETGRAPH_IP_INPUT opt_netgraph.h NETGRAPH_IPFW opt_netgraph.h NETGRAPH_KSOCKET opt_netgraph.h NETGRAPH_L2TP opt_netgraph.h NETGRAPH_LMI opt_netgraph.h # MPPC compression requires proprietary files (not included) NETGRAPH_MPPC_COMPRESSION opt_netgraph.h NETGRAPH_MPPC_ENCRYPTION opt_netgraph.h NETGRAPH_NAT opt_netgraph.h NETGRAPH_NETFLOW opt_netgraph.h NETGRAPH_ONE2MANY opt_netgraph.h NETGRAPH_PPP opt_netgraph.h NETGRAPH_PPPOE opt_netgraph.h NETGRAPH_PPTPGRE opt_netgraph.h NETGRAPH_PRED1 opt_netgraph.h NETGRAPH_RFC1490 opt_netgraph.h NETGRAPH_SOCKET opt_netgraph.h NETGRAPH_SPLIT opt_netgraph.h NETGRAPH_SPPP opt_netgraph.h NETGRAPH_TAG opt_netgraph.h NETGRAPH_TCPMSS opt_netgraph.h NETGRAPH_TEE opt_netgraph.h NETGRAPH_TTY opt_netgraph.h NETGRAPH_UI opt_netgraph.h NETGRAPH_VJC opt_netgraph.h # NgATM options NGATM_ATM opt_netgraph.h NGATM_ATMBASE opt_netgraph.h NGATM_SSCOP opt_netgraph.h NGATM_SSCFU opt_netgraph.h NGATM_UNI opt_netgraph.h NGATM_CCATM opt_netgraph.h # DRM options DRM_DEBUG opt_drm.h ZERO_COPY_SOCKETS opt_zero.h TI_PRIVATE_JUMBOS opt_ti.h TI_JUMBO_HDRSPLIT opt_ti.h # ATM (HARP version) ATM_CORE opt_atm.h ATM_IP opt_atm.h ATM_SIGPVC opt_atm.h ATM_SPANS opt_atm.h ATM_UNI opt_atm.h # XXX Conflict: # of devices vs network protocol (Native ATM). # This makes "atm.h" unusable. NATM # DPT driver debug flags DPT_MEASURE_PERFORMANCE opt_dpt.h DPT_HANDLE_TIMEOUTS opt_dpt.h DPT_TIMEOUT_FACTOR opt_dpt.h DPT_LOST_IRQ opt_dpt.h DPT_RESET_HBA opt_dpt.h # Misc debug flags. Most of these should probably be replaced with # 'DEBUG', and then let people recompile just the interesting modules # with 'make CC="cc -DDEBUG"'. CLUSTERDEBUG opt_debug_cluster.h DEBUG_1284 opt_ppb_1284.h VP0_DEBUG opt_vpo.h LPT_DEBUG opt_lpt.h PLIP_DEBUG opt_plip.h LOCKF_DEBUG opt_debug_lockf.h NETATALKDEBUG opt_atalk.h SI_DEBUG opt_debug_si.h # Fb options FB_DEBUG opt_fb.h FB_INSTALL_CDEV opt_fb.h # ppbus related options PERIPH_1284 opt_ppb_1284.h DONTPROBE_1284 opt_ppb_1284.h # smbus related options ENABLE_ALART opt_intpm.h # These cause changes all over the kernel BLKDEV_IOSIZE opt_global.h MAXPHYS opt_global.h DFLTPHYS opt_global.h BURN_BRIDGES opt_global.h DEBUG opt_global.h DEBUG_LOCKS opt_global.h DEBUG_VFS_LOCKS opt_global.h DIAGNOSTIC opt_global.h INVARIANT_SUPPORT opt_global.h INVARIANTS opt_global.h MCLSHIFT opt_global.h MUTEX_DEBUG opt_global.h MUTEX_NOINLINE opt_global.h LOCK_PROFILING opt_global.h +LOCK_PROFILING_FAST opt_global.h MSIZE opt_global.h REGRESSION opt_global.h RESTARTABLE_PANICS opt_global.h RWLOCK_NOINLINE opt_global.h VFS_BIO_DEBUG opt_global.h # These are VM related options VM_KMEM_SIZE opt_vm.h VM_KMEM_SIZE_SCALE opt_vm.h VM_KMEM_SIZE_MAX opt_vm.h NO_SWAPPING opt_vm.h MALLOC_MAKE_FAILURES opt_vm.h MALLOC_PROFILE opt_vm.h PQ_NOOPT opt_vmpage.h # The MemGuard replacement allocator used for tamper-after-free detection DEBUG_MEMGUARD opt_vm.h # The RedZone malloc(9) protection DEBUG_REDZONE opt_vm.h # Standard SMP options SMP opt_global.h # Size of the kernel message buffer MSGBUF_SIZE opt_msgbuf.h # NFS options NFS_MINATTRTIMO opt_nfs.h NFS_MAXATTRTIMO opt_nfs.h NFS_MINDIRATTRTIMO opt_nfs.h NFS_MAXDIRATTRTIMO opt_nfs.h NFS_GATHERDELAY opt_nfs.h NFS_WDELAYHASHSIZ opt_nfs.h NFS_DEBUG opt_nfs.h # For the Bt848/Bt848A/Bt849/Bt878/Bt879 driver OVERRIDE_CARD opt_bktr.h OVERRIDE_TUNER opt_bktr.h OVERRIDE_DBX opt_bktr.h OVERRIDE_MSP opt_bktr.h BROOKTREE_SYSTEM_DEFAULT opt_bktr.h BROOKTREE_ALLOC_PAGES opt_bktr.h BKTR_OVERRIDE_CARD opt_bktr.h BKTR_OVERRIDE_TUNER opt_bktr.h BKTR_OVERRIDE_DBX opt_bktr.h BKTR_OVERRIDE_MSP opt_bktr.h BKTR_SYSTEM_DEFAULT opt_bktr.h BKTR_ALLOC_PAGES opt_bktr.h BKTR_USE_PLL opt_bktr.h BKTR_GPIO_ACCESS opt_bktr.h BKTR_NO_MSP_RESET opt_bktr.h BKTR_430_FX_MODE opt_bktr.h BKTR_SIS_VIA_MODE opt_bktr.h BKTR_USE_FREEBSD_SMBUS opt_bktr.h BKTR_NEW_MSP34XX_DRIVER opt_bktr.h BREAK_TO_DEBUGGER opt_comconsole.h ALT_BREAK_TO_DEBUGGER opt_comconsole.h # Options to support PPS UART_PPS_ON_CTS opt_uart.h # options for bus/device framework BUS_DEBUG opt_bus.h # options for USB support USB_DEBUG opt_usb.h USBVERBOSE opt_usb.h UKBD_DFLT_KEYMAP opt_ukbd.h UPLCOM_INTR_INTERVAL opt_uplcom.h UVSCOM_DEFAULT_OPKTSIZE opt_uvscom.h UVSCOM_INTR_INTERVAL opt_uvscom.h # Embedded system options INIT_PATH ROOTDEVNAME FDC_DEBUG opt_fdc.h PCFCLOCK_VERBOSE opt_pcfclock.h PCFCLOCK_MAX_RETRIES opt_pcfclock.h KTR opt_global.h KTR_ALQ opt_ktr.h KTR_MASK opt_ktr.h KTR_CPUMASK opt_ktr.h KTR_COMPILE opt_global.h KTR_ENTRIES opt_global.h KTR_VERBOSE opt_ktr.h WITNESS opt_global.h WITNESS_KDB opt_witness.h WITNESS_SKIPSPIN opt_witness.h # options for ACPI support ACPI_DEBUG opt_acpi.h ACPI_MAX_THREADS opt_acpi.h ACPI_NO_SEMAPHORES opt_acpi.h ACPICA_PEDANTIC opt_acpi.h # ISA support DEV_ISA opt_isa.h ISAPNP opt_isa.h # various 'device presence' options. DEV_BPF opt_bpf.h DEV_MCA opt_mca.h DEV_CARP opt_carp.h DEV_SPLASH opt_splash.h # EISA support DEV_EISA opt_eisa.h EISA_SLOTS opt_eisa.h # ed driver ED_HPP opt_ed.h ED_3C503 opt_ed.h ED_SIC opt_ed.h # bce driver BCE_DEBUG opt_bce.h # wi driver WI_SYMBOL_FIRMWARE opt_wi.h SOCKBUF_DEBUG opt_global.h # options for ubsec driver UBSEC_DEBUG opt_ubsec.h UBSEC_RNDTEST opt_ubsec.h UBSEC_NO_RNG opt_ubsec.h # options for hifn driver HIFN_DEBUG opt_hifn.h HIFN_RNDTEST opt_hifn.h # options for safenet driver SAFE_DEBUG opt_safe.h SAFE_NO_RNG opt_safe.h SAFE_RNDTEST opt_safe.h # syscons options MAXCONS opt_syscons.h SC_ALT_MOUSE_IMAGE opt_syscons.h SC_CUT_SPACES2TABS opt_syscons.h SC_CUT_SEPCHARS opt_syscons.h SC_DEBUG_LEVEL opt_syscons.h SC_DFLT_FONT opt_syscons.h SC_DISABLE_KDBKEY opt_syscons.h SC_DISABLE_REBOOT opt_syscons.h SC_HISTORY_SIZE opt_syscons.h SC_KERNEL_CONS_ATTR opt_syscons.h SC_KERNEL_CONS_REV_ATTR opt_syscons.h SC_MOUSE_CHAR opt_syscons.h SC_NO_CUTPASTE opt_syscons.h SC_NO_FONT_LOADING opt_syscons.h SC_NO_HISTORY opt_syscons.h SC_NO_MODE_CHANGE opt_syscons.h SC_NO_SUSPEND_VTYSWITCH opt_syscons.h SC_NO_SYSMOUSE opt_syscons.h SC_NORM_ATTR opt_syscons.h SC_NORM_REV_ATTR opt_syscons.h SC_PIXEL_MODE opt_syscons.h SC_RENDER_DEBUG opt_syscons.h SC_TWOBUTTON_MOUSE opt_syscons.h # options for printf PRINTF_BUFR_SIZE opt_printf.h # kbd options KBD_DISABLE_KEYMAP_LOAD opt_kbd.h KBD_INSTALL_CDEV opt_kbd.h KBD_MAXRETRY opt_kbd.h KBD_MAXWAIT opt_kbd.h KBD_RESETDELAY opt_kbd.h KBDIO_DEBUG opt_kbd.h # options for the Atheros driver ATH_DEBUG opt_ath.h ATH_TXBUF opt_ath.h ATH_RXBUF opt_ath.h ATH_DIAGAPI opt_ath.h ATH_TX99_DIAG opt_ath.h # dcons options DCONS_BUF_SIZE opt_dcons.h DCONS_POLL_HZ opt_dcons.h DCONS_FORCE_CONSOLE opt_dcons.h DCONS_FORCE_GDB opt_dcons.h # Static unit counts NI4BTRC opt_i4b.h NI4BRBCH opt_i4b.h NI4BTEL opt_i4b.h NI4BIPR opt_i4b.h NI4BING opt_i4b.h NI4BISPPP opt_i4b.h # VFS options LOOKUP_SHARED opt_vfs.h # HWPMC options HWPMC_HOOKS # XBOX options for FreeBSD/i386, but some files are MI XBOX opt_xbox.h # XFS XFS # snd_emu10kx sound driver options SND_EMU10KX_MULTICHANNEL opt_emu10kx.h Index: head/sys/kern/kern_mutex.c =================================================================== --- head/sys/kern/kern_mutex.c (revision 167053) +++ head/sys/kern/kern_mutex.c (revision 167054) @@ -1,757 +1,768 @@ /*- * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Berkeley Software Design Inc's name may not be used to endorse or * promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ */ /* * Machine independent bits of mutex implementation. */ #include __FBSDID("$FreeBSD$"); #include "opt_adaptive_mutexes.h" #include "opt_ddb.h" #include "opt_global.h" #include "opt_mutex_wake_all.h" #include "opt_sched.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Force MUTEX_WAKE_ALL for now. * single thread wakeup needs fixes to avoid race conditions with * priority inheritance. */ #ifndef MUTEX_WAKE_ALL #define MUTEX_WAKE_ALL #endif /* * Internal utility macros. */ #define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) #define mtx_owner(m) ((struct thread *)((m)->mtx_lock & ~MTX_FLAGMASK)) #ifdef DDB static void db_show_mtx(struct lock_object *lock); #endif /* * Lock classes for sleep and spin mutexes. */ struct lock_class lock_class_mtx_sleep = { "sleep mutex", LC_SLEEPLOCK | LC_RECURSABLE, #ifdef DDB db_show_mtx #endif }; struct lock_class lock_class_mtx_spin = { "spin mutex", LC_SPINLOCK | LC_RECURSABLE, #ifdef DDB db_show_mtx #endif }; /* * System-wide mutexes */ struct mtx sched_lock; struct mtx Giant; #ifdef LOCK_PROFILING static inline void lock_profile_init(void) { int i; /* Initialize the mutex profiling locks */ for (i = 0; i < LPROF_LOCK_SIZE; i++) { mtx_init(&lprof_locks[i], "mprof lock", NULL, MTX_SPIN|MTX_QUIET|MTX_NOPROFILE); } } #else static inline void lock_profile_init(void) {;} #endif /* * Function versions of the inlined __mtx_* macros. These are used by * modules and can also be called from assembly language if needed. */ void _mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) { MPASS(curthread != NULL); KASSERT(m->mtx_lock != MTX_DESTROYED, ("mtx_lock() of destroyed mutex @ %s:%d", file, line)); KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep, ("mtx_lock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, file, line)); WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line); _get_sleep_lock(m, curthread, opts, file, line); LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file, line); WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); curthread->td_locks++; } void _mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) { + struct lock_object lo; + MPASS(curthread != NULL); KASSERT(m->mtx_lock != MTX_DESTROYED, ("mtx_unlock() of destroyed mutex @ %s:%d", file, line)); KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep, ("mtx_unlock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, file, line)); curthread->td_locks--; WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file, line); mtx_assert(m, MA_OWNED); - - lock_profile_release_lock(&m->mtx_object); +#ifdef LOCK_PROFILING + memcpy(&lo, &m->mtx_object, sizeof(lo)); + m->mtx_object.lo_flags &= ~LO_CONTESTED; +#endif _rel_sleep_lock(m, curthread, opts, file, line); + lock_profile_release_lock(&lo); } void _mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) { MPASS(curthread != NULL); KASSERT(m->mtx_lock != MTX_DESTROYED, ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line)); KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin, ("mtx_lock_spin() of sleep mutex %s @ %s:%d", m->mtx_object.lo_name, file, line)); WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line); _get_spin_lock(m, curthread, opts, file, line); LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file, line); WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); } void _mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) { + struct lock_object lo; + MPASS(curthread != NULL); KASSERT(m->mtx_lock != MTX_DESTROYED, ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line)); KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin, ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", m->mtx_object.lo_name, file, line)); WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file, line); mtx_assert(m, MA_OWNED); - lock_profile_release_lock(&m->mtx_object); +#ifdef LOCK_PROFILING + memcpy(&lo, &m->mtx_object, sizeof(lo)); + m->mtx_object.lo_flags &= ~LO_CONTESTED; +#endif _rel_spin_lock(m); + lock_profile_release_lock(&lo); } /* * The important part of mtx_trylock{,_flags}() * Tries to acquire lock `m.' If this function is called on a mutex that * is already owned, it will recursively acquire the lock. */ int _mtx_trylock(struct mtx *m, int opts, const char *file, int line) { int rval, contested = 0; uint64_t waittime = 0; MPASS(curthread != NULL); KASSERT(m->mtx_lock != MTX_DESTROYED, ("mtx_trylock() of destroyed mutex @ %s:%d", file, line)); KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep, ("mtx_trylock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, file, line)); if (mtx_owned(m) && (m->mtx_object.lo_flags & LO_RECURSABLE) != 0) { m->mtx_recurse++; atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); rval = 1; } else rval = _obtain_lock(m, (uintptr_t)curthread); LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line); if (rval) { WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK, file, line); curthread->td_locks++; if (m->mtx_recurse == 0) lock_profile_obtain_lock_success(&m->mtx_object, contested, waittime, file, line); } return (rval); } /* * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. * * We call this if the lock is either contested (i.e. we need to go to * sleep waiting for it), or if we need to recurse on it. */ void _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file, int line) { #if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) volatile struct thread *owner; #endif #ifdef KTR int cont_logged = 0; #endif uintptr_t v; if (mtx_owned(m)) { KASSERT((m->mtx_object.lo_flags & LO_RECURSABLE) != 0, ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n", m->mtx_object.lo_name, file, line)); m->mtx_recurse++; atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); return; } if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR4(KTR_LOCK, "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", m->mtx_object.lo_name, (void *)m->mtx_lock, file, line); while (!_obtain_lock(m, tid)) { turnstile_lock(&m->mtx_object); v = m->mtx_lock; /* * Check if the lock has been released while spinning for * the turnstile chain lock. */ if (v == MTX_UNOWNED) { turnstile_release(&m->mtx_object); cpu_spinwait(); continue; } #ifdef MUTEX_WAKE_ALL MPASS(v != MTX_CONTESTED); #else /* * The mutex was marked contested on release. This means that * there are other threads blocked on it. Grab ownership of * it and propagate its priority to the current thread if * necessary. */ if (v == MTX_CONTESTED) { m->mtx_lock = tid | MTX_CONTESTED; turnstile_claim(&m->mtx_object); break; } #endif /* * If the mutex isn't already contested and a failure occurs * setting the contested bit, the mutex was either released * or the state of the MTX_RECURSED bit changed. */ if ((v & MTX_CONTESTED) == 0 && !atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) { turnstile_release(&m->mtx_object); cpu_spinwait(); continue; } #if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) /* * If the current owner of the lock is executing on another * CPU, spin instead of blocking. */ owner = (struct thread *)(v & ~MTX_FLAGMASK); #ifdef ADAPTIVE_GIANT if (TD_IS_RUNNING(owner)) #else if (m != &Giant && TD_IS_RUNNING(owner)) #endif { turnstile_release(&m->mtx_object); while (mtx_owner(m) == owner && TD_IS_RUNNING(owner)) { cpu_spinwait(); } continue; } #endif /* SMP && !NO_ADAPTIVE_MUTEXES */ /* * We definitely must sleep for this lock. */ mtx_assert(m, MA_NOTOWNED); #ifdef KTR if (!cont_logged) { CTR6(KTR_CONTENTION, "contention: %p at %s:%d wants %s, taken by %s:%d", (void *)tid, file, line, m->mtx_object.lo_name, WITNESS_FILE(&m->mtx_object), WITNESS_LINE(&m->mtx_object)); cont_logged = 1; } #endif /* * Block on the turnstile. */ turnstile_wait(&m->mtx_object, mtx_owner(m), TS_EXCLUSIVE_QUEUE); } #ifdef KTR if (cont_logged) { CTR4(KTR_CONTENTION, "contention end: %s acquired by %p at %s:%d", m->mtx_object.lo_name, (void *)tid, file, line); } #endif return; } #ifdef SMP /* * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. * * This is only called if we need to actually spin for the lock. Recursion * is handled inline. */ void _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file, int line) { int i = 0; struct thread *td; if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); while (!_obtain_lock(m, tid)) { /* Give interrupts a chance while we spin. */ spinlock_exit(); while (m->mtx_lock != MTX_UNOWNED) { if (i++ < 10000000) { cpu_spinwait(); continue; } if (i < 60000000 || kdb_active || panicstr != NULL) DELAY(1); else { td = mtx_owner(m); /* If the mutex is unlocked, try again. */ if (td == NULL) continue; printf( "spin lock %p (%s) held by %p (tid %d) too long\n", m, m->mtx_object.lo_name, td, td->td_tid); #ifdef WITNESS witness_display_spinlock(&m->mtx_object, td); #endif panic("spin lock held too long"); } cpu_spinwait(); } spinlock_enter(); } if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); return; } #endif /* SMP */ /* * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. * * We are only called here if the lock is recursed or contested (i.e. we * need to wake up a blocked thread). */ void _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) { struct turnstile *ts; #ifndef PREEMPTION struct thread *td, *td1; #endif if (mtx_recursed(m)) { if (--(m->mtx_recurse) == 0) atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); return; } turnstile_lock(&m->mtx_object); ts = turnstile_lookup(&m->mtx_object); if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); #if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) if (ts == NULL) { _release_lock_quick(m); if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p no sleepers", m); turnstile_release(&m->mtx_object); return; } #else MPASS(ts != NULL); #endif #ifndef PREEMPTION /* XXX */ td1 = turnstile_head(ts, TS_EXCLUSIVE_QUEUE); #endif #ifdef MUTEX_WAKE_ALL turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); _release_lock_quick(m); #else if (turnstile_signal(ts, TS_EXCLUSIVE_QUEUE)) { _release_lock_quick(m); if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); } else { m->mtx_lock = MTX_CONTESTED; if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p still contested", m); } #endif turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); #ifndef PREEMPTION /* * XXX: This is just a hack until preemption is done. However, * once preemption is done we need to either wrap the * turnstile_signal() and release of the actual lock in an * extra critical section or change the preemption code to * always just set a flag and never do instant-preempts. */ td = curthread; if (td->td_critnest > 0 || td1->td_priority >= td->td_priority) return; mtx_lock_spin(&sched_lock); if (!TD_IS_RUNNING(td1)) { #ifdef notyet if (td->td_ithd != NULL) { struct ithd *it = td->td_ithd; if (it->it_interrupted) { if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p interrupted %p", it, it->it_interrupted); intr_thd_fixup(it); } } #endif if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p switching out lock=%p", m, (void *)m->mtx_lock); mi_switch(SW_INVOL, NULL); if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", m, (void *)m->mtx_lock); } mtx_unlock_spin(&sched_lock); #endif return; } /* * All the unlocking of MTX_SPIN locks is done inline. * See the _rel_spin_lock() macro for the details. */ /* * The backing function for the INVARIANTS-enabled mtx_assert() */ #ifdef INVARIANT_SUPPORT void _mtx_assert(struct mtx *m, int what, const char *file, int line) { if (panicstr != NULL || dumping) return; switch (what) { case MA_OWNED: case MA_OWNED | MA_RECURSED: case MA_OWNED | MA_NOTRECURSED: if (!mtx_owned(m)) panic("mutex %s not owned at %s:%d", m->mtx_object.lo_name, file, line); if (mtx_recursed(m)) { if ((what & MA_NOTRECURSED) != 0) panic("mutex %s recursed at %s:%d", m->mtx_object.lo_name, file, line); } else if ((what & MA_RECURSED) != 0) { panic("mutex %s unrecursed at %s:%d", m->mtx_object.lo_name, file, line); } break; case MA_NOTOWNED: if (mtx_owned(m)) panic("mutex %s owned at %s:%d", m->mtx_object.lo_name, file, line); break; default: panic("unknown mtx_assert at %s:%d", file, line); } } #endif /* * The MUTEX_DEBUG-enabled mtx_validate() * * Most of these checks have been moved off into the LO_INITIALIZED flag * maintained by the witness code. */ #ifdef MUTEX_DEBUG void mtx_validate(struct mtx *); void mtx_validate(struct mtx *m) { /* * XXX: When kernacc() does not require Giant we can reenable this check */ #ifdef notyet /* * Can't call kernacc() from early init386(), especially when * initializing Giant mutex, because some stuff in kernacc() * requires Giant itself. */ if (!cold) if (!kernacc((caddr_t)m, sizeof(m), VM_PROT_READ | VM_PROT_WRITE)) panic("Can't read and write to mutex %p", m); #endif } #endif /* * General init routine used by the MTX_SYSINIT() macro. */ void mtx_sysinit(void *arg) { struct mtx_args *margs = arg; mtx_init(margs->ma_mtx, margs->ma_desc, NULL, margs->ma_opts); } /* * Mutex initialization routine; initialize lock `m' of type contained in * `opts' with options contained in `opts' and name `name.' The optional * lock type `type' is used as a general lock category name for use with * witness. */ void mtx_init(struct mtx *m, const char *name, const char *type, int opts) { struct lock_class *class; int flags; MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE)) == 0); #ifdef MUTEX_DEBUG /* Diagnostic and error correction */ mtx_validate(m); #endif /* Determine lock class and lock flags. */ if (opts & MTX_SPIN) class = &lock_class_mtx_spin; else class = &lock_class_mtx_sleep; flags = 0; if (opts & MTX_QUIET) flags |= LO_QUIET; if (opts & MTX_RECURSE) flags |= LO_RECURSABLE; if ((opts & MTX_NOWITNESS) == 0) flags |= LO_WITNESS; if (opts & MTX_DUPOK) flags |= LO_DUPOK; if (opts & MTX_NOPROFILE) flags |= LO_NOPROFILE; /* Initialize mutex. */ m->mtx_lock = MTX_UNOWNED; m->mtx_recurse = 0; lock_profile_object_init(&m->mtx_object, class, name); lock_init(&m->mtx_object, class, name, type, flags); } /* * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be * passed in as a flag here because if the corresponding mtx_init() was * called with MTX_QUIET set, then it will already be set in the mutex's * flags. */ void mtx_destroy(struct mtx *m) { if (!mtx_owned(m)) MPASS(mtx_unowned(m)); else { MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); /* Perform the non-mtx related part of mtx_unlock_spin(). */ if (LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin) spinlock_exit(); else curthread->td_locks--; /* Tell witness this isn't locked to make it happy. */ WITNESS_UNLOCK(&m->mtx_object, LOP_EXCLUSIVE, __FILE__, __LINE__); } m->mtx_lock = MTX_DESTROYED; lock_profile_object_destroy(&m->mtx_object); lock_destroy(&m->mtx_object); } /* * Intialize the mutex code and system mutexes. This is called from the MD * startup code prior to mi_startup(). The per-CPU data space needs to be * setup before this is called. */ void mutex_init(void) { /* Setup turnstiles so that sleep mutexes work. */ init_turnstiles(); /* * Initialize mutexes. */ mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE); mtx_init(&sched_lock, "sched lock", NULL, MTX_SPIN | MTX_RECURSE); mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK); mtx_init(&devmtx, "cdev", NULL, MTX_DEF); mtx_lock(&Giant); lock_profile_init(); } #ifdef DDB void db_show_mtx(struct lock_object *lock) { struct thread *td; struct mtx *m; m = (struct mtx *)lock; db_printf(" flags: {"); if (LOCK_CLASS(lock) == &lock_class_mtx_spin) db_printf("SPIN"); else db_printf("DEF"); if (m->mtx_object.lo_flags & LO_RECURSABLE) db_printf(", RECURSE"); if (m->mtx_object.lo_flags & LO_DUPOK) db_printf(", DUPOK"); db_printf("}\n"); db_printf(" state: {"); if (mtx_unowned(m)) db_printf("UNOWNED"); else { db_printf("OWNED"); if (m->mtx_lock & MTX_CONTESTED) db_printf(", CONTESTED"); if (m->mtx_lock & MTX_RECURSED) db_printf(", RECURSED"); } db_printf("}\n"); if (!mtx_unowned(m)) { td = mtx_owner(m); db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td, td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm); if (mtx_recursed(m)) db_printf(" recursed: %d\n", m->mtx_recurse); } } #endif Index: head/sys/kern/kern_rwlock.c =================================================================== --- head/sys/kern/kern_rwlock.c (revision 167053) +++ head/sys/kern/kern_rwlock.c (revision 167054) @@ -1,837 +1,837 @@ /*- * Copyright (c) 2006 John Baldwin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the author nor the names of any co-contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Machine independent bits of reader/writer lock implementation. */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #ifdef DDB #include static void db_show_rwlock(struct lock_object *lock); #endif struct lock_class lock_class_rw = { "rw", LC_SLEEPLOCK | LC_RECURSABLE | LC_UPGRADABLE, #ifdef DDB db_show_rwlock #endif }; /* * Return a pointer to the owning thread if the lock is write-locked or * NULL if the lock is unlocked or read-locked. */ #define rw_wowner(rw) \ ((rw)->rw_lock & RW_LOCK_READ ? NULL : \ (struct thread *)RW_OWNER((rw)->rw_lock)) /* * Return a pointer to the owning thread for this lock who should receive * any priority lent by threads that block on this lock. Currently this * is identical to rw_wowner(). */ #define rw_owner(rw) rw_wowner(rw) #ifndef INVARIANTS #define _rw_assert(rw, what, file, line) #endif void rw_init(struct rwlock *rw, const char *name) { rw->rw_lock = RW_UNLOCKED; lock_profile_object_init(&rw->rw_object, &lock_class_rw, name); lock_init(&rw->rw_object, &lock_class_rw, name, NULL, LO_WITNESS | LO_RECURSABLE | LO_UPGRADABLE); } void rw_destroy(struct rwlock *rw) { KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock not unlocked")); lock_profile_object_destroy(&rw->rw_object); lock_destroy(&rw->rw_object); } void rw_sysinit(void *arg) { struct rw_args *args = arg; rw_init(args->ra_rw, args->ra_desc); } int rw_wowned(struct rwlock *rw) { return (rw_wowner(rw) == curthread); } void _rw_wlock(struct rwlock *rw, const char *file, int line) { MPASS(curthread != NULL); KASSERT(rw_wowner(rw) != curthread, ("%s (%s): wlock already held @ %s:%d", __func__, rw->rw_object.lo_name, file, line)); WITNESS_CHECKORDER(&rw->rw_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, line); __rw_wlock(rw, curthread, file, line); LOCK_LOG_LOCK("WLOCK", &rw->rw_object, 0, 0, file, line); WITNESS_LOCK(&rw->rw_object, LOP_EXCLUSIVE, file, line); curthread->td_locks++; } void _rw_wunlock(struct rwlock *rw, const char *file, int line) { MPASS(curthread != NULL); _rw_assert(rw, RA_WLOCKED, file, line); curthread->td_locks--; WITNESS_UNLOCK(&rw->rw_object, LOP_EXCLUSIVE, file, line); LOCK_LOG_LOCK("WUNLOCK", &rw->rw_object, 0, 0, file, line); lock_profile_release_lock(&rw->rw_object); __rw_wunlock(rw, curthread, file, line); } void _rw_rlock(struct rwlock *rw, const char *file, int line) { #ifdef SMP volatile struct thread *owner; #endif - uint64_t waitstart; - int contested; + uint64_t waitstart = 0; + int contested = 0; uintptr_t x; KASSERT(rw_wowner(rw) != curthread, ("%s (%s): wlock already held @ %s:%d", __func__, rw->rw_object.lo_name, file, line)); WITNESS_CHECKORDER(&rw->rw_object, LOP_NEWORDER, file, line); /* * Note that we don't make any attempt to try to block read * locks once a writer has blocked on the lock. The reason is * that we currently allow for read locks to recurse and we * don't keep track of all the holders of read locks. Thus, if * we were to block readers once a writer blocked and a reader * tried to recurse on their reader lock after a writer had * blocked we would end up in a deadlock since the reader would * be blocked on the writer, and the writer would be blocked * waiting for the reader to release its original read lock. */ for (;;) { /* * Handle the easy case. If no other thread has a write * lock, then try to bump up the count of read locks. Note * that we have to preserve the current state of the * RW_LOCK_WRITE_WAITERS flag. If we fail to acquire a * read lock, then rw_lock must have changed, so restart * the loop. Note that this handles the case of a * completely unlocked rwlock since such a lock is encoded * as a read lock with no waiters. */ x = rw->rw_lock; if (x & RW_LOCK_READ) { /* * The RW_LOCK_READ_WAITERS flag should only be set * if another thread currently holds a write lock, * and in that case RW_LOCK_READ should be clear. */ MPASS((x & RW_LOCK_READ_WAITERS) == 0); if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) { lock_profile_obtain_lock_success(&rw->rw_object, contested, waitstart, file, line); if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR4(KTR_LOCK, "%s: %p succeed %p -> %p", __func__, rw, (void *)x, (void *)(x + RW_ONE_READER)); break; } lock_profile_obtain_lock_failed(&rw->rw_object, &contested, &waitstart); cpu_spinwait(); continue; } /* * Okay, now it's the hard case. Some other thread already * has a write lock, so acquire the turnstile lock so we can * begin the process of blocking. */ turnstile_lock(&rw->rw_object); /* * The lock might have been released while we spun, so * recheck its state and restart the loop if there is no * longer a write lock. */ x = rw->rw_lock; if (x & RW_LOCK_READ) { turnstile_release(&rw->rw_object); cpu_spinwait(); continue; } /* * Ok, it's still a write lock. If the RW_LOCK_READ_WAITERS * flag is already set, then we can go ahead and block. If * it is not set then try to set it. If we fail to set it * drop the turnstile lock and restart the loop. */ if (!(x & RW_LOCK_READ_WAITERS)) { if (!atomic_cmpset_ptr(&rw->rw_lock, x, x | RW_LOCK_READ_WAITERS)) { turnstile_release(&rw->rw_object); cpu_spinwait(); continue; } if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR2(KTR_LOCK, "%s: %p set read waiters flag", __func__, rw); } #ifdef SMP /* * If the owner is running on another CPU, spin until * the owner stops running or the state of the lock * changes. */ owner = (struct thread *)RW_OWNER(x); if (TD_IS_RUNNING(owner)) { lock_profile_obtain_lock_failed(&rw->rw_object, &contested, &waitstart); turnstile_release(&rw->rw_object); if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR3(KTR_LOCK, "%s: spinning on %p held by %p", __func__, rw, owner); while ((struct thread*)RW_OWNER(rw->rw_lock)== owner && TD_IS_RUNNING(owner)) cpu_spinwait(); continue; } #endif /* * We were unable to acquire the lock and the read waiters * flag is set, so we must block on the turnstile. */ if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, rw); turnstile_wait(&rw->rw_object, rw_owner(rw), TS_SHARED_QUEUE); if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR2(KTR_LOCK, "%s: %p resuming from turnstile", __func__, rw); } /* * TODO: acquire "owner of record" here. Here be turnstile dragons * however. turnstiles don't like owners changing between calls to * turnstile_wait() currently. */ LOCK_LOG_LOCK("RLOCK", &rw->rw_object, 0, 0, file, line); WITNESS_LOCK(&rw->rw_object, 0, file, line); curthread->td_locks++; } void _rw_runlock(struct rwlock *rw, const char *file, int line) { struct turnstile *ts; uintptr_t x; _rw_assert(rw, RA_RLOCKED, file, line); curthread->td_locks--; WITNESS_UNLOCK(&rw->rw_object, 0, file, line); LOCK_LOG_LOCK("RUNLOCK", &rw->rw_object, 0, 0, file, line); /* TODO: drop "owner of record" here. */ for (;;) { /* * See if there is more than one read lock held. If so, * just drop one and return. */ x = rw->rw_lock; if (RW_READERS(x) > 1) { if (atomic_cmpset_ptr(&rw->rw_lock, x, x - RW_ONE_READER)) { if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR4(KTR_LOCK, "%s: %p succeeded %p -> %p", __func__, rw, (void *)x, (void *)(x - RW_ONE_READER)); break; } continue; } else lock_profile_release_lock(&rw->rw_object); /* * We should never have read waiters while at least one * thread holds a read lock. (See note above) */ KASSERT(!(x & RW_LOCK_READ_WAITERS), ("%s: waiting readers", __func__)); /* * If there aren't any waiters for a write lock, then try * to drop it quickly. */ if (!(x & RW_LOCK_WRITE_WAITERS)) { /* * There shouldn't be any flags set and we should * be the only read lock. If we fail to release * the single read lock, then another thread might * have just acquired a read lock, so go back up * to the multiple read locks case. */ MPASS(x == RW_READERS_LOCK(1)); if (atomic_cmpset_ptr(&rw->rw_lock, RW_READERS_LOCK(1), RW_UNLOCKED)) { if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR2(KTR_LOCK, "%s: %p last succeeded", __func__, rw); break; } continue; } /* * There should just be one reader with one or more * writers waiting. */ MPASS(x == (RW_READERS_LOCK(1) | RW_LOCK_WRITE_WAITERS)); /* * Ok, we know we have a waiting writer and we think we * are the last reader, so grab the turnstile lock. */ turnstile_lock(&rw->rw_object); /* * Try to drop our lock leaving the lock in a unlocked * state. * * If you wanted to do explicit lock handoff you'd have to * do it here. You'd also want to use turnstile_signal() * and you'd have to handle the race where a higher * priority thread blocks on the write lock before the * thread you wakeup actually runs and have the new thread * "steal" the lock. For now it's a lot simpler to just * wakeup all of the waiters. * * As above, if we fail, then another thread might have * acquired a read lock, so drop the turnstile lock and * restart. */ if (!atomic_cmpset_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | RW_LOCK_WRITE_WAITERS, RW_UNLOCKED)) { turnstile_release(&rw->rw_object); continue; } if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR2(KTR_LOCK, "%s: %p last succeeded with waiters", __func__, rw); /* * Ok. The lock is released and all that's left is to * wake up the waiters. Note that the lock might not be * free anymore, but in that case the writers will just * block again if they run before the new lock holder(s) * release the lock. */ ts = turnstile_lookup(&rw->rw_object); MPASS(ts != NULL); turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); turnstile_unpend(ts, TS_SHARED_LOCK); break; } } /* * This function is called when we are unable to obtain a write lock on the * first try. This means that at least one other thread holds either a * read or write lock. */ void _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) { #ifdef SMP volatile struct thread *owner; #endif uintptr_t v; if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, rw->rw_object.lo_name, (void *)rw->rw_lock, file, line); while (!_rw_write_lock(rw, tid)) { turnstile_lock(&rw->rw_object); v = rw->rw_lock; /* * If the lock was released while spinning on the * turnstile chain lock, try again. */ if (v == RW_UNLOCKED) { turnstile_release(&rw->rw_object); cpu_spinwait(); continue; } /* * If the lock was released by a writer with both readers * and writers waiting and a reader hasn't woken up and * acquired the lock yet, rw_lock will be set to the * value RW_UNLOCKED | RW_LOCK_WRITE_WAITERS. If we see * that value, try to acquire it once. Note that we have * to preserve the RW_LOCK_WRITE_WAITERS flag as there are * other writers waiting still. If we fail, restart the * loop. */ if (v == (RW_UNLOCKED | RW_LOCK_WRITE_WAITERS)) { if (atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED | RW_LOCK_WRITE_WAITERS, tid | RW_LOCK_WRITE_WAITERS)) { turnstile_claim(&rw->rw_object); CTR2(KTR_LOCK, "%s: %p claimed by new writer", __func__, rw); break; } turnstile_release(&rw->rw_object); cpu_spinwait(); continue; } /* * If the RW_LOCK_WRITE_WAITERS flag isn't set, then try to * set it. If we fail to set it, then loop back and try * again. */ if (!(v & RW_LOCK_WRITE_WAITERS)) { if (!atomic_cmpset_ptr(&rw->rw_lock, v, v | RW_LOCK_WRITE_WAITERS)) { turnstile_release(&rw->rw_object); cpu_spinwait(); continue; } if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR2(KTR_LOCK, "%s: %p set write waiters flag", __func__, rw); } #ifdef SMP /* * If the lock is write locked and the owner is * running on another CPU, spin until the owner stops * running or the state of the lock changes. */ owner = (struct thread *)RW_OWNER(v); if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) { turnstile_release(&rw->rw_object); if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR3(KTR_LOCK, "%s: spinning on %p held by %p", __func__, rw, owner); while ((struct thread*)RW_OWNER(rw->rw_lock)== owner && TD_IS_RUNNING(owner)) cpu_spinwait(); continue; } #endif /* * We were unable to acquire the lock and the write waiters * flag is set, so we must block on the turnstile. */ if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, rw); turnstile_wait(&rw->rw_object, rw_owner(rw), TS_EXCLUSIVE_QUEUE); if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR2(KTR_LOCK, "%s: %p resuming from turnstile", __func__, rw); } } /* * This function is called if the first try at releasing a write lock failed. * This means that one of the 2 waiter bits must be set indicating that at * least one thread is waiting on this lock. */ void _rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) { struct turnstile *ts; uintptr_t v; int queue; KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), ("%s: neither of the waiter flags are set", __func__)); if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR2(KTR_LOCK, "%s: %p contested", __func__, rw); turnstile_lock(&rw->rw_object); ts = turnstile_lookup(&rw->rw_object); #ifdef SMP /* * There might not be a turnstile for this lock if all of * the waiters are adaptively spinning. In that case, just * reset the lock to the unlocked state and return. */ if (ts == NULL) { atomic_store_rel_ptr(&rw->rw_lock, RW_UNLOCKED); if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR2(KTR_LOCK, "%s: %p no sleepers", __func__, rw); turnstile_release(&rw->rw_object); return; } #else MPASS(ts != NULL); #endif /* * Use the same algo as sx locks for now. Prefer waking up shared * waiters if we have any over writers. This is probably not ideal. * * 'v' is the value we are going to write back to rw_lock. If we * have waiters on both queues, we need to preserve the state of * the waiter flag for the queue we don't wake up. For now this is * hardcoded for the algorithm mentioned above. * * In the case of both readers and writers waiting we wakeup the * readers but leave the RW_LOCK_WRITE_WAITERS flag set. If a * new writer comes in before a reader it will claim the lock up * above. There is probably a potential priority inversion in * there that could be worked around either by waking both queues * of waiters or doing some complicated lock handoff gymnastics. * * Note that in the SMP case, if both flags are set, there might * not be any actual writers on the turnstile as they might all * be spinning. In that case, we don't want to preserve the * RW_LOCK_WRITE_WAITERS flag as the turnstile is going to go * away once we wakeup all the readers. */ v = RW_UNLOCKED; if (rw->rw_lock & RW_LOCK_READ_WAITERS) { queue = TS_SHARED_QUEUE; #ifdef SMP if (rw->rw_lock & RW_LOCK_WRITE_WAITERS && !turnstile_empty(ts, TS_EXCLUSIVE_QUEUE)) v |= RW_LOCK_WRITE_WAITERS; #else v |= (rw->rw_lock & RW_LOCK_WRITE_WAITERS); #endif } else queue = TS_EXCLUSIVE_QUEUE; #ifdef SMP /* * We have to make sure that we actually have waiters to * wakeup. If they are all spinning, then we just need to * disown the turnstile and return. */ if (turnstile_empty(ts, queue)) { if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR2(KTR_LOCK, "%s: %p no sleepers 2", __func__, rw); atomic_store_rel_ptr(&rw->rw_lock, v); turnstile_disown(ts); return; } #endif /* Wake up all waiters for the specific queue. */ if (LOCK_LOG_TEST(&rw->rw_object, 0)) CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw, queue == TS_SHARED_QUEUE ? "read" : "write"); turnstile_broadcast(ts, queue); atomic_store_rel_ptr(&rw->rw_lock, v); turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); } /* * Attempt to do a non-blocking upgrade from a read lock to a write * lock. This will only succeed if this thread holds a single read * lock. Returns true if the upgrade succeeded and false otherwise. */ int _rw_try_upgrade(struct rwlock *rw, const char *file, int line) { uintptr_t v, tid; int success; _rw_assert(rw, RA_RLOCKED, file, line); /* * Attempt to switch from one reader to a writer. If there * are any write waiters, then we will have to lock the * turnstile first to prevent races with another writer * calling turnstile_wait() before we have claimed this * turnstile. So, do the simple case of no waiters first. */ tid = (uintptr_t)curthread; if (!(rw->rw_lock & RW_LOCK_WRITE_WAITERS)) { success = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_READERS_LOCK(1), tid); goto out; } /* * Ok, we think we have write waiters, so lock the * turnstile. */ turnstile_lock(&rw->rw_object); /* * Try to switch from one reader to a writer again. This time * we honor the current state of the RW_LOCK_WRITE_WAITERS * flag. If we obtain the lock with the flag set, then claim * ownership of the turnstile. In the SMP case it is possible * for there to not be an associated turnstile even though there * are waiters if all of the waiters are spinning. */ v = rw->rw_lock & RW_LOCK_WRITE_WAITERS; success = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v, tid | v); #ifdef SMP if (success && v && turnstile_lookup(&rw->rw_object) != NULL) #else if (success && v) #endif turnstile_claim(&rw->rw_object); else turnstile_release(&rw->rw_object); out: LOCK_LOG_TRY("WUPGRADE", &rw->rw_object, 0, success, file, line); if (success) WITNESS_UPGRADE(&rw->rw_object, LOP_EXCLUSIVE | LOP_TRYLOCK, file, line); return (success); } /* * Downgrade a write lock into a single read lock. */ void _rw_downgrade(struct rwlock *rw, const char *file, int line) { struct turnstile *ts; uintptr_t tid, v; _rw_assert(rw, RA_WLOCKED, file, line); WITNESS_DOWNGRADE(&rw->rw_object, 0, file, line); /* * Convert from a writer to a single reader. First we handle * the easy case with no waiters. If there are any waiters, we * lock the turnstile, "disown" the lock, and awaken any read * waiters. */ tid = (uintptr_t)curthread; if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1))) goto out; /* * Ok, we think we have waiters, so lock the turnstile so we can * read the waiter flags without any races. */ turnstile_lock(&rw->rw_object); v = rw->rw_lock; MPASS(v & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)); /* * Downgrade from a write lock while preserving * RW_LOCK_WRITE_WAITERS and give up ownership of the * turnstile. If there are any read waiters, wake them up. * * For SMP, we have to allow for the fact that all of the * read waiters might be spinning. In that case, act as if * RW_LOCK_READ_WAITERS is not set. Also, only preserve * the RW_LOCK_WRITE_WAITERS flag if at least one writer is * blocked on the turnstile. */ ts = turnstile_lookup(&rw->rw_object); #ifdef SMP if (ts == NULL) v &= ~(RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS); else if (v & RW_LOCK_READ_WAITERS && turnstile_empty(ts, TS_SHARED_QUEUE)) v &= ~RW_LOCK_READ_WAITERS; else if (v & RW_LOCK_WRITE_WAITERS && turnstile_empty(ts, TS_EXCLUSIVE_QUEUE)) v &= ~RW_LOCK_WRITE_WAITERS; #else MPASS(ts != NULL); #endif if (v & RW_LOCK_READ_WAITERS) turnstile_broadcast(ts, TS_SHARED_QUEUE); atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | (v & RW_LOCK_WRITE_WAITERS)); if (v & RW_LOCK_READ_WAITERS) turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); #ifdef SMP else if (ts == NULL) turnstile_release(&rw->rw_object); #endif else turnstile_disown(ts); out: LOCK_LOG_LOCK("WDOWNGRADE", &rw->rw_object, 0, 0, file, line); } #ifdef INVARIANT_SUPPORT #ifndef INVARIANTS #undef _rw_assert #endif /* * In the non-WITNESS case, rw_assert() can only detect that at least * *some* thread owns an rlock, but it cannot guarantee that *this* * thread owns an rlock. */ void _rw_assert(struct rwlock *rw, int what, const char *file, int line) { if (panicstr != NULL) return; switch (what) { case RA_LOCKED: case RA_RLOCKED: #ifdef WITNESS witness_assert(&rw->rw_object, what, file, line); #else /* * If some other thread has a write lock or we have one * and are asserting a read lock, fail. Also, if no one * has a lock at all, fail. */ if (rw->rw_lock == RW_UNLOCKED || (!(rw->rw_lock & RW_LOCK_READ) && (what == RA_RLOCKED || rw_wowner(rw) != curthread))) panic("Lock %s not %slocked @ %s:%d\n", rw->rw_object.lo_name, (what == RA_RLOCKED) ? "read " : "", file, line); #endif break; case RA_WLOCKED: if (rw_wowner(rw) != curthread) panic("Lock %s not exclusively locked @ %s:%d\n", rw->rw_object.lo_name, file, line); break; case RA_UNLOCKED: #ifdef WITNESS witness_assert(&rw->rw_object, what, file, line); #else /* * If we hold a write lock fail. We can't reliably check * to see if we hold a read lock or not. */ if (rw_wowner(rw) == curthread) panic("Lock %s exclusively locked @ %s:%d\n", rw->rw_object.lo_name, file, line); #endif break; default: panic("Unknown rw lock assertion: %d @ %s:%d", what, file, line); } } #endif /* INVARIANT_SUPPORT */ #ifdef DDB void db_show_rwlock(struct lock_object *lock) { struct rwlock *rw; struct thread *td; rw = (struct rwlock *)lock; db_printf(" state: "); if (rw->rw_lock == RW_UNLOCKED) db_printf("UNLOCKED\n"); else if (rw->rw_lock & RW_LOCK_READ) db_printf("RLOCK: %jd locks\n", (intmax_t)(RW_READERS(rw->rw_lock))); else { td = rw_wowner(rw); db_printf("WLOCK: %p (tid %d, pid %d, \"%s\")\n", td, td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm); } db_printf(" waiters: "); switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) { case RW_LOCK_READ_WAITERS: db_printf("readers\n"); break; case RW_LOCK_WRITE_WAITERS: db_printf("writers\n"); break; case RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS: db_printf("readers and waiters\n"); break; default: db_printf("none\n"); break; } } #endif Index: head/sys/kern/kern_sx.c =================================================================== --- head/sys/kern/kern_sx.c (revision 167053) +++ head/sys/kern/kern_sx.c (revision 167054) @@ -1,467 +1,482 @@ /*- * Copyright (C) 2001 Jason Evans . All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice(s), this list of conditions and the following disclaimer as * the first lines of this file unmodified other than the possible * addition of one or more copyright notices. * 2. Redistributions in binary form must reproduce the above copyright * notice(s), this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH * DAMAGE. */ /* * Shared/exclusive locks. This implementation assures deterministic lock * granting behavior, so that slocks and xlocks are interleaved. * * Priority propagation will not generally raise the priority of lock holders, * so should not be relied upon in combination with sx locks. */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #ifdef DDB #include static void db_show_sx(struct lock_object *lock); #endif struct lock_class lock_class_sx = { "sx", LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE, #ifdef DDB db_show_sx #endif }; #ifndef INVARIANTS #define _sx_assert(sx, what, file, line) #endif void sx_sysinit(void *arg) { struct sx_args *sargs = arg; sx_init(sargs->sa_sx, sargs->sa_desc); } void sx_init(struct sx *sx, const char *description) { sx->sx_lock = mtx_pool_find(mtxpool_lockbuilder, sx); sx->sx_cnt = 0; cv_init(&sx->sx_shrd_cv, description); sx->sx_shrd_wcnt = 0; cv_init(&sx->sx_excl_cv, description); sx->sx_excl_wcnt = 0; sx->sx_xholder = NULL; lock_profile_object_init(&sx->sx_object, &lock_class_sx, description); lock_init(&sx->sx_object, &lock_class_sx, description, NULL, LO_WITNESS | LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE); } void sx_destroy(struct sx *sx) { KASSERT((sx->sx_cnt == 0 && sx->sx_shrd_wcnt == 0 && sx->sx_excl_wcnt == 0), ("%s (%s): holders or waiters\n", __func__, sx->sx_object.lo_name)); sx->sx_lock = NULL; cv_destroy(&sx->sx_shrd_cv); cv_destroy(&sx->sx_excl_cv); lock_profile_object_destroy(&sx->sx_object); lock_destroy(&sx->sx_object); } void _sx_slock(struct sx *sx, const char *file, int line) { uint64_t waittime = 0; int contested = 0; mtx_lock(sx->sx_lock); KASSERT(sx->sx_xholder != curthread, ("%s (%s): slock while xlock is held @ %s:%d\n", __func__, sx->sx_object.lo_name, file, line)); WITNESS_CHECKORDER(&sx->sx_object, LOP_NEWORDER, file, line); /* * Loop in case we lose the race for lock acquisition. */ while (sx->sx_cnt < 0) { sx->sx_shrd_wcnt++; lock_profile_obtain_lock_failed(&sx->sx_object, &contested, &waittime); cv_wait(&sx->sx_shrd_cv, sx->sx_lock); sx->sx_shrd_wcnt--; } /* Acquire a shared lock. */ sx->sx_cnt++; if (sx->sx_cnt == 1) lock_profile_obtain_lock_success(&sx->sx_object, contested, waittime, file, line); LOCK_LOG_LOCK("SLOCK", &sx->sx_object, 0, 0, file, line); WITNESS_LOCK(&sx->sx_object, 0, file, line); curthread->td_locks++; mtx_unlock(sx->sx_lock); } int _sx_try_slock(struct sx *sx, const char *file, int line) { mtx_lock(sx->sx_lock); if (sx->sx_cnt >= 0) { sx->sx_cnt++; LOCK_LOG_TRY("SLOCK", &sx->sx_object, 0, 1, file, line); WITNESS_LOCK(&sx->sx_object, LOP_TRYLOCK, file, line); curthread->td_locks++; mtx_unlock(sx->sx_lock); return (1); } else { LOCK_LOG_TRY("SLOCK", &sx->sx_object, 0, 0, file, line); mtx_unlock(sx->sx_lock); return (0); } } void _sx_xlock(struct sx *sx, const char *file, int line) { int contested = 0; uint64_t waittime = 0; mtx_lock(sx->sx_lock); /* * With sx locks, we're absolutely not permitted to recurse on * xlocks, as it is fatal (deadlock). Normally, recursion is handled * by WITNESS, but as it is not semantically correct to hold the * xlock while in here, we consider it API abuse and put it under * INVARIANTS. */ KASSERT(sx->sx_xholder != curthread, ("%s (%s): xlock already held @ %s:%d", __func__, sx->sx_object.lo_name, file, line)); WITNESS_CHECKORDER(&sx->sx_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, line); /* Loop in case we lose the race for lock acquisition. */ while (sx->sx_cnt != 0) { sx->sx_excl_wcnt++; lock_profile_obtain_lock_failed(&sx->sx_object, &contested, &waittime); cv_wait(&sx->sx_excl_cv, sx->sx_lock); sx->sx_excl_wcnt--; } MPASS(sx->sx_cnt == 0); /* Acquire an exclusive lock. */ sx->sx_cnt--; sx->sx_xholder = curthread; lock_profile_obtain_lock_success(&sx->sx_object, contested, waittime, file, line); LOCK_LOG_LOCK("XLOCK", &sx->sx_object, 0, 0, file, line); WITNESS_LOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line); curthread->td_locks++; mtx_unlock(sx->sx_lock); } int _sx_try_xlock(struct sx *sx, const char *file, int line) { mtx_lock(sx->sx_lock); if (sx->sx_cnt == 0) { sx->sx_cnt--; sx->sx_xholder = curthread; LOCK_LOG_TRY("XLOCK", &sx->sx_object, 0, 1, file, line); WITNESS_LOCK(&sx->sx_object, LOP_EXCLUSIVE | LOP_TRYLOCK, file, line); curthread->td_locks++; mtx_unlock(sx->sx_lock); return (1); } else { LOCK_LOG_TRY("XLOCK", &sx->sx_object, 0, 0, file, line); mtx_unlock(sx->sx_lock); return (0); } } void _sx_sunlock(struct sx *sx, const char *file, int line) { - + struct lock_object lo; + int count = -1; + _sx_assert(sx, SX_SLOCKED, file, line); mtx_lock(sx->sx_lock); curthread->td_locks--; WITNESS_UNLOCK(&sx->sx_object, 0, file, line); /* Release. */ sx->sx_cnt--; - if (sx->sx_cnt == 0) - lock_profile_release_lock(&sx->sx_object); +#ifdef LOCK_PROFILING + if (sx->sx_cnt == 0) { + memcpy(&lo, &sx->sx_object, sizeof(lo)); + sx->sx_object.lo_flags &= ~LO_CONTESTED; + count = 0; + } +#endif /* * If we just released the last shared lock, wake any waiters up, giving * exclusive lockers precedence. In order to make sure that exclusive * lockers won't be blocked forever, don't wake shared lock waiters if * there are exclusive lock waiters. */ if (sx->sx_excl_wcnt > 0) { if (sx->sx_cnt == 0) cv_signal(&sx->sx_excl_cv); } else if (sx->sx_shrd_wcnt > 0) cv_broadcast(&sx->sx_shrd_cv); LOCK_LOG_LOCK("SUNLOCK", &sx->sx_object, 0, 0, file, line); mtx_unlock(sx->sx_lock); + if (count == 0) + lock_profile_release_lock(&lo); + } void _sx_xunlock(struct sx *sx, const char *file, int line) { - + struct lock_object lo; + _sx_assert(sx, SX_XLOCKED, file, line); mtx_lock(sx->sx_lock); MPASS(sx->sx_cnt == -1); curthread->td_locks--; WITNESS_UNLOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line); /* Release. */ sx->sx_cnt++; sx->sx_xholder = NULL; - lock_profile_release_lock(&sx->sx_object); +#ifdef LOCK_PROFILING + memcpy(&lo, &sx->sx_object, sizeof(lo)); + sx->sx_object.lo_flags &= ~LO_CONTESTED; +#endif /* * Wake up waiters if there are any. Give precedence to slock waiters. */ if (sx->sx_shrd_wcnt > 0) cv_broadcast(&sx->sx_shrd_cv); else if (sx->sx_excl_wcnt > 0) cv_signal(&sx->sx_excl_cv); LOCK_LOG_LOCK("XUNLOCK", &sx->sx_object, 0, 0, file, line); mtx_unlock(sx->sx_lock); + lock_profile_release_lock(&lo); } int _sx_try_upgrade(struct sx *sx, const char *file, int line) { _sx_assert(sx, SX_SLOCKED, file, line); mtx_lock(sx->sx_lock); if (sx->sx_cnt == 1) { sx->sx_cnt = -1; sx->sx_xholder = curthread; LOCK_LOG_TRY("XUPGRADE", &sx->sx_object, 0, 1, file, line); WITNESS_UPGRADE(&sx->sx_object, LOP_EXCLUSIVE | LOP_TRYLOCK, file, line); mtx_unlock(sx->sx_lock); return (1); } else { LOCK_LOG_TRY("XUPGRADE", &sx->sx_object, 0, 0, file, line); mtx_unlock(sx->sx_lock); return (0); } } void _sx_downgrade(struct sx *sx, const char *file, int line) { _sx_assert(sx, SX_XLOCKED, file, line); mtx_lock(sx->sx_lock); MPASS(sx->sx_cnt == -1); WITNESS_DOWNGRADE(&sx->sx_object, 0, file, line); sx->sx_cnt = 1; sx->sx_xholder = NULL; if (sx->sx_shrd_wcnt > 0) cv_broadcast(&sx->sx_shrd_cv); LOCK_LOG_LOCK("XDOWNGRADE", &sx->sx_object, 0, 0, file, line); mtx_unlock(sx->sx_lock); } #ifdef INVARIANT_SUPPORT #ifndef INVARIANTS #undef _sx_assert #endif /* * In the non-WITNESS case, sx_assert() can only detect that at least * *some* thread owns an slock, but it cannot guarantee that *this* * thread owns an slock. */ void _sx_assert(struct sx *sx, int what, const char *file, int line) { if (panicstr != NULL) return; switch (what) { case SX_LOCKED: case SX_SLOCKED: #ifdef WITNESS witness_assert(&sx->sx_object, what, file, line); #else mtx_lock(sx->sx_lock); if (sx->sx_cnt <= 0 && (what == SX_SLOCKED || sx->sx_xholder != curthread)) panic("Lock %s not %slocked @ %s:%d\n", sx->sx_object.lo_name, (what == SX_SLOCKED) ? "share " : "", file, line); mtx_unlock(sx->sx_lock); #endif break; case SX_XLOCKED: mtx_lock(sx->sx_lock); if (sx->sx_xholder != curthread) panic("Lock %s not exclusively locked @ %s:%d\n", sx->sx_object.lo_name, file, line); mtx_unlock(sx->sx_lock); break; case SX_UNLOCKED: #ifdef WITNESS witness_assert(&sx->sx_object, what, file, line); #else /* * We are able to check only exclusive lock here, * we cannot assert that *this* thread owns slock. */ mtx_lock(sx->sx_lock); if (sx->sx_xholder == curthread) panic("Lock %s exclusively locked @ %s:%d\n", sx->sx_object.lo_name, file, line); mtx_unlock(sx->sx_lock); #endif break; default: panic("Unknown sx lock assertion: %d @ %s:%d", what, file, line); } } #endif /* INVARIANT_SUPPORT */ #ifdef DDB void db_show_sx(struct lock_object *lock) { struct thread *td; struct sx *sx; sx = (struct sx *)lock; db_printf(" state: "); if (sx->sx_cnt < 0) { td = sx->sx_xholder; db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td, td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm); } else if (sx->sx_cnt > 0) db_printf("SLOCK: %d locks\n", sx->sx_cnt); else db_printf("UNLOCKED\n"); db_printf(" waiters: %d shared, %d exclusive\n", sx->sx_shrd_wcnt, sx->sx_excl_wcnt); } /* * Check to see if a thread that is blocked on a sleep queue is actually * blocked on an sx lock. If so, output some details and return true. * If the lock has an exclusive owner, return that in *ownerp. */ int sx_chain(struct thread *td, struct thread **ownerp) { struct sx *sx; struct cv *cv; /* * First, see if it looks like td is blocked on a condition * variable. */ cv = td->td_wchan; if (cv->cv_description != td->td_wmesg) return (0); /* * Ok, see if it looks like td is blocked on the exclusive * condition variable. */ sx = (struct sx *)((char *)cv - offsetof(struct sx, sx_excl_cv)); if (LOCK_CLASS(&sx->sx_object) == &lock_class_sx && sx->sx_excl_wcnt > 0) goto ok; /* * Second, see if it looks like td is blocked on the shared * condition variable. */ sx = (struct sx *)((char *)cv - offsetof(struct sx, sx_shrd_cv)); if (LOCK_CLASS(&sx->sx_object) == &lock_class_sx && sx->sx_shrd_wcnt > 0) goto ok; /* Doesn't seem to be an sx lock. */ return (0); ok: /* We think we have an sx lock, so output some details. */ db_printf("blocked on sx \"%s\" ", td->td_wmesg); if (sx->sx_cnt >= 0) { db_printf("SLOCK (count %d)\n", sx->sx_cnt); *ownerp = NULL; } else { db_printf("XLOCK\n"); *ownerp = sx->sx_xholder; } return (1); } #endif Index: head/sys/sys/lock.h =================================================================== --- head/sys/sys/lock.h (revision 167053) +++ head/sys/sys/lock.h (revision 167054) @@ -1,330 +1,331 @@ /*- * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Berkeley Software Design Inc's name may not be used to endorse or * promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from BSDI $Id: mutex.h,v 2.7.2.35 2000/04/27 03:10:26 cp Exp $ * $FreeBSD$ */ #ifndef _SYS_LOCK_H_ #define _SYS_LOCK_H_ #include #include struct thread; /* * Lock classes. Each lock has a class which describes characteristics * common to all types of locks of a given class. * * Spin locks in general must always protect against preemption, as it is * an error to perform any type of context switch while holding a spin lock. * Also, for an individual lock to be recursable, its class must allow * recursion and the lock itself must explicitly allow recursion. */ struct lock_class { const char *lc_name; u_int lc_flags; void (*lc_ddb_show)(struct lock_object *lock); }; #define LC_SLEEPLOCK 0x00000001 /* Sleep lock. */ #define LC_SPINLOCK 0x00000002 /* Spin lock. */ #define LC_SLEEPABLE 0x00000004 /* Sleeping allowed with this lock. */ #define LC_RECURSABLE 0x00000008 /* Locks of this type may recurse. */ #define LC_UPGRADABLE 0x00000010 /* Upgrades and downgrades permitted. */ #define LO_CLASSFLAGS 0x0000ffff /* Class specific flags. */ #define LO_INITIALIZED 0x00010000 /* Lock has been initialized. */ #define LO_WITNESS 0x00020000 /* Should witness monitor this lock. */ #define LO_QUIET 0x00040000 /* Don't log locking operations. */ #define LO_RECURSABLE 0x00080000 /* Lock may recurse. */ #define LO_SLEEPABLE 0x00100000 /* Lock may be held while sleeping. */ #define LO_UPGRADABLE 0x00200000 /* Lock may be upgraded/downgraded. */ #define LO_DUPOK 0x00400000 /* Don't check for duplicate acquires */ #define LO_ENROLLPEND 0x00800000 /* On the pending enroll list. */ #define LO_CLASSMASK 0x0f000000 /* Class index bitmask. */ #define LO_NOPROFILE 0x10000000 /* Don't profile this lock */ +#define LO_CONTESTED 0x20000000 /* Lock was contested */ /* * Lock classes are statically assigned an index into the gobal lock_classes * array. Debugging code looks up the lock class for a given lock object * by indexing the array. */ #define LO_CLASSSHIFT 24 #define LO_CLASSINDEX(lock) ((((lock)->lo_flags) & LO_CLASSMASK) >> LO_CLASSSHIFT) #define LOCK_CLASS(lock) (lock_classes[LO_CLASSINDEX((lock))]) #define LOCK_CLASS_MAX (LO_CLASSMASK >> LO_CLASSSHIFT) #define LI_RECURSEMASK 0x0000ffff /* Recursion depth of lock instance. */ #define LI_EXCLUSIVE 0x00010000 /* Exclusive lock instance. */ /* * Option flags passed to lock operations that witness also needs to know * about or that are generic across all locks. */ #define LOP_NEWORDER 0x00000001 /* Define a new lock order. */ #define LOP_QUIET 0x00000002 /* Don't log locking operations. */ #define LOP_TRYLOCK 0x00000004 /* Don't check lock order. */ #define LOP_EXCLUSIVE 0x00000008 /* Exclusive lock. */ #define LOP_DUPOK 0x00000010 /* Don't check for duplicate acquires */ /* Flags passed to witness_assert. */ #define LA_UNLOCKED 0x00000000 /* Lock is unlocked. */ #define LA_LOCKED 0x00000001 /* Lock is at least share locked. */ #define LA_SLOCKED 0x00000002 /* Lock is exactly share locked. */ #define LA_XLOCKED 0x00000004 /* Lock is exclusively locked. */ #define LA_RECURSED 0x00000008 /* Lock is recursed. */ #define LA_NOTRECURSED 0x00000010 /* Lock is not recursed. */ #ifdef _KERNEL /* * Lock instances. A lock instance is the data associated with a lock while * it is held by witness. For example, a lock instance will hold the * recursion count of a lock. Lock instances are held in lists. Spin locks * are held in a per-cpu list while sleep locks are held in per-thread list. */ struct lock_instance { struct lock_object *li_lock; const char *li_file; /* File and line of last acquire. */ int li_line; u_int li_flags; /* Recursion count and LI_* flags. */ }; /* * A simple list type used to build the list of locks held by a thread * or CPU. We can't simply embed the list in struct lock_object since a * lock may be held by more than one thread if it is a shared lock. Locks * are added to the head of the list, so we fill up each list entry from * "the back" logically. To ease some of the arithmetic, we actually fill * in each list entry the normal way (childer[0] then children[1], etc.) but * when we traverse the list we read children[count-1] as the first entry * down to children[0] as the final entry. */ #define LOCK_NCHILDREN 3 struct lock_list_entry { struct lock_list_entry *ll_next; struct lock_instance ll_children[LOCK_NCHILDREN]; u_int ll_count; }; /* * If any of WITNESS, INVARIANTS, or KTR_LOCK KTR tracing has been enabled, * then turn on LOCK_DEBUG. When this option is on, extra debugging * facilities such as tracking the file and line number of lock operations * are enabled. Also, mutex locking operations are not inlined to avoid * bloat from all the extra debugging code. We also have to turn on all the * calling conventions for this debugging code in modules so that modules can * work with both debug and non-debug kernels. */ #if defined(KLD_MODULE) || defined(WITNESS) || defined(INVARIANTS) || defined(INVARIANT_SUPPORT) || defined(KTR) || defined(LOCK_PROFILING) #define LOCK_DEBUG 1 #else #define LOCK_DEBUG 0 #endif /* * In the LOCK_DEBUG case, use the filename and line numbers for debugging * operations. Otherwise, use default values to avoid the unneeded bloat. */ #if LOCK_DEBUG > 0 #define LOCK_FILE __FILE__ #define LOCK_LINE __LINE__ #else #define LOCK_FILE NULL #define LOCK_LINE 0 #endif /* * Macros for KTR_LOCK tracing. * * opname - name of this operation (LOCK/UNLOCK/SLOCK, etc.) * lo - struct lock_object * for this lock * flags - flags passed to the lock operation * recurse - this locks recursion level (or 0 if class is not recursable) * result - result of a try lock operation * file - file name * line - line number */ #define LOCK_LOG_TEST(lo, flags) \ (((flags) & LOP_QUIET) == 0 && ((lo)->lo_flags & LO_QUIET) == 0) #define LOCK_LOG_LOCK(opname, lo, flags, recurse, file, line) do { \ if (LOCK_LOG_TEST((lo), (flags))) \ CTR5(KTR_LOCK, opname " (%s) %s r = %d at %s:%d", \ LOCK_CLASS(lo)->lc_name, (lo)->lo_name, \ (u_int)(recurse), (file), (line)); \ } while (0) #define LOCK_LOG_TRY(opname, lo, flags, result, file, line) do { \ if (LOCK_LOG_TEST((lo), (flags))) \ CTR5(KTR_LOCK, "TRY_" opname " (%s) %s result=%d at %s:%d",\ LOCK_CLASS(lo)->lc_name, (lo)->lo_name, \ (u_int)(result), (file), (line)); \ } while (0) #define LOCK_LOG_INIT(lo, flags) do { \ if (LOCK_LOG_TEST((lo), (flags))) \ CTR4(KTR_LOCK, "%s: %p (%s) %s", __func__, (lo), \ LOCK_CLASS(lo)->lc_name, (lo)->lo_name); \ } while (0) #define LOCK_LOG_DESTROY(lo, flags) LOCK_LOG_INIT(lo, flags) #define lock_initalized(lo) ((lo)->lo_flags & LO_INITIALIZED) /* * Helpful macros for quickly coming up with assertions with informative * panic messages. */ #define MPASS(ex) MPASS4(ex, #ex, __FILE__, __LINE__) #define MPASS2(ex, what) MPASS4(ex, what, __FILE__, __LINE__) #define MPASS3(ex, file, line) MPASS4(ex, #ex, file, line) #define MPASS4(ex, what, file, line) \ KASSERT((ex), ("Assertion %s failed at %s:%d", what, file, line)) extern struct lock_class lock_class_mtx_sleep; extern struct lock_class lock_class_mtx_spin; extern struct lock_class lock_class_sx; extern struct lock_class lock_class_rw; extern struct lock_class lock_class_lockmgr; extern struct lock_class *lock_classes[]; void lock_init(struct lock_object *, struct lock_class *, const char *, const char *, int); void lock_destroy(struct lock_object *); void spinlock_enter(void); void spinlock_exit(void); void witness_init(struct lock_object *); void witness_destroy(struct lock_object *); int witness_defineorder(struct lock_object *, struct lock_object *); void witness_checkorder(struct lock_object *, int, const char *, int); void witness_lock(struct lock_object *, int, const char *, int); void witness_upgrade(struct lock_object *, int, const char *, int); void witness_downgrade(struct lock_object *, int, const char *, int); void witness_unlock(struct lock_object *, int, const char *, int); void witness_save(struct lock_object *, const char **, int *); void witness_restore(struct lock_object *, const char *, int); int witness_list_locks(struct lock_list_entry **); int witness_warn(int, struct lock_object *, const char *, ...); void witness_assert(struct lock_object *, int, const char *, int); void witness_display_spinlock(struct lock_object *, struct thread *); int witness_line(struct lock_object *); const char *witness_file(struct lock_object *); #ifdef WITNESS /* Flags for witness_warn(). */ #define WARN_GIANTOK 0x01 /* Giant is exempt from this check. */ #define WARN_PANIC 0x02 /* Panic if check fails. */ #define WARN_SLEEPOK 0x04 /* Sleepable locks are exempt from check. */ #define WITNESS_INIT(lock) \ witness_init((lock)) #define WITNESS_DESTROY(lock) \ witness_destroy(lock) #define WITNESS_CHECKORDER(lock, flags, file, line) \ witness_checkorder((lock), (flags), (file), (line)) #define WITNESS_DEFINEORDER(lock1, lock2) \ witness_defineorder((struct lock_object *)(lock1), \ (struct lock_object *)(lock2)) #define WITNESS_LOCK(lock, flags, file, line) \ witness_lock((lock), (flags), (file), (line)) #define WITNESS_UPGRADE(lock, flags, file, line) \ witness_upgrade((lock), (flags), (file), (line)) #define WITNESS_DOWNGRADE(lock, flags, file, line) \ witness_downgrade((lock), (flags), (file), (line)) #define WITNESS_UNLOCK(lock, flags, file, line) \ witness_unlock((lock), (flags), (file), (line)) #define WITNESS_CHECK(flags, lock, fmt, ...) \ witness_warn((flags), (lock), (fmt), ## __VA_ARGS__) #define WITNESS_WARN(flags, lock, fmt, ...) \ witness_warn((flags), (lock), (fmt), ## __VA_ARGS__) #define WITNESS_SAVE_DECL(n) \ const char * __CONCAT(n, __wf); \ int __CONCAT(n, __wl) #define WITNESS_SAVE(lock, n) \ witness_save((lock), &__CONCAT(n, __wf), &__CONCAT(n, __wl)) #define WITNESS_RESTORE(lock, n) \ witness_restore((lock), __CONCAT(n, __wf), __CONCAT(n, __wl)) #define WITNESS_FILE(lock) \ witness_file(lock) #define WITNESS_LINE(lock) \ witness_line(lock) #else /* WITNESS */ #define WITNESS_INIT(lock) #define WITNESS_DESTROY(lock) #define WITNESS_DEFINEORDER(lock1, lock2) 0 #define WITNESS_CHECKORDER(lock, flags, file, line) #define WITNESS_LOCK(lock, flags, file, line) #define WITNESS_UPGRADE(lock, flags, file, line) #define WITNESS_DOWNGRADE(lock, flags, file, line) #define WITNESS_UNLOCK(lock, flags, file, line) #define WITNESS_CHECK(flags, lock, fmt, ...) 0 #define WITNESS_WARN(flags, lock, fmt, ...) #define WITNESS_SAVE_DECL(n) #define WITNESS_SAVE(lock, n) #define WITNESS_RESTORE(lock, n) #define WITNESS_FILE(lock) ("?") #define WITNESS_LINE(lock) (0) #endif /* WITNESS */ /* * Helper macros to allow developers to add explicit lock order checks * wherever they please without having to actually grab a lock to do so. */ #define witness_check_mutex(m) \ WITNESS_CHECKORDER(&(m)->mtx_object, LOP_EXCLUSIVE, LOCK_FILE, \ LOCK_LINE) #define witness_check_shared_sx(sx) \ WITNESS_CHECKORDER(&(sx)->sx_object, 0, LOCK_FILE, LOCK_LINE) #define witness_check_exclusive_sx(sx) \ WITNESS_CHECKORDER(&(sx)->sx_object, LOP_EXCLUSIVE, LOCK_FILE, \ LOCK_LINE) #endif /* _KERNEL */ #endif /* _SYS_LOCK_H_ */ Index: head/sys/sys/lock_profile.h =================================================================== --- head/sys/sys/lock_profile.h (revision 167053) +++ head/sys/sys/lock_profile.h (revision 167054) @@ -1,156 +1,163 @@ /*- * Copyright (c) 2006 Kip Macy kmacy@FreeBSD.org * Copyright (c) 2006 Kris Kennaway kris@FreeBSD.org * Copyright (c) 2006 Dag-Erling Smorgrav des@des.no * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHAL THE AUTHORS BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_LOCK_PROFILE_H_ #define _SYS_LOCK_PROFILE_H_ #ifdef LOCK_PROFILING #include #include #include #include #include #ifndef LPROF_HASH_SIZE #define LPROF_HASH_SIZE 4096 #define LPROF_HASH_MASK (LPROF_HASH_SIZE - 1) #endif #ifndef USE_CPU_NANOSECONDS u_int64_t nanoseconds(void); #endif struct lock_prof { const char *name; const char *type; const char *file; u_int namehash; int line; uintmax_t cnt_max; uintmax_t cnt_tot; uintmax_t cnt_wait; uintmax_t cnt_cur; uintmax_t cnt_contest_holding; uintmax_t cnt_contest_locking; }; extern struct lock_prof lprof_buf[LPROF_HASH_SIZE]; #define LPROF_SBUF_SIZE 256 * 400 /* We keep a smaller pool of spin mutexes for protecting the lprof hash entries */ #define LPROF_LOCK_SIZE 16 #define LPROF_LOCK_MASK (LPROF_LOCK_SIZE - 1) #define LPROF_LHASH(hash) ((hash) & LPROF_LOCK_MASK) #define LPROF_LOCK(hash) mtx_lock_spin(&lprof_locks[LPROF_LHASH(hash)]) #define LPROF_UNLOCK(hash) mtx_unlock_spin(&lprof_locks[LPROF_LHASH(hash)]) extern struct mtx lprof_locks[LPROF_LOCK_SIZE]; extern int lock_prof_enable; void _lock_profile_obtain_lock_success(struct lock_object *lo, int contested, uint64_t waittime, const char *file, int line); void _lock_profile_update_wait(struct lock_object *lo, uint64_t waitstart); void _lock_profile_release_lock(struct lock_object *lo); static inline void lock_profile_object_init(struct lock_object *lo, struct lock_class *class, const char *name) { const char *p; u_int hash = 0; struct lock_profile_object *l = &lo->lo_profile_obj; lo->lo_flags = 0; lo->lo_name = name; l->lpo_acqtime = 0; l->lpo_waittime = 0; l->lpo_filename = NULL; l->lpo_lineno = 0; l->lpo_contest_holding = 0; l->lpo_contest_locking = 0; l->lpo_type = class->lc_name; /* Hash the mutex name to an int so we don't have to strcmp() it repeatedly */ for (p = name; *p != '\0'; p++) hash = 31 * hash + *p; l->lpo_namehash = hash; #if 0 if (opts & MTX_PROFILE) l->lpo_stack = stack_create(); #endif } static inline void lock_profile_object_destroy(struct lock_object *lo) { #if 0 struct lock_profile_object *l = &lo->lo_profile_obj; if (lo->lo_flags & LO_PROFILE) stack_destroy(l->lpo_stack); #endif } static inline void lock_profile_obtain_lock_failed(struct lock_object *lo, int *contested, uint64_t *waittime) { struct lock_profile_object *l = &lo->lo_profile_obj; if (lock_prof_enable && *contested == 0) { *waittime = nanoseconds(); + lo->lo_flags |= LO_CONTESTED; atomic_add_int(&l->lpo_contest_holding, 1); *contested = 1; } } static inline void lock_profile_obtain_lock_success(struct lock_object *lo, int contested, uint64_t waittime, const char *file, int line) { if (lock_prof_enable) { _lock_profile_obtain_lock_success(lo, contested, waittime, file, line); } } static inline void lock_profile_release_lock(struct lock_object *lo) { struct lock_profile_object *l = &lo->lo_profile_obj; - if (lock_prof_enable || l->lpo_acqtime) +#ifdef LOCK_PROFILING_FAST + if((lo->lo_flags & LO_CONTESTED) == 0) + return; +#endif + if (lock_prof_enable || l->lpo_acqtime) { + lo->lo_flags &= ~LO_CONTESTED; _lock_profile_release_lock(lo); + } } #else /* !LOCK_PROFILING */ static inline void lock_profile_update_wait(struct lock_object *lo, uint64_t waitstart) {;} static inline void lock_profile_update_contest_locking(struct lock_object *lo, int contested) {;} static inline void lock_profile_release_lock(struct lock_object *lo) {;} static inline void lock_profile_obtain_lock_failed(struct lock_object *lo, int *contested, uint64_t *waittime) {;} static inline void lock_profile_obtain_lock_success(struct lock_object *lo, int contested, uint64_t waittime, const char *file, int line) {;} static inline void lock_profile_object_destroy(struct lock_object *lo) {;} static inline void lock_profile_object_init(struct lock_object *lo, struct lock_class *class, const char *name) {;} #endif /* !LOCK_PROFILING */ #endif /* _SYS_LOCK_PROFILE_H_ */