diff --git a/usr.sbin/ctladm/ctladm.8 b/usr.sbin/ctladm/ctladm.8
index ba2712cb278c..6e084c30de36 100644
--- a/usr.sbin/ctladm/ctladm.8
+++ b/usr.sbin/ctladm/ctladm.8
@@ -1,1097 +1,1112 @@
 .\"
 .\" Copyright (c) 2003 Silicon Graphics International Corp.
 .\" Copyright (c) 2015-2021 Alexander Motin <mav@FreeBSD.org>
 .\" Copyright (c) 2018 Marcelo Araujo <araujo@FreeBSD.org>
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
 .\" modification, are permitted provided that the following conditions
 .\" are met:
 .\" 1. Redistributions of source code must retain the above copyright
 .\"    notice, this list of conditions, and the following disclaimer,
 .\"    without modification.
 .\" 2. Redistributions in binary form must reproduce at minimum a disclaimer
 .\"    substantially similar to the "NO WARRANTY" disclaimer below
 .\"    ("Disclaimer") and any redistribution must be conditioned upon
 .\"    including a substantially similar Disclaimer requirement for further
 .\"    binary redistribution.
 .\"
 .\" NO WARRANTY
 .\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 .\" "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 .\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
 .\" A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 .\" HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 .\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 .\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 .\" POSSIBILITY OF SUCH DAMAGES.
 .\"
 .\" ctladm utility man page.
 .\"
 .\" Author: Ken Merry <ken@FreeBSD.org>
 .\"
 .\" $Id: //depot/users/kenm/FreeBSD-test2/usr.sbin/ctladm/ctladm.8#3 $
 .\"
-.Dd December 27, 2023
+.Dd May 2, 2024
 .Dt CTLADM 8
 .Os
 .Sh NAME
 .Nm ctladm
 .Nd CAM Target Layer control utility
 .Sh SYNOPSIS
 .Nm
 .Aq Ar command
 .Op lun
 .Op generic args
 .Op command args
 .Nm
 .Ic tur
 .Aq lun
 .Op general options
 .Nm
 .Ic inquiry
 .Aq lun
 .Op general options
 .Nm
 .Ic reqsense
 .Aq lun
 .Op general options
 .Nm
 .Ic reportluns
 .Aq lun
 .Op general options
 .Nm
 .Ic read
 .Aq lun
 .Op general options
 .Aq Fl l Ar lba
 .Aq Fl d Ar datalen
 .Aq Fl f Ar file|-
 .Aq Fl b Ar blocksize_bytes
 .Op Fl c Ar cdbsize
 .Op Fl N
 .Nm
 .Ic write
 .Aq lun
 .Op general options
 .Aq Fl l Ar lba
 .Aq Fl d Ar datalen
 .Aq Fl f Ar file|-
 .Aq Fl b Ar blocksize_bytes
 .Op Fl c Ar cdbsize
 .Op Fl N
 .Nm
 .Ic readcap
 .Aq lun
 .Op general options
 .Op Fl c Ar cdbsize
 .Nm
 .Ic modesense
 .Aq lun
 .Aq Fl m Ar page | Fl l
 .Op Fl P Ar pc
 .Op Fl d
 .Op Fl S Ar subpage
 .Op Fl c Ar size
 .Nm
 .Ic start
 .Aq lun
 .Op general options
 .Op Fl i
 .Op Fl o
 .Nm
 .Ic stop
 .Aq lun
 .Op general options
 .Op Fl i
 .Op Fl o
 .Nm
 .Ic synccache
 .Aq lun
 .Op general options
 .Op Fl l Ar lba
 .Op Fl b Ar blockcount
 .Op Fl r
 .Op Fl i
 .Op Fl c Ar cdbsize
 .Nm
 .Ic lunlist
 .Nm
 .Ic delay
 .Aq lun
 .Aq Fl l Ar datamove|done
 .Aq Fl t Ar secs
 .Op Fl T Ar oneshot|cont
 .Nm
 .Ic inject
 .Aq Fl i Ar action
 .Aq Fl p Ar pattern
 .Op Fl r Ar lba,len
 .Op Fl s Ar len fmt Op Ar args
 .Op Fl c
 .Op Fl d Ar delete_id
 .Nm
 .Ic create
 .Aq Fl b Ar backend
 .Op Fl B Ar blocksize
 .Op Fl d Ar device_id
 .Op Fl l Ar lun_id
 .Op Fl o Ar name=value
 .Op Fl s Ar size_bytes
 .Op Fl S Ar serial_num
 .Op Fl t Ar device_type
 .Nm
 .Ic remove
 .Aq Fl b Ar backend
 .Aq Fl l Ar lun_id
 .Op Fl o Ar name=value
 .Nm
 .Ic modify
 .Aq Fl b Ar backend
 .Aq Fl l Ar lun_id
 .Op Fl o Ar name=value
 .Aq Fl s Ar size_bytes
 .Nm
 .Ic devlist
 .Op Fl b Ar backend
 .Op Fl v
 .Op Fl x
 .Nm
 .Ic port
 .Op Fl c
 .Op Fl o Ar on|off
 .Op Fl w Ar wwpn
 .Op Fl W Ar wwnn
 .Op Fl O Ar pp|vp
 .Op Fl p Ar targ_port
 .Op Fl r
 .Op Fl t Ar fe_type
 .Nm
 .Ic portlist
 .Op Fl f Ar frontend
 .Op Fl i
 .Op Fl l
 .Op Fl p Ar targ_port
 .Op Fl q
 .Op Fl v
 .Op Fl x
 .Nm
 .Ic lunmap
 .Aq Fl p Ar targ_port
 .Op Fl l Ar pLUN
 .Op Fl L Ar cLUN
 .Nm
 .Ic dumpooa
 .Nm
 .Ic dumpstructs
 .Nm
 .Ic islist
 .Op Fl v
 .Op Fl x
 .Nm
 .Ic islogout
 .Aq Fl a | Fl c Ar connection-id | Fl i Ar name | Fl p Ar portal
 .Nm
 .Ic isterminate
 .Aq Fl a | Fl c Ar connection-id | Fl i Ar name | Fl p Ar portal
 .Nm
+.Ic nvlist
+.Op Fl v
+.Op Fl x
+.Nm
 .Ic help
 .Sh DESCRIPTION
 The
 .Nm
 utility is designed to provide a way to access and control the CAM Target
 Layer (CTL).
 It provides a way to send
 .Tn SCSI
 commands to the CTL layer, and also provides
 some meta-commands that utilize
 .Tn SCSI
 commands.
 (For instance, the
 .Ic lunlist
 command is implemented using the
 .Tn SCSI
 REPORT LUNS and INQUIRY commands.)
 .Pp
 The
 .Nm
 utility has a number of primary functions, many of which require a device
 identifier.
 The device identifier takes the following form:
 .Bl -tag -width 14n
 .It lun
 Specify the LUN number to operate on.
 .El
 Many of the primary functions of the
 .Nm
 utility take the following optional arguments:
 .Bl -tag -width 10n
 .It Fl C Ar retries
 Specify the number of times to retry a command in the event of failure.
 .It Fl D Ar device
 Specify the device to open.
 This allows opening a device other than the default device,
 .Pa /dev/cam/ctl ,
 to be opened for sending commands.
 .It Fl I Ar id
 Specify the initiator number to use.
 By default,
 .Nm
 will use 7 as the initiator number.
 .El
 .Pp
 Primary commands:
 .Bl -tag -width 11n
 .It Ic tur
 Send the
 .Tn SCSI
 TEST UNIT READY command to the device and report whether or not it is
 ready.
 .It Ic inquiry
 Send the
 .Tn SCSI
 INQUIRY command to the device and display some of the returned inquiry
 data.
 .It Ic reqsense
 Send the
 .Tn SCSI
 REQUEST SENSE command to the device and display the returned sense
 information.
 .It Ic reportluns
 Send the
 .Tn SCSI
 REPORT LUNS command to the device and display supported LUNs.
 .It Ic read
 Send a
 .Tn SCSI
 READ command to the device, and write the requested data to a file or
 stdout.
 .Bl -tag -width 12n
 .It Fl l Ar lba
 Specify the starting Logical Block Address for the READ.
 This can be specified in decimal, octal (starting with 0),
 hexadecimal (starting with 0x) or any other base supported by
 .Xr strtoull 3 .
 .It Fl d Ar datalen
 Specify the length, in 512 byte blocks, of the READ request.
 .It Fl f Ar file
 Specify the destination for the data read by the READ command.
 Either a filename or
 .Sq -
 for stdout may be specified.
 .It Fl c Ar cdbsize
 Specify the minimum
 .Tn SCSI
 CDB (Command Data Block) size to be used for the READ request.
 Allowable values are 6, 10, 12 and 16.
 Depending upon the LBA and amount of data requested, a larger CDB
 size may be used to satisfy the request.  (e.g., for LBAs above 0xffffffff,
 READ(16) must be used to satisfy the request.)
 .It Fl b Ar blocksize
 Specify the blocksize of the underlying
 .Tn SCSI
 device, so the transfer length
 can be calculated accurately.
 The blocksize can be obtained via the
 .Tn SCSI
 READ CAPACITY command.
 .It Fl N
 Do not copy data to
 .Nm
 from the kernel when doing a read, just execute the command without copying
 data.
 This is to be used for performance testing.
 .El
 .It Ic write
 Read data from a file or stdin, and write the data to the device using the
 .Tn SCSI
 WRITE command.
 .Bl -tag -width 12n
 .It Fl l Ar lba
 Specify the starting Logical Block Address for the WRITE.
 This can be specified in decimal, octal (starting with 0), hexadecimal
 (starting with 0x) or any other base supported by
 .Xr strtoull 3 .
 .It Fl d Ar atalen
 Specify the length, in 512 byte blocks, of the WRITE request.
 .It Fl f Ar file
 Specify the source for the data to be written by the WRITE command.
 Either a filename or
 .Sq -
 for stdin may be specified.
 .It Fl c Ar cdbsize
 Specify the minimum
 .Tn SCSI
 CDB (Command Data Block) size to be used for the READ request.
 Allowable values are 6, 10, 12 and 16.
 Depending upon the LBA and amount of data requested, a larger CDB size
 may be used to satisfy the request.  (e.g., for LBAs above 0xffffffff, READ(16)
 must be used to satisfy the request.)
 .It Fl b Ar blocksize
 Specify the blocksize of the underlying
 .Tn SCSI
 device, so the transfer length
 can be calculated accurately.
 The blocksize can be obtained via the
 .Tn SCSI
 READ CAPACITY command.
 .It Fl N
 Do not copy data to
 .Nm
 to the kernel when doing a write, just execute the command without copying
 data.
 This is to be used for performance testing.
 .El
 .It Ic readcap
 Send the
 .Tn SCSI
 READ CAPACITY command to the device and display the device size and device
 block size.
 By default, READ CAPACITY(10) is used.
 If the device returns a maximum LBA of 0xffffffff, however,
 .Nm
 will automatically issue a READ CAPACITY(16), which is implemented as a
 service action of the SERVICE ACTION IN(16) opcode.
 The user can specify the minimum CDB size with the
 .Fl c
 argument.
 Valid values for the
 .Fl c
 option are 10 and 16.
 If a 10 byte CDB is specified, the request will be automatically reissued
 with a 16 byte CDB if the maximum LBA returned is 0xffffffff.
 .It Ic modesense
 Send a
 .Tn SCSI
 MODE SENSE command to the device, and display the requested mode page(s) or
 page list.
 .Bl -tag -width 10n
 .It Fl m Ar page
 Specify the mode page to display.
 This option and the
 .Fl l
 option are mutually exclusive.
 One of the two must be specified, though.
 Mode page numbers may be specified in decimal or hexadecimal.
 .It Fl l
 Request that the list of mode pages supported by the device be returned.
 This option and the
 .Fl m
 option are mutually exclusive.
 One of the two must be specified, though.
 .It Fl P Ar pc
 Specify the mode page control value.
 Possible values are:
 .Bl -tag -width 2n -compact
 .It 0
 Current values.
 .It 1
 Changeable value bitmask.
 .It 2
 Default values.
 .It 3
 Saved values.
 .El
 .It Fl d
 Disable block descriptors when sending the mode sense request.
 .It Fl S Ar subpage
 Specify the subpage used with the mode sense request.
 .It Fl c Ar cdbsize
 Specify the CDB size used for the mode sense request.
 Supported values are 6 and 10.
 .El
 .It Ic start
 Send the
 .Tn SCSI
 START STOP UNIT command to the specified LUN with the start
 bit set.
 .Bl -tag -width 4n
 .It Fl i
 Set the immediate bit in the CDB.
 Note that CTL does not support the immediate bit, so this is primarily
 useful for making sure that CTL returns the proper error.
 .El
 .It Ic stop
 Send the
 .Tn SCSI
 START STOP UNIT command to the specified LUN with the start
 bit cleared.
 We use an ordered tag to stop the LUN, so we can guarantee that all pending
 I/O executes before it is stopped.
 (CTL guarantees this anyway, but
 .Nm
 sends an ordered tag for completeness.)
 .Bl -tag -width 4n
 .It Fl i
 Set the immediate bit in the CDB.
 Note that CTL does not support the immediate bit, so this is primarily
 useful for making sure that CTL returns the proper error.
 .El
 .It Ic synccache
 Send the
 .Tn SCSI
 SYNCHRONIZE CACHE command to the device.
 By default, SYNCHRONIZE CACHE(10) is used.
 If the specified starting LBA is greater than 0xffffffff or the length is
 greater than 0xffff, though, SYNCHRONIZE CACHE(16) will be used.
 The 16 byte command will also be used if the user specifies a 16 byte CDB with the
 .Fl c
 argument.
 .Bl -tag -width 14n
 .It Fl l Ar lba
 Specify the starting LBA of the cache region to synchronize.
 This option is a no-op for CTL.
 If you send a SYNCHRONIZE CACHE command, it will sync the cache for the entire LUN.
 .It Fl b Ar blockcount
 Specify the length of the cache region to synchronize.
 This option is a no-op for CTL.
 If you send a SYNCHRONIZE CACHE command, it will sync the cache for the entire LUN.
 .It Fl r
 Specify relative addressing for the starting LBA.
 CTL does not support relative addressing, since it only works for linked commands,
 and CTL does not support linked commands.
 .It Fl i
 Tell the target to return status immediately after issuing the SYNCHRONIZE CACHE
 command rather than waiting for the cache to finish syncing.
 CTL does not support this bit.
 .It Fl c Ar cdbsize
 Specify the minimum CDB size.
 Valid values are 10 and 16 bytes.
 .El
 .It Ic lunlist
 List all LUNs registered with CTL.
 Because this command uses the ioctl port, it will only work when the FETDs
 (Front End Target Drivers) are enabled.
 This command is the equivalent of doing a REPORT LUNS on one LUN and then
 an INQUIRY on each LUN in the system.
 .It Ic delay
 Delay commands at the given location.
 There are two places where commands may be delayed currently: before data is transferred
 .Pq Dq datamove
 and just prior to sending status to the host
 .Pq Dq done .
 One of the two must be supplied as an argument to the
 .Fl l
 option.
 The
 .Fl t
 option must also be specified.
 .Bl -tag -width 12n
 .It Fl l Ar delayloc
 Delay command(s) at the specified location.
 This can either be at the data movement stage (datamove) or prior to
 command completion (done).
 .It Fl t Ar delaytime
 Delay command(s) for the specified number of seconds.
 This must be specified.
 If set to 0, it will clear out any previously set delay for this particular
 location (datamove or done).
 .It Fl T Ar delaytype
 Specify the delay type.
 By default, the
 .Ic delay
 option will delay the next command sent to the given LUN.
 With the
 .Fl T Ar cont
 option, every command will be delayed by the specified period of time.
 With the
 .Fl T Ar oneshot
 the next command sent to the given LUN will be delayed and all subsequent
 commands will be completed normally.
 This is the default.
 .El
 .It Ic inject
 Inject the specified type of error for the LUN specified, when a command
 that matches the given pattern is seen.
 The sense data returned is in either fixed or descriptor format, depending
 upon the status of the D_SENSE bit in the control mode page (page 0xa) for
 the LUN.
 .Pp
 Errors are only injected for commands that have not already failed for
 other reasons.
 By default, only the first command matching the pattern specified is
 returned with the supplied error.
 .Pp
 If the
 .Fl c
 flag is specified, all commands matching the pattern will be returned with
 the specified error until the error injection command is deleted with
 .Fl d
 flag.
 .Bl -tag -width 17n
 .It Fl i Ar action
 Specify the error to return:
 .Bl -tag -width 10n
 .It aborted
 Return the next matching command on the specified LUN with the sense key
 ABORTED COMMAND (0x0b), and the ASC/ASCQ 0x45,0x00 ("Select or reselect
 failure").
 .It mediumerr
 Return the next matching command on the specified LUN with the sense key
 MEDIUM ERROR (0x03) and the ASC/ASCQ 0x11,0x00 ("Unrecovered read error") for
 reads, or ASC/ASCQ 0x0c,0x02 ("Write error - auto reallocation failed")
 for write errors.
 .It ua
 Return the next matching command on the specified LUN with the sense key
 UNIT ATTENTION (0x06) and the ASC/ASCQ 0x29,0x00 ("POWER ON, RESET, OR BUS
 DEVICE RESET OCCURRED").
 .It custom
 Return the next matching command on the specified LUN with the supplied
 sense data.
 The
 .Fl s
 argument must be specified.
 .El
 .It Fl p Ar pattern
 Specify which commands should be returned with the given error.
 .Bl -tag -width 10n
 .It read
 The error should apply to READ(6), READ(10), READ(12), READ(16), etc.
 .It write
 The error should apply to WRITE(6), WRITE(10), WRITE(12), WRITE(16), WRITE
 AND VERIFY(10), etc.
 .It rw
 The error should apply to both read and write type commands.
 .It readcap
 The error should apply to READ CAPACITY(10) and READ CAPACITY(16) commands.
 .It tur
 The error should apply to TEST UNIT READY commands.
 .It any
 The error should apply to any command.
 .El
 .It Fl r Ar lba,len
 Specify the starting lba and length of the range of LBAs which should
 trigger an error.
 This option is only applies when read and/or write patterns are specified.
 If used with other command types, the error will never be triggered.
 .It Fl s Ar len fmt Op Ar args
 Specify the sense data that is to be returned for custom actions.
 If the format is
 .Sq - ,
 len bytes of sense data will be read from standard input and written to the
 sense buffer.
 If len is longer than 252 bytes (the maximum allowable
 .Tn SCSI
 sense data length), it will be truncated to that length.
 The sense data format is described in
 .Xr cam_cdbparse 3 .
 .It Fl c
 The error injection should be persistent, instead of happening once.
 Persistent errors must be deleted with the
 .Fl d
 argument.
 .It Fl d Ar delete_id
 Delete the specified error injection serial number.
 The serial number is returned when the error is injected.
 .El
 .It Ic port
 Perform one of several CTL frontend port operations.
 Either get a list of frontend ports
 .Pq Fl l ,
 turn one or more frontends on
 or off
 .Pq Fl o Ar on|off ,
 or set the World Wide Node Name
 .Pq Fl w Ar wwnn
 or World Wide Port Name
 .Pq Fl W Ar wwpn
 for a given port.
 One of
 .Fl l ,
 .Fl o ,
 or
 .Fl w
 or
 .Fl W
 must be specified.
 The WWNN and WWPN may both be specified at the same time, but cannot be
 combined with enabling/disabling or listing ports.
 .Bl -tag -width 12n
 .It Fl c
 Create new frontend port using free pp and vp=0.
 .It Fl o Ar on|off
 Turn the specified CTL frontend ports on or off.
 If no port number or port type is specified, all ports are turned on or
 off.
 .It Fl O Ar pp|vp
 Specify generic options on the ioctl frontend port.
 At present, only pp and vp port numbers can be set.
 .It Fl p Ar targ_port
 Specify the frontend port number.
 The port numbers can be found in the frontend port list.
 .It Fl r
 Remove port specified with
 .Pq Fl p Ar targ_port .
 .It Fl t Ar fe_type
 Specify the frontend type.
 Currently defined port types are
 .Dq fc
 (Fibre Channel),
 .Dq scsi
 (Parallel SCSI),
 .Dq ioctl
 (CTL ioctl interface),
 and
 .Dq internal
 (CTL CAM SIM).
 .It Fl w Ar wwnn
 Set the World Wide Node Name for the given port.
 The
 .Fl n
 argument must be specified, since this is only possible to implement on a
 single port.
 As a general rule, the WWNN should be the same across all ports on the
 system.
 .It Fl W Ar wwpn
 Set the World Wide Port Name for the given port.
 The
 .Fl n
 argument must be specified, since this is only possible to implement on a
 single port.
 As a general rule, the WWPN must be different for every port in the system.
 .El
 .It Ic portlist
 List CTL frontend ports.
 .Bl -tag -width 12n
 .It Fl f Ar frontend
 Specify the frontend type.
 .It Fl i
 Report target and connected initiators addresses.
 .It Fl l
 Report LUN mapping.
 .It Fl p Ar targ_port
 Specify the frontend port number.
 .It Fl q
 Omit the header in the port list output.
 .It Fl v
 Enable verbose output (report all port options).
 .It Fl x
 Output the port list in XML format.
 .El
 .It Ic lunmap
 Change LUN mapping for specified port.
 If both
 .Ar pLUN
 and
 .Ar cLUN
 are specified -- LUN will be mapped.
 If
 .Ar pLUN
 is specified, but
 .Ar cLUN
 is not -- LUN will be unmapped.
 If neither
 .Ar pLUN
 nor
 .Ar cLUN
 are specified -- LUN mapping will be disabled, exposing all CTL LUNs.
 .Bl -tag -width 12n
 .It Fl p Ar targ_port
 Specify the frontend port number.
 .It Fl l Ar pLUN
 LUN number visible by specified port.
 .It Fl L Ar cLUN
 CTL LUN number.
 .El
 .It Ic dumpooa
 Dump the OOA (Order Of Arrival) queue for each LUN registered with CTL.
 .It Ic dumpstructs
 Dump the CTL structures to the console.
 .It Ic create
 Create a new LUN.
 The backend must be specified, and depending upon the backend requested,
 some of the other options may be required.
 If the LUN is created successfully, the LUN configuration will be
 displayed.
 If LUN creation fails, a message will be displayed describing the failure.
 .Bl -tag -width 14n
 .It Fl b Ar backend
 The
 .Fl b
 flag is required.
 This specifies the name backend to use when creating the LUN.
 Examples are
 .Dq ramdisk
 and
 .Dq block .
 .It Fl B Ar blocksize
 Specify the blocksize of the backend in bytes.
 .It Fl d Ar device_id
 Specify the LUN-associated string to use in the
 .Tn SCSI
 INQUIRY VPD page 0x83 data.
 .It Fl l Ar lun_id
 Request that a particular LUN number be assigned.
 If the requested LUN number is not available, the request will fail.
 .It Fl o Ar name=value
 Specify a backend-specific name/value pair.
 Multiple
 .Fl o
 arguments may be specified.
 Refer to the backend documentation for arguments that may be used.
 .It Fl s Ar size_bytes
 Specify the size of the LUN in bytes.
 Some backends may allow setting the size (e.g. the ramdisk backend) and for
 others the size may be implicit (e.g. the block backend).
 .It Fl S Ar serial_num
 Specify the serial number to be used in the
 .Tn SCSI
 INQUIRY VPD page 0x80 data.
 .It Fl t Ar device_type
 Specify the numeric SCSI device type to use when creating the LUN.
 If this flag is not used, the type of LUN created is backend-specific.
 Not all LUN types are supported.
 Currently CTL supports Direct Access (type 0), Processor (type 3)
 and CD/DVD (type 5) LUNs.
 The backend requested may or may not support all of the LUN types that CTL
 supports.
 .El
 .It Ic remove
 Remove a LUN.
 The backend must be specified, and the LUN number must also be specified.
 Backend-specific options may also be specified with the
 .Fl o
 flag.
 .Bl -tag -width 14n
 .It Fl b Ar backend
 Specify the backend that owns the LUN to be removed.
 Examples are
 .Dq ramdisk
 and
 .Dq block .
 .It Fl l Ar lun_id
 Specify the LUN number to remove.
 .It Fl o Ar name=value
 Specify a backend-specific name/value pair.
 Multiple
 .Fl o
 arguments may be specified.
 Refer to the backend documentation for arguments that may be used.
 .El
 .It Ic modify
 Modify a LUN size.
 The backend, the LUN number, and the size must be specified.
 .Bl -tag -width 14n
 .It Fl b Ar backend
 Specify the backend that owns the LUN to be modified.
 Examples are
 .Dq ramdisk
 and
 .Dq block .
 .It Fl l Ar lun_id
 Specify the LUN number to modify.
 .It Fl o Ar name=value
 Specify a backend-specific name/value pair.
 Multiple
 .Fl o
 arguments may be specified.
 Refer to the backend documentation for arguments that may be used.
 .It Fl s Ar size_bytes
 Specify the size of the LUN in bytes.
 For the
 .Dq block
 backend, an
 .Dq auto
 keyword may be passed instead; this will make CTL use the size of backing
 file or device.
 .El
 .It Ic devlist
 Get a list of all configured LUNs.
 This also includes the LUN size and blocksize, serial number and device ID.
 .Bl -tag -width 11n
 .It Fl b Ar backend
 Specify the backend.
 This restricts the LUN list to the named backend.
 Examples are
 .Dq ramdisk
 and
 .Dq block .
 .It Fl v
 Be verbose.
 This will also display any backend-specific LUN attributes in addition to
 the standard per-LUN information.
 .It Fl x
 Dump the raw XML.
 The LUN list information from the kernel comes in XML format, and this
 option allows the display of the raw XML data.
 This option and the
 .Fl v
 and
 .Fl b
 options are mutually exclusive.
 If you specify
 .Fl x ,
 the entire LUN database is displayed in XML format.
 .El
 .It Ic islist
 Get a list of currently running iSCSI sessions.
 This includes initiator and target names and the unique connection IDs.
 .Bl -tag -width 11n
 .It Fl v
 Verbose mode.
 .It Fl x
 Dump the raw XML.
 The sessions list information from the kernel comes in XML format, and this
 option allows the display of the raw XML data.
 .El
 .It Ic islogout
 Ask the initiator to log out iSCSI sessions matching criteria.
 .Bl -tag -width 11n
 .It Fl a
 Log out all sessions.
 .It Fl c
 Specify connection ID.
 .It Fl i
 Specify initiator name.
 .It Fl p
 Specify initiator portal (hostname or IP address).
 .El
 .It Ic isterminate
 Forcibly terminate iSCSI sessions matching criteria.
 .Bl -tag -width 11n
 .It Fl a
 Terminate all sessions.
 .It Fl c
 Specify connection ID.
 .It Fl i
 Specify initiator name.
 .It Fl p
 Specify initiator portal (hostname or IP address).
 .El
+.It Ic nvlist
+Get a list of currently running NVMeoF associations.
+This includes host and controller names and the unique controller IDs.
+.Bl -tag -width 11n
+.It Fl v
+Verbose mode.
+.It Fl x
+Dump the raw XML.
+The sessions list information from the kernel comes in XML format, and this
+option allows the display of the raw XML data.
+.El
 .It Ic help
 Display
 .Nm
 usage information.
 .El
 .Sh OPTIONS
 Number of additional configuration options may be specified for LUNs.
 Some options are global, others are backend-specific.
 .Pp
 Global options:
 .Bl -tag -width 12n
 .It Va vendor
 Specifies LUN vendor string up to 8 chars.
 .It Va product
 Specifies LUN product string up to 16 chars.
 .It Va revision
 Specifies LUN revision string up to 4 chars.
 .It Va scsiname
 Specifies LUN SCSI name string.
 .It Va eui
 Specifies LUN EUI-64 identifier.
 .It Va naa
 Specifies LUN NAA identifier.
 .It Va uuid
 Specifies LUN locally assigned RFC 4122 UUID identifier.
 EUI, NAA or UUID identifier should be set to UNIQUE value to allow
 EXTENDED COPY command access the LUN.
 Non-unique LUN identifiers may lead to data corruption.
 Some initiators may not support later introduced UUID identifiers.
 .It Va ident_info
 Specified LUN identification information (string or 0x + hex).
 .It Va text_ident_info
 Specified LUN text identification information (UTF-8 string).
 .It Va ha_role
 Setting to "primary" or "secondary" overrides default role of the node
 in HA cluster, set by kern.cam.ctl.ha_role sysctl.
 .It Va insecure_tpc
 Setting to "on" allows EXTENDED COPY command sent to this LUN access
 other LUNs on this host, not accessible otherwise.
 This allows to offload copying between different iSCSI targets residing
 on the same host in trusted environments.
 .It Va readcache
 Set to "off", disables read caching for the LUN, if supported by the backend.
 .It Va readonly
 Set to "on", blocks all media write operations to the LUN, reporting it
 as write protected.
 .It Va removable
 Set to "on", makes LUN removable.
 .It Va reordering
 Set to "unrestricted", allows target to process commands with SIMPLE task
 attribute in arbitrary order.
 Any data integrity exposures related to command sequence order shall be
 explicitly handled by the application client through the selection of
 appropriate commands and task attributes.
 The default value is "restricted".
 It improves data integrity, but may introduce some additional delays.
 .It Va serseq
 Set to "on" to fully serialize consecutive reads/writes.
 Set to "read" to fully serialize consecutive reads.
 Set to "soft" to slightly serialize consecutive reads.
 Set to "off" to allow them be issued in parallel.
 Parallel issue of consecutive operations may confuse logic of the
 backing file system, hurting performance; but it may improve performance
 of backing stores without prefetch/write-back.
 .It Va pblocksize
 .It Va pblockoffset
 Specify physical block size and offset of the device.
 .It Va ublocksize
 .It Va ublockoffset
 Specify UNMAP block size and offset of the device.
 .It Va rpm
 Specifies medium rotation rate of the device: 0 -- not reported,
 1 -- non-rotating (SSD), >1024 -- value in revolutions per minute.
 .It Va formfactor
 Specifies nominal form factor of the device: 0 -- not reported, 1 -- 5.25",
 2 -- 3.5", 3 -- 2.5", 4 -- 1.8", 5 -- less then 1.8".
 .It Va temperature
 .It Va reftemperature
 Specify current and reference (maximum) temperatures of the device.
 .It Va provisioning_type
 When UNMAP support is enabled, this option specifies provisioning type:
 "resource", "thin" or "unknown".
 Default value is "thin".
 Logical units without UNMAP support are reported as fully provisioned.
 .It Va unmap
 Setting to "on" or "off" controls UNMAP support for the logical unit.
 Default value is "on" if supported by the backend.
 .It Va unmap_max_lba
 .It Va unmap_max_descr
 Specify maximum allowed number of LBAs and block descriptors per UNMAP
 command to report in Block Limits VPD page.
 .It Va write_same_max_lba
 Specify maximum allowed number of LBAs per WRITE SAME command to report
 in Block Limits VPD page.
 .It Va avail-threshold
 .It Va used-threshold
 .It Va pool-avail-threshold
 .It Va pool-used-threshold
 Set per-LUN/-pool thin provisioning soft thresholds.
 LUN will establish UNIT ATTENTION condition if its or pool available space
 get below configured avail values, or its or pool used space get above
 configured used values.
 Pool thresholds are working only for ZVOL-backed LUNs.
 .It Va writecache
 Set to "off", disables write caching for the LUN, if supported by the backend.
 .El
 .Pp
 Options specific for block backend:
 .Bl -tag -width 12n
 .It Va file
 Specifies file or device name to use for backing store.
 .It Va num_threads
 Specifies number of backend threads to use for this LUN.
 .El
 .Pp
 Options specific for ramdisk backend:
 .Bl -tag -width 12n
 .It Va capacity
 Specifies capacity of backing store (maximum RAM for data).
 The default value is zero, that disables backing store completely,
 making all writes go to nowhere, while all reads return zeroes.
 .El
 .Sh EXAMPLES
 .Pp
 Send a
 .Tn SCSI
 TEST UNIT READY command to LUN 1.
 .Pp
 .Dl ctladm tur 1
 .Pp
 Display the list of mode pages supported by LUN 1.
 .Pp
 .Dl ctladm modesense 1 -l
 .Pp
 Display the saved version of the Control mode page (page 10) on LUN 0.
 Disable fetching block descriptors, and use a 10 byte MODE SENSE command
 instead of the default 6 byte command.
 .Pp
 .Dl ctladm modesense 0 -m 10 -P 3 -d -c 10
 .Pp
 Read the first 512 byte block from LUN 2 and dump it to the file
 .Bd -literal
 .Dl ctladm read 2 -l 0 -d 1 -b 512 -f - > foo
 .Ed
 .Pp
 Read 10240 bytes from the file
 .Pa /tmp/bar
 and write it to LUN 3.
 starting at LBA 0xff432140.
 .Pp
 .Bd -literal
 .Dl ctladm write 3 -l 0xff432140 -d 20 -b 512 -f /tmp/bar
 .Ed
 .Pp
 Create a LUN with the
 .Dq fake
 ramdisk as a backing store.
 The LUN will claim to have a size of approximately 10 terabytes,
 while having no real data store (all written data are lost).
 .Pp
 .Dl ctladm create -b ramdisk -s 10485760000000000
 .Pp
 Create a thin provisioned LUN with a ramdisk as a backing store.
 The LUN will have maximal backing store capacity of 10 gigabytes,
 while reporting size of 10 terabytes,
 .Pp
 .Dl ctladm create -b ramdisk -s 10T -o capacity=10G
 .Pp
 Create a LUN using the block backend, specify the ZFS volume
 .Pa tank/example
 as the backing store, and specify the
 .Tn SCSI
 VPD page 0x80 and 0x83 serial number
 .Fl ( S )
 and device ID
 .Fl ( d ) .
 The size of the LUN will be derived from the size of the ZVOL.
 .Pp
 .Dl ctladm create -b block -o file=/dev/zvol/tank/example -S MYSERIAL321 -d MYDEVID123
 .Pp
 Use to specify generic options on ioctl frontend port, now it is
 only possible to set pp and/or vp port number.
 .Pp
 .Dl ctladm port -c -O pp=11 -O vp=12
 .Pp
 Remove specified targ_port.
 .Pp
 .Dl ctladm port -r -p 4
 .Pp
 .Pp
 Remove LUN 12, which is handled by the block backend, from the system.
 .Pp
 .Dl ctladm remove -b block -l 12
 .Pp
 List configured LUNs in the system, along with their backend and serial
 number.
 This works when the Front End Target Drivers are enabled or disabled.
 .Pp
 .Dl ctladm devlist
 .Pp
 List all LUNs in the system, along with their inquiry data and device type.
 This only works when the FETDs are enabled, since the commands go through the
 ioctl port.
 .Pp
 .Dl ctladm lunlist
 .Pp
 Inject a medium error on LUN 6 for every read that covers the first 512
 blocks of the LUN.
 .Pp
 .Dl ctladm inject 6 -i mediumerr -p read -r 0,512 -c
 .Pp
 Inject a custom error on LUN 6 for the next TEST UNIT READY command only.
 This will result in a sense key of NOT READY (0x02), and an ASC/ASCQ of
 0x04,0x02 ("Logical unit not ready, initializing command required").
 .Pp
 .Bd -literal -offset indent
 ctladm inject 6 -i custom -p tur -s 18 "f0 0 02 s12 04 02"
 .Ed
 .Sh SEE ALSO
 .Xr cam 3 ,
 .Xr cam_cdbparse 3 ,
 .Xr cam 4 ,
 .Xr ctl 4 ,
 .Xr xpt 4 ,
 .Xr camcontrol 8 ,
 .Xr ctld 8 ,
 .Xr ctlstat 8
 .Sh HISTORY
 The
 .Nm
 utility was originally written during the Winter/Spring of 2003 as an
 interface to CTL.
 .Sh AUTHORS
 .An Ken Merry Aq Mt ken@FreeBSD.org
diff --git a/usr.sbin/ctladm/ctladm.c b/usr.sbin/ctladm/ctladm.c
index 36cedf548ae5..d0456986fd12 100644
--- a/usr.sbin/ctladm/ctladm.c
+++ b/usr.sbin/ctladm/ctladm.c
@@ -1,4278 +1,4530 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause
  *
  * Copyright (c) 2003, 2004 Silicon Graphics International Corp.
  * Copyright (c) 1997-2007 Kenneth D. Merry
  * Copyright (c) 2012 The FreeBSD Foundation
  * Copyright (c) 2018 Marcelo Araujo <araujo@FreeBSD.org>
  * All rights reserved.
  *
  * Portions of this software were developed by Edward Tomasz Napierala
  * under sponsorship from the FreeBSD Foundation.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions, and the following disclaimer,
  *    without modification.
  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  *    substantially similar to the "NO WARRANTY" disclaimer below
  *    ("Disclaimer") and any redistribution must be conditioned upon
  *    including a substantially similar Disclaimer requirement for further
  *    binary redistribution.
  *
  * NO WARRANTY
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGES.
  *
  * $Id: //depot/users/kenm/FreeBSD-test2/usr.sbin/ctladm/ctladm.c#4 $
  */
 /*
  * CAM Target Layer exercise program.
  *
  * Author: Ken Merry <ken@FreeBSD.org>
  */
 
 #include <sys/param.h>
 #include <sys/callout.h>
 #include <sys/ioctl.h>
 #include <sys/linker.h>
 #include <sys/module.h>
 #include <sys/queue.h>
 #include <sys/sbuf.h>
 #include <sys/nv.h>
 #include <sys/stat.h>
 #include <bsdxml.h>
 #include <ctype.h>
 #include <err.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <getopt.h>
 #include <stdlib.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
 #include <cam/scsi/scsi_all.h>
 #include <cam/scsi/scsi_message.h>
 #include <cam/ctl/ctl.h>
 #include <cam/ctl/ctl_io.h>
 #include <cam/ctl/ctl_backend.h>
 #include <cam/ctl/ctl_ioctl.h>
 #include <cam/ctl/ctl_util.h>
 #include <cam/ctl/ctl_scsi_all.h>
+#include <dev/nvmf/nvmf_proto.h>
 #include <camlib.h>
 #include <libutil.h>
 #include "ctladm.h"
 
 #ifdef min
 #undef min
 #endif
 #define min(x,y) (x < y) ? x : y
 
 typedef enum {
 	CTLADM_CMD_TUR,
 	CTLADM_CMD_INQUIRY,
 	CTLADM_CMD_REQ_SENSE,
 	CTLADM_CMD_ARRAYLIST,
 	CTLADM_CMD_REPORT_LUNS,
 	CTLADM_CMD_HELP,
 	CTLADM_CMD_DEVLIST,
 	CTLADM_CMD_ADDDEV,
 	CTLADM_CMD_RM,
 	CTLADM_CMD_CREATE,
 	CTLADM_CMD_READ,
 	CTLADM_CMD_WRITE,
 	CTLADM_CMD_PORT,
 	CTLADM_CMD_PORTLIST,
 	CTLADM_CMD_READCAPACITY,
 	CTLADM_CMD_MODESENSE,
 	CTLADM_CMD_DUMPOOA,
 	CTLADM_CMD_DUMPSTRUCTS,
 	CTLADM_CMD_START,
 	CTLADM_CMD_STOP,
 	CTLADM_CMD_SYNC_CACHE,
 	CTLADM_CMD_LUNLIST,
 	CTLADM_CMD_DELAY,
 	CTLADM_CMD_ERR_INJECT,
 	CTLADM_CMD_PRES_IN,
 	CTLADM_CMD_PRES_OUT,
 	CTLADM_CMD_INQ_VPD_DEVID,
 	CTLADM_CMD_RTPG,
 	CTLADM_CMD_MODIFY,
 	CTLADM_CMD_ISLIST,
 	CTLADM_CMD_ISLOGOUT,
 	CTLADM_CMD_ISTERMINATE,
-	CTLADM_CMD_LUNMAP
+	CTLADM_CMD_LUNMAP,
+	CTLADM_CMD_NVLIST
 } ctladm_cmdfunction;
 
 typedef enum {
 	CTLADM_ARG_NONE		= 0x0000000,
 	CTLADM_ARG_AUTOSENSE	= 0x0000001,
 	CTLADM_ARG_DEVICE	= 0x0000002,
 	CTLADM_ARG_ARRAYSIZE	= 0x0000004,
 	CTLADM_ARG_BACKEND	= 0x0000008,
 	CTLADM_ARG_CDBSIZE	= 0x0000010,
 	CTLADM_ARG_DATALEN	= 0x0000020,
 	CTLADM_ARG_FILENAME	= 0x0000040,
 	CTLADM_ARG_LBA		= 0x0000080,
 	CTLADM_ARG_PC		= 0x0000100,
 	CTLADM_ARG_PAGE_CODE	= 0x0000200,
 	CTLADM_ARG_PAGE_LIST	= 0x0000400,
 	CTLADM_ARG_SUBPAGE	= 0x0000800,
 	CTLADM_ARG_PAGELIST	= 0x0001000,
 	CTLADM_ARG_DBD		= 0x0002000,
 	CTLADM_ARG_TARG_LUN	= 0x0004000,
 	CTLADM_ARG_BLOCKSIZE	= 0x0008000,
 	CTLADM_ARG_IMMED	= 0x0010000,
 	CTLADM_ARG_RELADR	= 0x0020000,
 	CTLADM_ARG_RETRIES	= 0x0040000,
 	CTLADM_ARG_ONOFFLINE	= 0x0080000,
 	CTLADM_ARG_ONESHOT	= 0x0100000,
 	CTLADM_ARG_TIMEOUT	= 0x0200000,
 	CTLADM_ARG_INITIATOR	= 0x0400000,
 	CTLADM_ARG_NOCOPY	= 0x0800000,
 	CTLADM_ARG_NEED_TL	= 0x1000000
 } ctladm_cmdargs;
 
 struct ctladm_opts {
 	const char	*optname;
 	uint32_t	cmdnum;
 	ctladm_cmdargs	argnum;
 	const char	*subopt;
 };
 
 typedef enum {
 	CC_OR_NOT_FOUND,
 	CC_OR_AMBIGUOUS,
 	CC_OR_FOUND
 } ctladm_optret;
 
 static const char rw_opts[] = "Nb:c:d:f:l:";
 static const char startstop_opts[] = "i";
 
 static struct ctladm_opts option_table[] = {
 	{"adddev", CTLADM_CMD_ADDDEV, CTLADM_ARG_NONE, NULL},
 	{"create", CTLADM_CMD_CREATE, CTLADM_ARG_NONE, "b:B:d:l:o:s:S:t:"},
 	{"delay", CTLADM_CMD_DELAY, CTLADM_ARG_NEED_TL, "T:l:t:"},
 	{"devid", CTLADM_CMD_INQ_VPD_DEVID, CTLADM_ARG_NEED_TL, NULL},
 	{"devlist", CTLADM_CMD_DEVLIST, CTLADM_ARG_NONE, "b:vx"},
 	{"dumpooa", CTLADM_CMD_DUMPOOA, CTLADM_ARG_NONE, NULL},
 	{"dumpstructs", CTLADM_CMD_DUMPSTRUCTS, CTLADM_ARG_NONE, NULL},
 	{"help", CTLADM_CMD_HELP, CTLADM_ARG_NONE, NULL},
 	{"inject", CTLADM_CMD_ERR_INJECT, CTLADM_ARG_NEED_TL, "cd:i:p:r:s:"},
 	{"inquiry", CTLADM_CMD_INQUIRY, CTLADM_ARG_NEED_TL, NULL},
 	{"islist", CTLADM_CMD_ISLIST, CTLADM_ARG_NONE, "vx"},
 	{"islogout", CTLADM_CMD_ISLOGOUT, CTLADM_ARG_NONE, "ac:i:p:"},
 	{"isterminate", CTLADM_CMD_ISTERMINATE, CTLADM_ARG_NONE, "ac:i:p:"},
 	{"lunlist", CTLADM_CMD_LUNLIST, CTLADM_ARG_NONE, NULL},
 	{"lunmap", CTLADM_CMD_LUNMAP, CTLADM_ARG_NONE, "p:l:L:"},
 	{"modesense", CTLADM_CMD_MODESENSE, CTLADM_ARG_NEED_TL, "P:S:dlm:c:"},
 	{"modify", CTLADM_CMD_MODIFY, CTLADM_ARG_NONE, "b:l:o:s:"},
+	{"nvlist", CTLADM_CMD_NVLIST, CTLADM_ARG_NONE, "vx"},
 	{"port", CTLADM_CMD_PORT, CTLADM_ARG_NONE, "lo:O:d:crp:qt:w:W:x"},
 	{"portlist", CTLADM_CMD_PORTLIST, CTLADM_ARG_NONE, "f:ilp:qvx"},
 	{"prin", CTLADM_CMD_PRES_IN, CTLADM_ARG_NEED_TL, "a:"},
 	{"prout", CTLADM_CMD_PRES_OUT, CTLADM_ARG_NEED_TL, "a:k:r:s:"},
 	{"read", CTLADM_CMD_READ, CTLADM_ARG_NEED_TL, rw_opts},
 	{"readcapacity", CTLADM_CMD_READCAPACITY, CTLADM_ARG_NEED_TL, "c:"},
 	{"remove", CTLADM_CMD_RM, CTLADM_ARG_NONE, "b:l:o:"},
 	{"reportluns", CTLADM_CMD_REPORT_LUNS, CTLADM_ARG_NEED_TL, NULL},
 	{"reqsense", CTLADM_CMD_REQ_SENSE, CTLADM_ARG_NEED_TL, NULL},
 	{"rtpg", CTLADM_CMD_RTPG, CTLADM_ARG_NEED_TL, NULL},
 	{"start", CTLADM_CMD_START, CTLADM_ARG_NEED_TL, startstop_opts},
 	{"stop", CTLADM_CMD_STOP, CTLADM_ARG_NEED_TL, startstop_opts},
 	{"synccache", CTLADM_CMD_SYNC_CACHE, CTLADM_ARG_NEED_TL, "b:c:il:r"},
 	{"tur", CTLADM_CMD_TUR, CTLADM_ARG_NEED_TL, NULL},
 	{"write", CTLADM_CMD_WRITE, CTLADM_ARG_NEED_TL, rw_opts},
 	{"-?", CTLADM_CMD_HELP, CTLADM_ARG_NONE, NULL},
 	{"-h", CTLADM_CMD_HELP, CTLADM_ARG_NONE, NULL},
 	{NULL, 0, 0, NULL}
 };
 
 
 ctladm_optret getoption(struct ctladm_opts *table, char *arg, uint32_t *cmdnum,
 			ctladm_cmdargs *argnum, const char **subopt);
 static int cctl_dump_ooa(int fd, int argc, char **argv);
 static int cctl_port(int fd, int argc, char **argv, char *combinedopt);
 static int cctl_do_io(int fd, int retries, union ctl_io *io, const char *func);
 static int cctl_delay(int fd, int lun, int argc, char **argv,
 		      char *combinedopt);
 static int cctl_lunlist(int fd);
 static int cctl_sync_cache(int fd, int lun, int iid, int retries,
 			   int argc, char **argv, char *combinedopt);
 static int cctl_start_stop(int fd, int lun, int iid, int retries,
 			   int start, int argc, char **argv, char *combinedopt);
 static int cctl_mode_sense(int fd, int lun, int iid, int retries,
 			   int argc, char **argv, char *combinedopt);
 static int cctl_read_capacity(int fd, int lun, int iid,
 			      int retries, int argc, char **argv,
 			      char *combinedopt);
 static int cctl_read_write(int fd, int lun, int iid, int retries,
 			   int argc, char **argv, char *combinedopt,
 			   ctladm_cmdfunction command);
 static int cctl_get_luns(int fd, int lun, int iid, int retries,
 			 struct scsi_report_luns_data **lun_data,
 			 uint32_t *num_luns);
 static int cctl_report_luns(int fd, int lun, int iid, int retries);
 static int cctl_tur(int fd, int lun, int iid, int retries);
 static int cctl_get_inquiry(int fd, int lun, int iid, int retries,
 			    char *path_str, int path_len,
 			    struct scsi_inquiry_data *inq_data);
 static int cctl_inquiry(int fd, int lun, int iid, int retries);
 static int cctl_req_sense(int fd, int lun, int iid, int retries);
 static int cctl_persistent_reserve_in(int fd, int lun,
 				      int initiator, int argc, char **argv,
 				      char *combinedopt, int retry_count);
 static int cctl_persistent_reserve_out(int fd, int lun,
 				       int initiator, int argc, char **argv,
 				       char *combinedopt, int retry_count);
 static int cctl_create_lun(int fd, int argc, char **argv, char *combinedopt);
 static int cctl_inquiry_vpd_devid(int fd, int lun, int initiator);
 static int cctl_report_target_port_group(int fd, int lun, int initiator);
 static int cctl_modify_lun(int fd, int argc, char **argv, char *combinedopt);
 static int cctl_portlist(int fd, int argc, char **argv, char *combinedopt);
 
 ctladm_optret
 getoption(struct ctladm_opts *table, char *arg, uint32_t *cmdnum,
 	  ctladm_cmdargs *argnum, const char **subopt)
 {
 	struct ctladm_opts *opts;
 	int num_matches = 0;
 
 	for (opts = table; (opts != NULL) && (opts->optname != NULL);
 	     opts++) {
 		if (strncmp(opts->optname, arg, strlen(arg)) == 0) {
 			*cmdnum = opts->cmdnum;
 			*argnum = opts->argnum;
 			*subopt = opts->subopt;
 
 			if (strcmp(opts->optname, arg) == 0)
 				return (CC_OR_FOUND);
 
 			if (++num_matches > 1)
 				return(CC_OR_AMBIGUOUS);
 		}
 	}
 
 	if (num_matches > 0)
 		return(CC_OR_FOUND);
 	else
 		return(CC_OR_NOT_FOUND);
 }
 
 static int
 cctl_dump_ooa(int fd, int argc, char **argv)
 {
 	struct ctl_ooa ooa;
 	long double cmd_latency;
 	int num_entries, len, lun = -1, retval = 0;
 	unsigned int i;
 
 	num_entries = 104;
 
 	if ((argc > 2) && (isdigit(argv[2][0])))
 		lun = strtol(argv[2], NULL, 0);
 retry:
 
 	len = num_entries * sizeof(struct ctl_ooa_entry);
 	bzero(&ooa, sizeof(ooa));
 	ooa.entries = malloc(len);
 	if (ooa.entries == NULL) {
 		warn("%s: error mallocing %d bytes", __func__, len);
 		return (1);
 	}
 	if (lun >= 0) {
 		ooa.lun_num = lun;
 	} else
 		ooa.flags |= CTL_OOA_FLAG_ALL_LUNS;
 	ooa.alloc_len = len;
 	ooa.alloc_num = num_entries;
 	if (ioctl(fd, CTL_GET_OOA, &ooa) == -1) {
 		warn("%s: CTL_GET_OOA ioctl failed", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	if (ooa.status == CTL_OOA_NEED_MORE_SPACE) {
 		num_entries = num_entries * 2;
 		free(ooa.entries);
 		ooa.entries = NULL;
 		goto retry;
 	}
 
 	if (ooa.status != CTL_OOA_OK) {
 		warnx("%s: CTL_GET_OOA ioctl returned error %d", __func__,
 		      ooa.status);
 		retval = 1;
 		goto bailout;
 	}
 
 	fprintf(stdout, "Dumping OOA queues\n");
 	for (i = 0; i < ooa.fill_num; i++) {
 		struct ctl_ooa_entry *entry;
 		char cdb_str[(SCSI_MAX_CDBLEN * 3) +1];
 		struct bintime delta_bt;
 		struct timespec ts;
 
 		entry = &ooa.entries[i];
 
 		delta_bt = ooa.cur_bt;
 		bintime_sub(&delta_bt, &entry->start_bt);
 		bintime2timespec(&delta_bt, &ts);
 		cmd_latency = ts.tv_sec * 1000;
 		if (ts.tv_nsec > 0)
 			cmd_latency += ts.tv_nsec / 1000000;
 
 		fprintf(stdout, "LUN %jd tag 0x%jx%s%s%s%s%s%s%s: %s. CDB: %s "
 			"(%0.0Lf ms)\n",
 			(intmax_t)entry->lun_num, entry->tag_num,
 			(entry->cmd_flags & CTL_OOACMD_FLAG_BLOCKED) ?
 			 " BLOCKED" : "",
 			(entry->cmd_flags & CTL_OOACMD_FLAG_RTR) ? " RTR" :"",
 			(entry->cmd_flags & CTL_OOACMD_FLAG_DMA_QUEUED) ?
 			 " DMAQUEUED" : "",
 			(entry->cmd_flags & CTL_OOACMD_FLAG_DMA) ? " DMA" : "",
 			(entry->cmd_flags & CTL_OOACMD_FLAG_STATUS_QUEUED) ?
 			 " STATUSQUEUED" : "",
 			(entry->cmd_flags & CTL_OOACMD_FLAG_STATUS_SENT) ? " STATUS" : "",
 			(entry->cmd_flags & CTL_OOACMD_FLAG_ABORT) ?
 			 " ABORT" : "",
 			scsi_op_desc(entry->cdb[0], NULL),
 			scsi_cdb_string(entry->cdb, cdb_str, sizeof(cdb_str)),
 			cmd_latency);
 	}
 	fprintf(stdout, "OOA queues dump done\n");
 
 bailout:
 	free(ooa.entries);
 	return (retval);
 }
 
 static int
 cctl_dump_structs(int fd, ctladm_cmdargs cmdargs __unused)
 {
 	if (ioctl(fd, CTL_DUMP_STRUCTS) == -1) {
 		warn(__func__);
 		return (1);
 	}
 	return (0);
 }
 
 typedef enum {
 	CCTL_PORT_MODE_NONE,
 	CCTL_PORT_MODE_LIST,
 	CCTL_PORT_MODE_SET,
 	CCTL_PORT_MODE_ON,
 	CCTL_PORT_MODE_OFF,
 	CCTL_PORT_MODE_CREATE,
 	CCTL_PORT_MODE_REMOVE
 } cctl_port_mode;
 
 static struct ctladm_opts cctl_fe_table[] = {
 	{"fc", CTL_PORT_FC, CTLADM_ARG_NONE, NULL},
 	{"scsi", CTL_PORT_SCSI, CTLADM_ARG_NONE, NULL},
 	{"internal", CTL_PORT_INTERNAL, CTLADM_ARG_NONE, NULL},
 	{"iscsi", CTL_PORT_ISCSI, CTLADM_ARG_NONE, NULL},
 	{"nvmf", CTL_PORT_NVMF, CTLADM_ARG_NONE, NULL},
 	{"sas", CTL_PORT_SAS, CTLADM_ARG_NONE, NULL},
 	{"all", CTL_PORT_ALL, CTLADM_ARG_NONE, NULL},
 	{NULL, 0, 0, NULL}
 };
 
 static int
 cctl_port(int fd, int argc, char **argv, char *combinedopt)
 {
 	int c;
 	int32_t targ_port = -1;
 	int retval = 0;
 	int wwnn_set = 0, wwpn_set = 0;
 	uint64_t wwnn = 0, wwpn = 0;
 	cctl_port_mode port_mode = CCTL_PORT_MODE_NONE;
 	struct ctl_port_entry entry;
 	struct ctl_req req;
 	char *driver = NULL;
 	nvlist_t *option_list;
 	ctl_port_type port_type = CTL_PORT_NONE;
 	int quiet = 0, xml = 0;
 
 	option_list = nvlist_create(0);
 	if (option_list == NULL)
 		err(1, "%s: unable to allocate nvlist", __func__);
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'l':
 			if (port_mode != CCTL_PORT_MODE_NONE)
 				goto bailout_badarg;
 
 			port_mode = CCTL_PORT_MODE_LIST;
 			break;
 		case 'c':
 			port_mode = CCTL_PORT_MODE_CREATE;
 			break;
 		case 'r':
 			port_mode = CCTL_PORT_MODE_REMOVE;
 			break;
 		case 'o':
 			if (port_mode != CCTL_PORT_MODE_NONE)
 				goto bailout_badarg;
 
 			if (strcasecmp(optarg, "on") == 0)
 				port_mode = CCTL_PORT_MODE_ON;
 			else if (strcasecmp(optarg, "off") == 0)
 				port_mode = CCTL_PORT_MODE_OFF;
 			else {
 				warnx("Invalid -o argument %s, \"on\" or "
 				      "\"off\" are the only valid args",
 				      optarg);
 				retval = 1;
 				goto bailout;
 			}
 			break;
 		case 'O': {
 			char *tmpstr;
 			char *name, *value;
 
 			tmpstr = strdup(optarg);
 			name = strsep(&tmpstr, "=");
 			if (name == NULL) {
 				warnx("%s: option -O takes \"name=value\""
 				      "argument", __func__);
 				retval = 1;
 				goto bailout;
 			}
 			value = strsep(&tmpstr, "=");
 			if (value == NULL) {
 				warnx("%s: option -O takes \"name=value\""
 				      "argument", __func__);
 				retval = 1;
 				goto bailout;
 			}
 
 			free(tmpstr);
 			nvlist_add_string(option_list, name, value);
 			break;
 		}
 		case 'd':
 			if (driver != NULL) {
 				warnx("%s: option -d cannot be specified twice",
 				    __func__);
 				retval = 1;
 				goto bailout;
 			}
 
 			driver = strdup(optarg);
 			break;
 		case 'p':
 			targ_port = strtol(optarg, NULL, 0);
 			break;
 		case 'q':
 			quiet = 1;
 			break;
 		case 't': {
 			ctladm_optret optret;
 			ctladm_cmdargs argnum;
 			const char *subopt;
 			ctl_port_type tmp_port_type;
 
 			optret = getoption(cctl_fe_table, optarg, &tmp_port_type,
 					   &argnum, &subopt);
 			if (optret == CC_OR_AMBIGUOUS) {
 				warnx("%s: ambiguous frontend type %s",
 				      __func__, optarg);
 				retval = 1;
 				goto bailout;
 			} else if (optret == CC_OR_NOT_FOUND) {
 				warnx("%s: invalid frontend type %s",
 				      __func__, optarg);
 				retval = 1;
 				goto bailout;
 			}
 
 			port_type |= tmp_port_type;
 			break;
 		}
 		case 'w':
 			if ((port_mode != CCTL_PORT_MODE_NONE)
 			 && (port_mode != CCTL_PORT_MODE_SET))
 				goto bailout_badarg;
 
 			port_mode = CCTL_PORT_MODE_SET;
 
 			wwnn = strtoull(optarg, NULL, 0);
 			wwnn_set = 1;
 			break;
 		case 'W':
 			if ((port_mode != CCTL_PORT_MODE_NONE)
 			 && (port_mode != CCTL_PORT_MODE_SET))
 				goto bailout_badarg;
 
 			port_mode = CCTL_PORT_MODE_SET;
 
 			wwpn = strtoull(optarg, NULL, 0);
 			wwpn_set = 1;
 			break;
 		case 'x':
 			xml = 1;
 			break;
 		}
 	}
 
 	if (driver == NULL)
 		driver = strdup("ioctl");
 
 	/*
 	 * The user can specify either one or more frontend types (-t), or
 	 * a specific frontend, but not both.
 	 *
 	 * If the user didn't specify a frontend type or number, set it to
 	 * all.  This is primarily needed for the enable/disable ioctls.
 	 * This will be a no-op for the listing code.  For the set ioctl,
 	 * we'll throw an error, since that only works on one port at a time.
 	 */
 	if ((port_type != CTL_PORT_NONE) && (targ_port != -1)) {
 		warnx("%s: can only specify one of -t or -n", __func__);
 		retval = 1;
 		goto bailout;
 	} else if ((targ_port == -1) && (port_type == CTL_PORT_NONE))
 		port_type = CTL_PORT_ALL;
 
 	bzero(&entry, sizeof(entry));
 
 	/*
 	 * These are needed for all but list/dump mode.
 	 */
 	entry.port_type = port_type;
 	entry.targ_port = targ_port;
 
 	switch (port_mode) {
 	case CCTL_PORT_MODE_LIST: {
 		char opts[] = "xq";
 		char argx[] = "-x";
 		char argq[] = "-q";
 		char *argvx[2];
 		int argcx = 0;
 
 		optind = 0;
 		optreset = 1;
 		if (xml)
 			argvx[argcx++] = argx;
 		if (quiet)
 			argvx[argcx++] = argq;
 		cctl_portlist(fd, argcx, argvx, opts);
 		break;
 	}
 	case CCTL_PORT_MODE_REMOVE:
 		if (targ_port == -1) {
 			warnx("%s: -r requires -p", __func__);
 			retval = 1;
 			goto bailout;
 		}
 		/* FALLTHROUGH */
 	case CCTL_PORT_MODE_CREATE: {
 		bzero(&req, sizeof(req));
 		strlcpy(req.driver, driver, sizeof(req.driver));
 
 		if (port_mode == CCTL_PORT_MODE_REMOVE) {
 			req.reqtype = CTL_REQ_REMOVE;
 			nvlist_add_stringf(option_list, "port_id", "%d",
 			    targ_port);
 		} else
 			req.reqtype = CTL_REQ_CREATE;
 
 		req.args = nvlist_pack(option_list, &req.args_len);
 		if (req.args == NULL) {
 			warn("%s: error packing nvlist", __func__);
 			retval = 1;
 			goto bailout;
 		}
 
 		retval = ioctl(fd, CTL_PORT_REQ, &req);
 		free(req.args);
 		if (retval == -1) {
 			warn("%s: CTL_PORT_REQ ioctl failed", __func__);
 			retval = 1;
 			goto bailout;
 		}
 
 		switch (req.status) {
 		case CTL_LUN_ERROR:
 			warnx("error: %s", req.error_str);
 			retval = 1;
 			goto bailout;
 		case CTL_LUN_WARNING:
 			warnx("warning: %s", req.error_str);
 			break;
 		case CTL_LUN_OK:
 			break;
 		default:
 			warnx("unknown status: %d", req.status);
 			retval = 1;
 			goto bailout;
 		}
 
 		break;
 	}
 	case CCTL_PORT_MODE_SET:
 		if (targ_port == -1) {
 			warnx("%s: -w and -W require -n", __func__);
 			retval = 1;
 			goto bailout;
 		}
 
 		if (wwnn_set) {
 			entry.flags |= CTL_PORT_WWNN_VALID;
 			entry.wwnn = wwnn;
 		}
 		if (wwpn_set) {
 			entry.flags |= CTL_PORT_WWPN_VALID;
 			entry.wwpn = wwpn;
 		}
 
 		if (ioctl(fd, CTL_SET_PORT_WWNS, &entry) == -1) {
 			warn("%s: CTL_SET_PORT_WWNS ioctl failed", __func__);
 			retval = 1;
 			goto bailout;
 		}
 		break;
 	case CCTL_PORT_MODE_ON:
 		if (ioctl(fd, CTL_ENABLE_PORT, &entry) == -1) {
 			warn("%s: CTL_ENABLE_PORT ioctl failed", __func__);
 			retval = 1;
 			goto bailout;
 		}
 		fprintf(stdout, "Front End Ports enabled\n");
 		break;
 	case CCTL_PORT_MODE_OFF:
 		if (ioctl(fd, CTL_DISABLE_PORT, &entry) == -1) {
 			warn("%s: CTL_DISABLE_PORT ioctl failed", __func__);
 			retval = 1;
 			goto bailout;
 		}
 		fprintf(stdout, "Front End Ports disabled\n");
 		break;
 	default:
 		warnx("%s: one of -l, -o or -w/-W must be specified", __func__);
 		retval = 1;
 		goto bailout;
 		break;
 	}
 
 bailout:
 	nvlist_destroy(option_list);
 	free(driver);
 	return (retval);
 
 bailout_badarg:
 	warnx("%s: only one of -l, -o or -w/-W may be specified", __func__);
 	return (1);
 }
 
 static int
 cctl_do_io(int fd, int retries, union ctl_io *io, const char *func)
 {
 	do {
 		if (ioctl(fd, CTL_IO, io) == -1) {
 			warn("%s: error sending CTL_IO ioctl", func);
 			return (-1);
 		}
 	} while (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
 	      && (retries-- > 0));
 
 	return (0);
 }
 
 static int
 cctl_delay(int fd, int lun, int argc, char **argv,
 	   char *combinedopt)
 {
 	struct ctl_io_delay_info delay_info;
 	char *delayloc = NULL;
 	char *delaytype = NULL;
 	int delaytime = -1;
 	int retval;
 	int c;
 
 	retval = 0;
 
 	memset(&delay_info, 0, sizeof(delay_info));
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'T':
 			delaytype = strdup(optarg);
 			break;
 		case 'l':
 			delayloc = strdup(optarg);
 			break;
 		case 't':
 			delaytime = strtoul(optarg, NULL, 0);
 			break;
 		}
 	}
 
 	if (delaytime == -1) {
 		warnx("%s: you must specify the delaytime with -t", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	if (strcasecmp(delayloc, "datamove") == 0)
 		delay_info.delay_loc = CTL_DELAY_LOC_DATAMOVE;
 	else if (strcasecmp(delayloc, "done") == 0)
 		delay_info.delay_loc = CTL_DELAY_LOC_DONE;
 	else {
 		warnx("%s: invalid delay location %s", __func__, delayloc);
 		retval = 1;
 		goto bailout;
 	}
 
 	if ((delaytype == NULL)
 	 || (strcmp(delaytype, "oneshot") == 0))
 		delay_info.delay_type = CTL_DELAY_TYPE_ONESHOT;
 	else if (strcmp(delaytype, "cont") == 0)
 		delay_info.delay_type = CTL_DELAY_TYPE_CONT;
 	else {
 		warnx("%s: invalid delay type %s", __func__, delaytype);
 		retval = 1;
 		goto bailout;
 	}
 
 	delay_info.lun_id = lun;
 	delay_info.delay_secs = delaytime;
 
 	if (ioctl(fd, CTL_DELAY_IO, &delay_info) == -1) {
 		warn("%s: CTL_DELAY_IO ioctl failed", __func__);
 		retval = 1;
 		goto bailout;
 	}
 	switch (delay_info.status) {
 	case CTL_DELAY_STATUS_NONE:
 		warnx("%s: no delay status??", __func__);
 		retval = 1;
 		break;
 	case CTL_DELAY_STATUS_OK:
 		break;
 	case CTL_DELAY_STATUS_INVALID_LUN:
 		warnx("%s: invalid lun %d", __func__, lun);
 		retval = 1;
 		break;
 	case CTL_DELAY_STATUS_INVALID_TYPE:
 		warnx("%s: invalid delay type %d", __func__,
 		      delay_info.delay_type);
 		retval = 1;
 		break;
 	case CTL_DELAY_STATUS_INVALID_LOC:
 		warnx("%s: delay location %s not implemented?", __func__,
 		      delayloc);
 		retval = 1;
 		break;
 	case CTL_DELAY_STATUS_NOT_IMPLEMENTED:
 		warnx("%s: delay not implemented in the kernel", __func__);
 		warnx("%s: recompile with the CTL_IO_DELAY flag set", __func__);
 		retval = 1;
 		break;
 	default:
 		warnx("%s: unknown delay return status %d", __func__,
 		      delay_info.status);
 		retval = 1;
 		break;
 	}
 
 bailout:
 	free(delayloc);
 	free(delaytype);
 	return (retval);
 }
 
 static struct ctladm_opts cctl_err_types[] = {
 	{"aborted", CTL_LUN_INJ_ABORTED, CTLADM_ARG_NONE, NULL},
 	{"mediumerr", CTL_LUN_INJ_MEDIUM_ERR, CTLADM_ARG_NONE, NULL},
 	{"ua", CTL_LUN_INJ_UA, CTLADM_ARG_NONE, NULL},
 	{"custom", CTL_LUN_INJ_CUSTOM, CTLADM_ARG_NONE, NULL},
 	{NULL, 0, 0, NULL}
 
 };
 
 static struct ctladm_opts cctl_err_patterns[] = {
 	{"read", CTL_LUN_PAT_READ, CTLADM_ARG_NONE, NULL},
 	{"write", CTL_LUN_PAT_WRITE, CTLADM_ARG_NONE, NULL},
 	{"rw", CTL_LUN_PAT_READWRITE, CTLADM_ARG_NONE, NULL},
 	{"readwrite", CTL_LUN_PAT_READWRITE, CTLADM_ARG_NONE, NULL},
 	{"readcap", CTL_LUN_PAT_READCAP, CTLADM_ARG_NONE, NULL},
 	{"tur", CTL_LUN_PAT_TUR, CTLADM_ARG_NONE, NULL},
 	{"any", CTL_LUN_PAT_ANY, CTLADM_ARG_NONE, NULL},
 #if 0
 	{"cmd", CTL_LUN_PAT_CMD,  CTLADM_ARG_NONE, NULL},
 #endif
 	{NULL, 0, 0, NULL}
 };
 
 static int
 cctl_error_inject(int fd, uint32_t lun, int argc, char **argv,
 		  char *combinedopt)
 {
 	int retval = 0;
 	struct ctl_error_desc err_desc;
 	uint64_t lba = 0;
 	uint32_t len = 0;
 	uint64_t delete_id = 0;
 	int delete_id_set = 0;
 	int continuous = 0;
 	int sense_len = 0;
 	int fd_sense = 0;
 	int c;
 
 	bzero(&err_desc, sizeof(err_desc));
 	err_desc.lun_id = lun;
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'c':
 			continuous = 1;
 			break;
 		case 'd':
 			delete_id = strtoull(optarg, NULL, 0);
 			delete_id_set = 1;
 			break;
 		case 'i':
 		case 'p': {
 			ctladm_optret optret;
 			ctladm_cmdargs argnum;
 			const char *subopt;
 
 			if (c == 'i') {
 				ctl_lun_error err_type;
 
 				if (err_desc.lun_error != CTL_LUN_INJ_NONE) {
 					warnx("%s: can't specify multiple -i "
 					      "arguments", __func__);
 					retval = 1;
 					goto bailout;
 				}
 				optret = getoption(cctl_err_types, optarg,
 						   &err_type, &argnum, &subopt);
 				err_desc.lun_error = err_type;
 			} else {
 				ctl_lun_error_pattern pattern;
 
 				optret = getoption(cctl_err_patterns, optarg,
 						   &pattern, &argnum, &subopt);
 				err_desc.error_pattern |= pattern;
 			}
 
 			if (optret == CC_OR_AMBIGUOUS) {
 				warnx("%s: ambiguous argument %s", __func__,
 				      optarg);
 				retval = 1;
 				goto bailout;
 			} else if (optret == CC_OR_NOT_FOUND) {
 				warnx("%s: argument %s not found", __func__,
 				      optarg);
 				retval = 1;
 				goto bailout;
 			}
 			break;
 		}
 		case 'r': {
 			char *tmpstr, *tmpstr2;
 
 			tmpstr = strdup(optarg);
 			if (tmpstr == NULL) {
 				warn("%s: error duplicating string %s",
 				     __func__, optarg);
 				retval = 1;
 				goto bailout;
 			}
 
 			tmpstr2 = strsep(&tmpstr, ",");
 			if (tmpstr2 == NULL) {
 				warnx("%s: invalid -r argument %s", __func__,
 				      optarg);
 				retval = 1;
 				free(tmpstr);
 				goto bailout;
 			}
 			lba = strtoull(tmpstr2, NULL, 0);
 			tmpstr2 = strsep(&tmpstr, ",");
 			if (tmpstr2 == NULL) {
 				warnx("%s: no len argument for -r lba,len, got"
 				      " %s", __func__, optarg);
 				retval = 1;
 				free(tmpstr);
 				goto bailout;
 			}
 			len = strtoul(tmpstr2, NULL, 0);
 			free(tmpstr);
 			break;
 		}
 		case 's': {
 			struct get_hook hook;
 			char *sensestr;
 
 			sense_len = strtol(optarg, NULL, 0);
 			if (sense_len <= 0) {
 				warnx("invalid number of sense bytes %d",
 				      sense_len);
 				retval = 1;
 				goto bailout;
 			}
 
 			sense_len = MIN(sense_len, SSD_FULL_SIZE);
 
 			hook.argc = argc - optind;
 			hook.argv = argv + optind;
 			hook.got = 0;
 
 			sensestr = cget(&hook, NULL);
 			if ((sensestr != NULL)
 			 && (sensestr[0] == '-')) {
 				fd_sense = 1;
 			} else {
 				buff_encode_visit(
 				    (uint8_t *)&err_desc.custom_sense,
 				    sense_len, sensestr, iget, &hook);
 			}
 			optind += hook.got;
 			break;
 		}
 		default:
 			break;
 		}
 	}
 
 	if (delete_id_set != 0) {
 		err_desc.serial = delete_id;
 		if (ioctl(fd, CTL_ERROR_INJECT_DELETE, &err_desc) == -1) {
 			warn("%s: error issuing CTL_ERROR_INJECT_DELETE ioctl",
 			     __func__);
 			retval = 1;
 		}
 		goto bailout;
 	}
 
 	if (err_desc.lun_error == CTL_LUN_INJ_NONE) {
 		warnx("%s: error injection command (-i) needed",
 		      __func__);
 		retval = 1;
 		goto bailout;
 	} else if ((err_desc.lun_error == CTL_LUN_INJ_CUSTOM)
 		&& (sense_len == 0)) {
 		warnx("%s: custom error requires -s", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	if (continuous != 0)
 		err_desc.lun_error |= CTL_LUN_INJ_CONTINUOUS;
 
 	/*
 	 * If fd_sense is set, we need to read the sense data the user
 	 * wants returned from stdin.
 	 */
         if (fd_sense == 1) {
 		ssize_t amt_read;
 		int amt_to_read = sense_len;
 		u_int8_t *buf_ptr = (uint8_t *)&err_desc.custom_sense;
 
 		for (amt_read = 0; amt_to_read > 0;
 		     amt_read = read(STDIN_FILENO, buf_ptr, amt_to_read)) {
 			if (amt_read == -1) {
 				warn("error reading sense data from stdin");
 				retval = 1;
 				goto bailout;
 			}
 			amt_to_read -= amt_read;
 			buf_ptr += amt_read;
 		}
 	}
 
 	if (err_desc.error_pattern == CTL_LUN_PAT_NONE) {
 		warnx("%s: command pattern (-p) needed", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	if (len != 0) {
 		err_desc.error_pattern |= CTL_LUN_PAT_RANGE;
 		/*
 		 * We could check here to see whether it's a read/write
 		 * command, but that will be pointless once we allow
 		 * custom patterns.  At that point, the user could specify
 		 * a READ(6) CDB type, and we wouldn't have an easy way here
 		 * to verify whether range checking is possible there.  The
 		 * user will just figure it out when his error never gets
 		 * executed.
 		 */
 #if 0
 		if ((err_desc.pattern & CTL_LUN_PAT_READWRITE) == 0) {
 			warnx("%s: need read and/or write pattern if range "
 			      "is specified", __func__);
 			retval = 1;
 			goto bailout;
 		}
 #endif
 		err_desc.lba_range.lba = lba;
 		err_desc.lba_range.len = len;
 	}
 
 	if (ioctl(fd, CTL_ERROR_INJECT, &err_desc) == -1) {
 		warn("%s: error issuing CTL_ERROR_INJECT ioctl", __func__);
 		retval = 1;
 	} else {
 		printf("Error injection succeeded, serial number is %ju\n",
 		       (uintmax_t)err_desc.serial);
 	}
 bailout:
 
 	return (retval);
 }
 
 static int
 cctl_lunlist(int fd)
 {
 	struct scsi_report_luns_data *lun_data;
 	struct scsi_inquiry_data *inq_data;
 	uint32_t num_luns;
 	int initid;
 	unsigned int i;
 	int retval;
 
 	inq_data = NULL;
 	initid = 7;
 
 	/*
 	 * XXX KDM assuming LUN 0 is fine, but we may need to change this
 	 * if we ever acquire the ability to have multiple targets.
 	 */
 	if ((retval = cctl_get_luns(fd, /*lun*/ 0, initid,
 				    /*retries*/ 2, &lun_data, &num_luns)) != 0)
 		goto bailout;
 
 	inq_data = malloc(sizeof(*inq_data));
 	if (inq_data == NULL) {
 		warn("%s: couldn't allocate memory for inquiry data\n",
 		     __func__);
 		retval = 1;
 		goto bailout;
 	}
 	for (i = 0; i < num_luns; i++) {
 		char scsi_path[40];
 		int lun_val;
 
 		switch (lun_data->luns[i].lundata[0] & RPL_LUNDATA_ATYP_MASK) {
 		case RPL_LUNDATA_ATYP_PERIPH:
 			lun_val = lun_data->luns[i].lundata[1];
 			break;
 		case RPL_LUNDATA_ATYP_FLAT:
 			lun_val = (lun_data->luns[i].lundata[0] &
 				RPL_LUNDATA_FLAT_LUN_MASK) |
 				(lun_data->luns[i].lundata[1] <<
 				RPL_LUNDATA_FLAT_LUN_BITS);
 			break;
 		case RPL_LUNDATA_ATYP_LUN:
 		case RPL_LUNDATA_ATYP_EXTLUN:
 		default:
 			fprintf(stdout, "Unsupported LUN format %d\n",
 				lun_data->luns[i].lundata[0] &
 				RPL_LUNDATA_ATYP_MASK);
 			lun_val = -1;
 			break;
 		}
 		if (lun_val == -1)
 			continue;
 
 		if ((retval = cctl_get_inquiry(fd, lun_val, initid,
 					       /*retries*/ 2, scsi_path,
 					       sizeof(scsi_path),
 					       inq_data)) != 0) {
 			goto bailout;
 		}
 		printf("%s", scsi_path);
 		scsi_print_inquiry(inq_data);
 	}
 bailout:
 
 	if (lun_data != NULL)
 		free(lun_data);
 
 	if (inq_data != NULL)
 		free(inq_data);
 
 	return (retval);
 }
 
 static int
 cctl_sync_cache(int fd, int lun, int iid, int retries,
 		int argc, char **argv, char *combinedopt)
 {
 	union ctl_io *io;
 	int cdb_size = -1;
 	int retval;
 	uint64_t our_lba = 0;
 	uint32_t our_block_count = 0;
 	int reladr = 0, immed = 0;
 	int c;
 
 	retval = 0;
 
 	io = ctl_scsi_alloc_io(iid);
 	if (io == NULL) {
 		warnx("%s: can't allocate memory", __func__);
 		return (1);
 	}
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'b':
 			our_block_count = strtoul(optarg, NULL, 0);
 			break;
 		case 'c':
 			cdb_size = strtol(optarg, NULL, 0);
 			break;
 		case 'i':
 			immed = 1;
 			break;
 		case 'l':
 			our_lba = strtoull(optarg, NULL, 0);
 			break;
 		case 'r':
 			reladr = 1;
 			break;
 		default:
 			break;
 		}
 	}
 
 	if (cdb_size != -1) {
 		switch (cdb_size) {
 		case 10:
 		case 16:
 			break;
 		default:
 			warnx("%s: invalid cdbsize %d, valid sizes are 10 "
 			      "and 16", __func__, cdb_size);
 			retval = 1;
 			goto bailout;
 			break; /* NOTREACHED */
 		}
 	} else
 		cdb_size = 10;
 
 	ctl_scsi_sync_cache(/*io*/ io,
 			    /*immed*/ immed,
 			    /*reladr*/ reladr,
 			    /*minimum_cdb_size*/ cdb_size,
 			    /*starting_lba*/ our_lba,
 			    /*block_count*/ our_block_count,
 			    /*tag_type*/ CTL_TAG_SIMPLE,
 			    /*control*/ 0);
 
 	io->io_hdr.nexus.targ_lun = lun;
 	io->io_hdr.nexus.initid = iid;
 
 	if (cctl_do_io(fd, retries, io, __func__) != 0) {
 		retval = 1;
 		goto bailout;
 	}
 
 	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) {
 		fprintf(stdout, "Cache synchronized successfully\n");
 	} else
 		ctl_io_error_print(io, NULL, stderr);
 bailout:
 	ctl_scsi_free_io(io);
 
 	return (retval);
 }
 
 static int
 cctl_start_stop(int fd, int lun, int iid, int retries, int start,
 		int argc, char **argv, char *combinedopt)
 {
 	union ctl_io *io;
 	char scsi_path[40];
 	int immed = 0;
 	int retval, c;
 
 	retval = 0;
 
 	io = ctl_scsi_alloc_io(iid);
 	if (io == NULL) {
 		warnx("%s: can't allocate memory", __func__);
 		return (1);
 	}
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'i':
 			immed = 1;
 			break;
 		default:
 			break;
 		}
 	}
 	/*
 	 * Use an ordered tag for the stop command, to guarantee that any
 	 * pending I/O will finish before the stop command executes.  This
 	 * would normally be the case anyway, since CTL will basically
 	 * treat the start/stop command as an ordered command with respect
 	 * to any other command except an INQUIRY.  (See ctl_ser_table.c.)
 	 */
 	ctl_scsi_start_stop(/*io*/ io,
 			    /*start*/ start,
 			    /*load_eject*/ 0,
 			    /*immediate*/ immed,
 			    /*power_conditions*/ SSS_PC_START_VALID,
 			    /*ctl_tag_type*/ start ? CTL_TAG_SIMPLE :
 						     CTL_TAG_ORDERED,
 			    /*control*/ 0);
 
 	io->io_hdr.nexus.targ_lun = lun;
 	io->io_hdr.nexus.initid = iid;
 
 	if (cctl_do_io(fd, retries, io, __func__) != 0) {
 		retval = 1;
 		goto bailout;
 	}
 
 	ctl_scsi_path_string(&io->io_hdr, scsi_path, sizeof(scsi_path));
 	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) {
 		fprintf(stdout, "%s LUN %s successfully\n", scsi_path,
 			(start) ?  "started" : "stopped");
 	} else
 		ctl_io_error_print(io, NULL, stderr);
 
 bailout:
 	ctl_scsi_free_io(io);
 
 	return (retval);
 }
 
 static int
 cctl_mode_sense(int fd, int lun, int iid, int retries,
 		int argc, char **argv, char *combinedopt)
 {
 	union ctl_io *io;
 	uint32_t datalen;
 	uint8_t *dataptr;
 	int pc = -1, cdbsize, retval, dbd = 0, subpage = -1;
 	int list = 0;
 	int page_code = -1;
 	int c;
 
 	cdbsize = 0;
 	retval = 0;
 	dataptr = NULL;
 
 	io = ctl_scsi_alloc_io(iid);
 	if (io == NULL) {
 		warn("%s: can't allocate memory", __func__);
 		return (1);
 	}
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'P':
 			pc = strtoul(optarg, NULL, 0);
 			break;
 		case 'S':
 			subpage = strtoul(optarg, NULL, 0);
 			break;
 		case 'd':
 			dbd = 1;
 			break;
 		case 'l':
 			list = 1;
 			break;
 		case 'm':
 			page_code = strtoul(optarg, NULL, 0);
 			break;
 		case 'c':
 			cdbsize = strtol(optarg, NULL, 0);
 			break;
 		default:
 			break;
 		}
 	}
 
 	if (((list == 0) && (page_code == -1))
 	 || ((list != 0) && (page_code != -1))) {
 		warnx("%s: you must specify either a page code (-m) or -l",
 		      __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	if ((page_code != -1)
 	 && ((page_code > SMS_ALL_PAGES_PAGE)
 	  || (page_code < 0))) {
 		warnx("%s: page code %d is out of range", __func__,
 		      page_code);
 		retval = 1;
 		goto bailout;
 	}
 
 	if (list == 1) {
 		page_code = SMS_ALL_PAGES_PAGE;
 		if (pc != -1) {
 			warnx("%s: arg -P makes no sense with -l",
 			      __func__);
 			retval = 1;
 			goto bailout;
 		}
 		if (subpage != -1) {
 			warnx("%s: arg -S makes no sense with -l", __func__);
 			retval = 1;
 			goto bailout;
 		}
 	}
 
 	if (pc == -1)
 		pc = SMS_PAGE_CTRL_CURRENT;
 	else {
 		if ((pc > 3)
 		 || (pc < 0)) {
 			warnx("%s: page control value %d is out of range: 0-3",
 			      __func__, pc);
 			retval = 1;
 			goto bailout;
 		}
 	}
 
 
 	if ((subpage != -1)
 	 && ((subpage > 255)
 	  || (subpage < 0))) {
 		warnx("%s: subpage code %d is out of range: 0-255", __func__,
 		      subpage);
 		retval = 1;
 		goto bailout;
 	}
 	if (cdbsize != 0) {
 		switch (cdbsize) {
 		case 6:
 		case 10:
 			break;
 		default:
 			warnx("%s: invalid cdbsize %d, valid sizes are 6 "
 			      "and 10", __func__, cdbsize);
 			retval = 1;
 			goto bailout;
 			break;
 		}
 	} else
 		cdbsize = 6;
 
 	if (subpage == -1)
 		subpage = 0;
 
 	if (cdbsize == 6)
 		datalen = 255;
 	else
 		datalen = 65535;
 
 	dataptr = (uint8_t *)malloc(datalen);
 	if (dataptr == NULL) {
 		warn("%s: can't allocate %d bytes", __func__, datalen);
 		retval = 1;
 		goto bailout;
 	}
 
 	memset(dataptr, 0, datalen);
 
 	ctl_scsi_mode_sense(io,
 			    /*data_ptr*/ dataptr,
 			    /*data_len*/ datalen,
 			    /*dbd*/ dbd,
 			    /*llbaa*/ 0,
 			    /*page_code*/ page_code,
 			    /*pc*/ pc << 6,
 			    /*subpage*/ subpage,
 			    /*minimum_cdb_size*/ cdbsize,
 			    /*tag_type*/ CTL_TAG_SIMPLE,
 			    /*control*/ 0);
 
 	io->io_hdr.nexus.targ_lun = lun;
 	io->io_hdr.nexus.initid = iid;
 
 	if (cctl_do_io(fd, retries, io, __func__) != 0) {
 		retval = 1;
 		goto bailout;
 	}
 
 	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) {
 		int pages_len, used_len;
 		uint32_t returned_len;
 		uint8_t *ndataptr;
 
 		if (io->scsiio.cdb[0] == MODE_SENSE_6) {
 			struct scsi_mode_hdr_6 *hdr6;
 			int bdlen;
 
 			hdr6 = (struct scsi_mode_hdr_6 *)dataptr;
 
 			returned_len = hdr6->datalen + 1;
 			bdlen = hdr6->block_descr_len;
 
 			ndataptr = (uint8_t *)((uint8_t *)&hdr6[1] + bdlen);
 		} else {
 			struct scsi_mode_hdr_10 *hdr10;
 			int bdlen;
 
 			hdr10 = (struct scsi_mode_hdr_10 *)dataptr;
 
 			returned_len = scsi_2btoul(hdr10->datalen) + 2;
 			bdlen = scsi_2btoul(hdr10->block_descr_len);
 
 			ndataptr = (uint8_t *)((uint8_t *)&hdr10[1] + bdlen);
 		}
 		/* just in case they can give us more than we allocated for */
 		returned_len = min(returned_len, datalen);
 		pages_len = returned_len - (ndataptr - dataptr);
 #if 0
 		fprintf(stdout, "returned_len = %d, pages_len = %d\n",
 			returned_len, pages_len);
 #endif
 		if (list == 1) {
 			fprintf(stdout, "Supported mode pages:\n");
 			for (used_len = 0; used_len < pages_len;) {
 				struct scsi_mode_page_header *header;
 
 				header = (struct scsi_mode_page_header *)
 					&ndataptr[used_len];
 				fprintf(stdout, "%d\n", header->page_code);
 				used_len += header->page_length + 2;
 			}
 		} else {
 			for (used_len = 0; used_len < pages_len; used_len++) {
 				fprintf(stdout, "0x%x ", ndataptr[used_len]);
 				if (((used_len+1) % 16) == 0)
 					fprintf(stdout, "\n");
 			}
 			fprintf(stdout, "\n");
 		}
 	} else
 		ctl_io_error_print(io, NULL, stderr);
 bailout:
 
 	ctl_scsi_free_io(io);
 
 	if (dataptr != NULL)
 		free(dataptr);
 
 	return (retval);
 }
 
 static int
 cctl_read_capacity(int fd, int lun, int iid, int retries,
 		   int argc, char **argv, char *combinedopt)
 {
 	union ctl_io *io;
 	struct scsi_read_capacity_data *data;
 	struct scsi_read_capacity_data_long *longdata;
 	int cdbsize = -1, retval;
 	uint8_t *dataptr;
 	int c;
 
 	cdbsize = 10;
 	dataptr = NULL;
 	retval = 0;
 
 	io = ctl_scsi_alloc_io(iid);
 	if (io == NULL) {
 		warn("%s: can't allocate memory\n", __func__);
 		return (1);
 	}
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'c':
 			cdbsize = strtol(optarg, NULL, 0);
 			break;
 		default:
 			break;
 		}
 	}
 	if (cdbsize != -1) {
 		switch (cdbsize) {
 		case 10:
 		case 16:
 			break;
 		default:
 			warnx("%s: invalid cdbsize %d, valid sizes are 10 "
 			      "and 16", __func__, cdbsize);
 			retval = 1;
 			goto bailout;
 			break; /* NOTREACHED */
 		}
 	} else
 		cdbsize = 10;
 
 	dataptr = (uint8_t *)malloc(sizeof(*longdata));
 	if (dataptr == NULL) {
 		warn("%s: can't allocate %zd bytes\n", __func__,
 		     sizeof(*longdata));
 		retval = 1;
 		goto bailout;
 	}
 	memset(dataptr, 0, sizeof(*longdata));
 
 retry:
 
 	switch (cdbsize) {
 	case 10:
 		ctl_scsi_read_capacity(io,
 				       /*data_ptr*/ dataptr,
 				       /*data_len*/ sizeof(*longdata),
 				       /*addr*/ 0,
 				       /*reladr*/ 0,
 				       /*pmi*/ 0,
 				       /*tag_type*/ CTL_TAG_SIMPLE,
 				       /*control*/ 0);
 		break;
 	case 16:
 		ctl_scsi_read_capacity_16(io,
 					  /*data_ptr*/ dataptr,
 					  /*data_len*/ sizeof(*longdata),
 					  /*addr*/ 0,
 					  /*reladr*/ 0,
 					  /*pmi*/ 0,
 					  /*tag_type*/ CTL_TAG_SIMPLE,
 					  /*control*/ 0);
 		break;
 	}
 
 	io->io_hdr.nexus.initid = iid;
 	io->io_hdr.nexus.targ_lun = lun;
 
 	if (cctl_do_io(fd, retries, io, __func__) != 0) {
 		retval = 1;
 		goto bailout;
 	}
 
 	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) {
 		uint64_t maxlba;
 		uint32_t blocksize;
 
 		if (cdbsize == 10) {
 
 			data = (struct scsi_read_capacity_data *)dataptr;
 
 			maxlba = scsi_4btoul(data->addr);
 			blocksize = scsi_4btoul(data->length);
 
 			if (maxlba == 0xffffffff) {
 				cdbsize = 16;
 				goto retry;
 			}
 		} else {
 			longdata=(struct scsi_read_capacity_data_long *)dataptr;
 
 			maxlba = scsi_8btou64(longdata->addr);
 			blocksize = scsi_4btoul(longdata->length);
 		}
 
 		fprintf(stdout, "Disk Capacity: %ju, Blocksize: %d\n",
 			(uintmax_t)maxlba, blocksize);
 	} else {
 		ctl_io_error_print(io, NULL, stderr);
 	}
 bailout:
 	ctl_scsi_free_io(io);
 
 	if (dataptr != NULL)
 		free(dataptr);
 
 	return (retval);
 }
 
 static int
 cctl_read_write(int fd, int lun, int iid, int retries,
 		int argc, char **argv, char *combinedopt,
 		ctladm_cmdfunction command)
 {
 	union ctl_io *io;
 	int file_fd, do_stdio;
 	int cdbsize = -1, databytes;
 	uint8_t *dataptr;
 	char *filename = NULL;
 	int datalen = -1, blocksize = -1;
 	uint64_t lba = 0;
 	int lba_set = 0;
 	int retval;
 	int c;
 
 	retval = 0;
 	do_stdio = 0;
 	dataptr = NULL;
 	file_fd = -1;
 
 	io = ctl_scsi_alloc_io(iid);
 	if (io == NULL) {
 		warn("%s: can't allocate memory\n", __func__);
 		return (1);
 	}
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'N':
 			io->io_hdr.flags |= CTL_FLAG_NO_DATAMOVE;
 			break;
 		case 'b':
 			blocksize = strtoul(optarg, NULL, 0);
 			break;
 		case 'c':
 			cdbsize = strtoul(optarg, NULL, 0);
 			break;
 		case 'd':
 			datalen = strtoul(optarg, NULL, 0);
 			break;
 		case 'f':
 			filename = strdup(optarg);
 			break;
 		case 'l':
 			lba = strtoull(optarg, NULL, 0);
 			lba_set = 1;
 			break;
 		default:
 			break;
 		}
 	}
 	if (filename == NULL) {
 		warnx("%s: you must supply a filename using -f", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	if (datalen == -1) {
 		warnx("%s: you must specify the data length with -d", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	if (lba_set == 0) {
 		warnx("%s: you must specify the LBA with -l", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	if (blocksize == -1) {
 		warnx("%s: you must specify the blocksize with -b", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	if (cdbsize != -1) {
 		switch (cdbsize) {
 		case 6:
 		case 10:
 		case 12:
 		case 16:
 			break;
 		default:
 			warnx("%s: invalid cdbsize %d, valid sizes are 6, "
 			      "10, 12 or 16", __func__, cdbsize);
 			retval = 1;
 			goto bailout;
 			break; /* NOTREACHED */
 		}
 	} else
 		cdbsize = 6;
 
 	databytes = datalen * blocksize;
 	dataptr = (uint8_t *)malloc(databytes);
 
 	if (dataptr == NULL) {
 		warn("%s: can't allocate %d bytes\n", __func__, databytes);
 		retval = 1;
 		goto bailout;
 	}
 	if (strcmp(filename, "-") == 0) {
 		if (command == CTLADM_CMD_READ)
 			file_fd = STDOUT_FILENO;
 		else
 			file_fd = STDIN_FILENO;
 		do_stdio = 1;
 	} else {
 		file_fd = open(filename, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
 		if (file_fd == -1) {
 			warn("%s: can't open file %s", __func__, filename);
 			retval = 1;
 			goto bailout;
 		}
 	}
 
 	memset(dataptr, 0, databytes);
 
 	if (command == CTLADM_CMD_WRITE) {
 		int bytes_read;
 
 		bytes_read = read(file_fd, dataptr, databytes);
 		if (bytes_read == -1) {
 			warn("%s: error reading file %s", __func__, filename);
 			retval = 1;
 			goto bailout;
 		}
 		if (bytes_read != databytes) {
 			warnx("%s: only read %d bytes from file %s",
 			      __func__, bytes_read, filename);
 			retval = 1;
 			goto bailout;
 		}
 	}
 	ctl_scsi_read_write(io,
 			    /*data_ptr*/ dataptr,
 			    /*data_len*/ databytes,
 			    /*read_op*/ (command == CTLADM_CMD_READ) ? 1 : 0,
 			    /*byte2*/ 0,
 			    /*minimum_cdb_size*/ cdbsize,
 			    /*lba*/ lba,
 			    /*num_blocks*/ datalen,
 			    /*tag_type*/ CTL_TAG_SIMPLE,
 			    /*control*/ 0);
 
 	io->io_hdr.nexus.targ_lun = lun;
 	io->io_hdr.nexus.initid = iid;
 
 	if (cctl_do_io(fd, retries, io, __func__) != 0) {
 		retval = 1;
 		goto bailout;
 	}
 
 	if (((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)
 	 && (command == CTLADM_CMD_READ)) {
 		int bytes_written;
 
 		bytes_written = write(file_fd, dataptr, databytes);
 		if (bytes_written == -1) {
 			warn("%s: can't write to %s", __func__, filename);
 			goto bailout;
 		}
 	} else if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
 		ctl_io_error_print(io, NULL, stderr);
 
 
 bailout:
 
 	ctl_scsi_free_io(io);
 
 	if (dataptr != NULL)
 		free(dataptr);
 
 	if ((do_stdio == 0)
 	 && (file_fd != -1))
 		close(file_fd);
 
 	return (retval);
 }
 
 static int
 cctl_get_luns(int fd, int lun, int iid, int retries, struct
 	      scsi_report_luns_data **lun_data, uint32_t *num_luns)
 {
 	union ctl_io *io;
 	uint32_t nluns;
 	int lun_datalen;
 	int retval;
 
 	retval = 0;
 
 	io = ctl_scsi_alloc_io(iid);
 	if (io == NULL) {
 		warnx("%s: can't allocate memory", __func__);
 		return (1);
 	}
 
 	/*
 	 * lun_data includes space for 1 lun, allocate space for 4 initially.
 	 * If that isn't enough, we'll allocate more.
 	 */
 	nluns = 4;
 retry:
 	lun_datalen = sizeof(*lun_data) +
 		(nluns * sizeof(struct scsi_report_luns_lundata));
 	*lun_data = malloc(lun_datalen);
 
 	if (*lun_data == NULL) {
 		warnx("%s: can't allocate memory", __func__);
 		ctl_scsi_free_io(io);
 		return (1);
 	}
 
 	ctl_scsi_report_luns(io,
 			     /*data_ptr*/ (uint8_t *)*lun_data,
 			     /*data_len*/ lun_datalen,
 			     /*select_report*/ RPL_REPORT_ALL,
 			     /*tag_type*/ CTL_TAG_SIMPLE,
 			     /*control*/ 0);
 
 	io->io_hdr.nexus.initid = iid;
 	io->io_hdr.nexus.targ_lun = lun;
 
 	if (cctl_do_io(fd, retries, io, __func__) != 0) {
 		retval = 1;
 		goto bailout;
 	}
 
 	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) {
 		uint32_t returned_len, returned_luns;
 
 		returned_len = scsi_4btoul((*lun_data)->length);
 		returned_luns = returned_len / 8;
 		if (returned_luns > nluns) {
 			nluns = returned_luns;
 			free(*lun_data);
 			goto retry;
 		}
 		/* These should be the same */
 		*num_luns = MIN(returned_luns, nluns);
 	} else {
 		ctl_io_error_print(io, NULL, stderr);
 		retval = 1;
 	}
 bailout:
 	ctl_scsi_free_io(io);
 
 	return (retval);
 }
 
 static int
 cctl_report_luns(int fd, int lun, int iid, int retries)
 {
 	struct scsi_report_luns_data *lun_data;
 	uint32_t num_luns, i;
 	int retval;
 
 	lun_data = NULL;
 
 	if ((retval = cctl_get_luns(fd, lun, iid, retries, &lun_data,
 				   &num_luns)) != 0)
 		goto bailout;
 
 	fprintf(stdout, "%u LUNs returned\n", num_luns);
 	for (i = 0; i < num_luns; i++) {
 		int lun_val;
 
 		/*
 		 * XXX KDM figure out a way to share this code with
 		 * cctl_lunlist()?
 		 */
 		switch (lun_data->luns[i].lundata[0] & RPL_LUNDATA_ATYP_MASK) {
 		case RPL_LUNDATA_ATYP_PERIPH:
 			lun_val = lun_data->luns[i].lundata[1];
 			break;
 		case RPL_LUNDATA_ATYP_FLAT:
 			lun_val = (lun_data->luns[i].lundata[0] &
 				RPL_LUNDATA_FLAT_LUN_MASK) |
 				(lun_data->luns[i].lundata[1] <<
 				RPL_LUNDATA_FLAT_LUN_BITS);
 			break;
 		case RPL_LUNDATA_ATYP_LUN:
 		case RPL_LUNDATA_ATYP_EXTLUN:
 		default:
 			fprintf(stdout, "Unsupported LUN format %d\n",
 				lun_data->luns[i].lundata[0] &
 				RPL_LUNDATA_ATYP_MASK);
 			lun_val = -1;
 			break;
 		}
 		if (lun_val == -1)
 			continue;
 
 		fprintf(stdout, "%d\n", lun_val);
 	}
 
 bailout:
 	if (lun_data != NULL)
 		free(lun_data);
 
 	return (retval);
 }
 
 static int
 cctl_tur(int fd, int lun, int iid, int retries)
 {
 	union ctl_io *io;
 
 	io = ctl_scsi_alloc_io(iid);
 	if (io == NULL) {
 		fprintf(stderr, "can't allocate memory\n");
 		return (1);
 	}
 
 	ctl_scsi_tur(io,
 		     /* tag_type */ CTL_TAG_SIMPLE,
 		     /* control */ 0);
 
 	io->io_hdr.nexus.targ_lun = lun;
 	io->io_hdr.nexus.initid = iid;
 
 	if (cctl_do_io(fd, retries, io, __func__) != 0) {
 		ctl_scsi_free_io(io);
 		return (1);
 	}
 
 	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)
 		fprintf(stdout, "Unit is ready\n");
 	else
 		ctl_io_error_print(io, NULL, stderr);
 
 	return (0);
 }
 
 static int
 cctl_get_inquiry(int fd, int lun, int iid, int retries,
 		 char *path_str, int path_len,
 		 struct scsi_inquiry_data *inq_data)
 {
 	union ctl_io *io;
 	int retval;
 
 	retval = 0;
 
 	io = ctl_scsi_alloc_io(iid);
 	if (io == NULL) {
 		warnx("cctl_inquiry: can't allocate memory\n");
 		return (1);
 	}
 
 	ctl_scsi_inquiry(/*io*/ io,
 			 /*data_ptr*/ (uint8_t *)inq_data,
 			 /*data_len*/ sizeof(*inq_data),
 			 /*byte2*/ 0,
 			 /*page_code*/ 0,
 			 /*tag_type*/ CTL_TAG_SIMPLE,
 			 /*control*/ 0);
 
 	io->io_hdr.nexus.targ_lun = lun;
 	io->io_hdr.nexus.initid = iid;
 
 	if (cctl_do_io(fd, retries, io, __func__) != 0) {
 		retval = 1;
 		goto bailout;
 	}
 
 	if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS) {
 		retval = 1;
 		ctl_io_error_print(io, NULL, stderr);
 	} else if (path_str != NULL)
 		ctl_scsi_path_string(&io->io_hdr, path_str, path_len);
 
 bailout:
 	ctl_scsi_free_io(io);
 
 	return (retval);
 }
 
 static int
 cctl_inquiry(int fd, int lun, int iid, int retries)
 {
 	struct scsi_inquiry_data *inq_data;
 	char scsi_path[40];
 	int retval;
 
 	inq_data = malloc(sizeof(*inq_data));
 	if (inq_data == NULL) {
 		warnx("%s: can't allocate inquiry data", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	if ((retval = cctl_get_inquiry(fd, lun, iid, retries, scsi_path,
 				       sizeof(scsi_path), inq_data)) != 0)
 		goto bailout;
 
 	printf("%s", scsi_path);
 	scsi_print_inquiry(inq_data);
 
 bailout:
 	if (inq_data != NULL)
 		free(inq_data);
 
 	return (retval);
 }
 
 static int
 cctl_req_sense(int fd, int lun, int iid, int retries)
 {
 	union ctl_io *io;
 	struct scsi_sense_data *sense_data;
 	int retval;
 
 	retval = 0;
 
 	io = ctl_scsi_alloc_io(iid);
 	if (io == NULL) {
 		warnx("cctl_req_sense: can't allocate memory\n");
 		return (1);
 	}
 	sense_data = malloc(sizeof(*sense_data));
 	memset(sense_data, 0, sizeof(*sense_data));
 
 	ctl_scsi_request_sense(/*io*/ io,
 			       /*data_ptr*/ (uint8_t *)sense_data,
 			       /*data_len*/ sizeof(*sense_data),
 			       /*byte2*/ 0,
 			       /*tag_type*/ CTL_TAG_SIMPLE,
 			       /*control*/ 0);
 
 	io->io_hdr.nexus.targ_lun = lun;
 	io->io_hdr.nexus.initid = iid;
 
 	if (cctl_do_io(fd, retries, io, __func__) != 0) {
 		retval = 1;
 		goto bailout;
 	}
 
 	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) {
 		bcopy(sense_data, &io->scsiio.sense_data, sizeof(*sense_data));
 		io->scsiio.sense_len = sizeof(*sense_data);
 		ctl_scsi_sense_print(&io->scsiio, NULL, stdout);
 	} else
 		ctl_io_error_print(io, NULL, stderr);
 
 bailout:
 
 	ctl_scsi_free_io(io);
 	free(sense_data);
 
 	return (retval);
 }
 
 static int
 cctl_report_target_port_group(int fd, int lun, int iid)
 {
 	union ctl_io *io;
 	uint32_t datalen;
 	uint8_t *dataptr;
 	int retval;
 
 	dataptr = NULL;
 	retval = 0;
 
 	io = ctl_scsi_alloc_io(iid);
 	if (io == NULL) {
 		warn("%s: can't allocate memory", __func__);
 		return (1);
 	}
 
 	datalen = 64;
 	dataptr = (uint8_t *)malloc(datalen);
 	if (dataptr == NULL) {
 		warn("%s: can't allocate %d bytes", __func__, datalen);
 		retval = 1;
 		goto bailout;
 	}
 
 	memset(dataptr, 0, datalen);
 
 	ctl_scsi_maintenance_in(/*io*/ io,
 				/*data_ptr*/ dataptr,
 				/*data_len*/ datalen,
 				/*action*/ SA_RPRT_TRGT_GRP,
 				/*tag_type*/ CTL_TAG_SIMPLE,
 				/*control*/ 0);
 
 	io->io_hdr.nexus.targ_lun = lun;
 	io->io_hdr.nexus.initid = iid;
 
 	if (cctl_do_io(fd, 0, io, __func__) != 0) {
 		retval = 1;
 		goto bailout;
 	}
 
 	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) {
 		int returned_len, used_len;
 
 		returned_len = scsi_4btoul(&dataptr[0]) + 4;
 
 		for (used_len = 0; used_len < returned_len; used_len++) {
 			fprintf(stdout, "0x%02x ", dataptr[used_len]);
 			if (((used_len+1) % 8) == 0)
 				fprintf(stdout, "\n");
 		}
 		fprintf(stdout, "\n");
 	} else
 		ctl_io_error_print(io, NULL, stderr);
 
 bailout:
 	ctl_scsi_free_io(io);
 
 	if (dataptr != NULL)
 		free(dataptr);
 
 	return (retval);
 }
 
 static int
 cctl_inquiry_vpd_devid(int fd, int lun, int iid)
 {
 	union ctl_io *io;
 	uint32_t datalen;
 	uint8_t *dataptr;
 	int retval;
 
 	retval = 0;
 	dataptr = NULL;
 
 	io = ctl_scsi_alloc_io(iid);
 	if (io == NULL) {
 		warn("%s: can't allocate memory", __func__);
 		return (1);
 	}
 
 	datalen = 256;
 	dataptr = (uint8_t *)malloc(datalen);
 	if (dataptr == NULL) {
 		warn("%s: can't allocate %d bytes", __func__, datalen);
 		retval = 1;
 		goto bailout;
 	}
 
 	memset(dataptr, 0, datalen);
 
 	ctl_scsi_inquiry(/*io*/        io,
 			 /*data_ptr*/  dataptr,
 			 /*data_len*/  datalen,
 			 /*byte2*/     SI_EVPD,
 			 /*page_code*/ SVPD_DEVICE_ID,
 			 /*tag_type*/  CTL_TAG_SIMPLE,
 			 /*control*/   0);
 
 	io->io_hdr.nexus.targ_lun = lun;
 	io->io_hdr.nexus.initid = iid;
 
 	if (cctl_do_io(fd, 0, io, __func__) != 0) {
 		retval = 1;
 		goto bailout;
 	}
 
 	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) {
 		int returned_len, used_len;
 
 		returned_len = scsi_2btoul(&dataptr[2]) + 4;
 
 		for (used_len = 0; used_len < returned_len; used_len++) {
 			fprintf(stdout, "0x%02x ", dataptr[used_len]);
 			if (((used_len+1) % 8) == 0)
 				fprintf(stdout, "\n");
 		}
 		fprintf(stdout, "\n");
 	} else
 		ctl_io_error_print(io, NULL, stderr);
 
 bailout:
 	ctl_scsi_free_io(io);
 
 	if (dataptr != NULL)
 		free(dataptr);
 
 	return (retval);
 }
 
 static int
 cctl_persistent_reserve_in(int fd, int lun, int iid,
                            int argc, char **argv, char *combinedopt,
 			   int retry_count)
 {
 	union ctl_io *io;
 	uint32_t datalen;
 	uint8_t *dataptr;
 	int action = -1;
 	int retval;
 	int c;
 
 	retval = 0;
 	dataptr = NULL;
 
 	io = ctl_scsi_alloc_io(iid);
 	if (io == NULL) {
 		warn("%s: can't allocate memory", __func__);
 		return (1);
 	}
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'a':
 			action = strtol(optarg, NULL, 0);
 			break;
 		default:
 			break;
 		}
 	}
 
 	if (action < 0 || action > 2) {
 		warn("action must be specified and in the range: 0-2");
 		retval = 1;
 		goto bailout;
 	}
 
 
 	datalen = 256;
 	dataptr = (uint8_t *)malloc(datalen);
 	if (dataptr == NULL) {
 		warn("%s: can't allocate %d bytes", __func__, datalen);
 		retval = 1;
 		goto bailout;
 	}
 
 	memset(dataptr, 0, datalen);
 
 	ctl_scsi_persistent_res_in(io,
 				   /*data_ptr*/ dataptr,
 				   /*data_len*/ datalen,
 				   /*action*/   action,
 				   /*tag_type*/ CTL_TAG_SIMPLE,
 				   /*control*/  0);
 
 	io->io_hdr.nexus.targ_lun = lun;
 	io->io_hdr.nexus.initid = iid;
 
 	if (cctl_do_io(fd, retry_count, io, __func__) != 0) {
 		retval = 1;
 		goto bailout;
 	}
 
 	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) {
 		int returned_len, used_len;
 
 		switch (action) {
 		case 0:
 			returned_len = scsi_4btoul(&dataptr[4]) + 8;
 			returned_len = min(returned_len, 256);
 			break;
 		case 1:
 			returned_len = scsi_4btoul(&dataptr[4]) + 8;
 			break;
 		case 2:
 			returned_len = 8;
 			break;
 		default:
 			warnx("%s: invalid action %d", __func__, action);
 			goto bailout;
 			break; /* NOTREACHED */
 		}
 
 		for (used_len = 0; used_len < returned_len; used_len++) {
 			fprintf(stdout, "0x%02x ", dataptr[used_len]);
 			if (((used_len+1) % 8) == 0)
 				fprintf(stdout, "\n");
 		}
 		fprintf(stdout, "\n");
 	} else
 		ctl_io_error_print(io, NULL, stderr);
 
 bailout:
 	ctl_scsi_free_io(io);
 
 	if (dataptr != NULL)
 		free(dataptr);
 
 	return (retval);
 }
 
 static int
 cctl_persistent_reserve_out(int fd, int lun, int iid,
 			    int argc, char **argv, char *combinedopt,
 			    int retry_count)
 {
 	union ctl_io *io;
 	uint32_t datalen;
 	uint64_t key = 0, sa_key = 0;
 	int action = -1, restype = -1;
 	uint8_t *dataptr;
 	int retval;
 	int c;
 
 	retval = 0;
 	dataptr = NULL;
 
 	io = ctl_scsi_alloc_io(iid);
 	if (io == NULL) {
 		warn("%s: can't allocate memory", __func__);
 		return (1);
 	}
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'a':
 			action = strtol(optarg, NULL, 0);
 			break;
 		case 'k':
 			key = strtoull(optarg, NULL, 0);
 			break;
 		case 'r':
 			restype = strtol(optarg, NULL, 0);
 			break;
 		case 's':
 			sa_key = strtoull(optarg, NULL, 0);
 			break;
 		default:
 			break;
 		}
 	}
 	if (action < 0 || action > 5) {
 		warn("action must be specified and in the range: 0-5");
 		retval = 1;
 		goto bailout;
 	}
 
 	if (restype < 0 || restype > 5) {
 		if (action != 0 && action != 5 && action != 3) {
 			warn("'restype' must specified and in the range: 0-5");
 			retval = 1;
 			goto bailout;
 		}
 	}
 
 	datalen = 24;
 	dataptr = (uint8_t *)malloc(datalen);
 	if (dataptr == NULL) {
 		warn("%s: can't allocate %d bytes", __func__, datalen);
 		retval = 1;
 		goto bailout;
 	}
 
 	memset(dataptr, 0, datalen);
 
 	ctl_scsi_persistent_res_out(io,
 				    /*data_ptr*/ dataptr,
 				    /*data_len*/ datalen,
 				    /*action*/   action,
 				    /*type*/     restype,
 				    /*key*/      key,
 				    /*sa key*/   sa_key,
 				    /*tag_type*/ CTL_TAG_SIMPLE,
 				    /*control*/  0);
 
 	io->io_hdr.nexus.targ_lun = lun;
 	io->io_hdr.nexus.initid = iid;
 
 	if (cctl_do_io(fd, retry_count, io, __func__) != 0) {
 		retval = 1;
 		goto bailout;
 	}
 	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) {
 		char scsi_path[40];
 		ctl_scsi_path_string(&io->io_hdr, scsi_path, sizeof(scsi_path));
 		fprintf( stdout, "%sPERSISTENT RESERVE OUT executed "
 			"successfully\n", scsi_path);
 	} else
 		ctl_io_error_print(io, NULL, stderr);
 
 bailout:
 	ctl_scsi_free_io(io);
 
 	if (dataptr != NULL)
 		free(dataptr);
 
 	return (retval);
 }
 
 static int
 cctl_create_lun(int fd, int argc, char **argv, char *combinedopt)
 {
 	struct ctl_lun_req req;
 	int device_type = -1;
 	uint64_t lun_size = 0;
 	uint32_t blocksize = 0, req_lun_id = 0;
 	char *serial_num = NULL;
 	char *device_id = NULL;
 	int lun_size_set = 0, blocksize_set = 0, lun_id_set = 0;
 	char *backend_name = NULL;
 	nvlist_t *option_list;
 	int retval = 0, c;
 
 	option_list = nvlist_create(0);
 	if (option_list == NULL)
 		err(1, "%s: unable to allocate nvlist", __func__);
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'b':
 			backend_name = strdup(optarg);
 			break;
 		case 'B':
 			blocksize = strtoul(optarg, NULL, 0);
 			blocksize_set = 1;
 			break;
 		case 'd':
 			device_id = strdup(optarg);
 			break;
 		case 'l':
 			req_lun_id = strtoul(optarg, NULL, 0);
 			lun_id_set = 1;
 			break;
 		case 'o': {
 			char *tmpstr;
 			char *name, *value;
 
 			tmpstr = strdup(optarg);
 			name = strsep(&tmpstr, "=");
 			if (name == NULL) {
 				warnx("%s: option -o takes \"name=value\""
 				      "argument", __func__);
 				retval = 1;
 				goto bailout;
 			}
 			value = strsep(&tmpstr, "=");
 			if (value == NULL) {
 				warnx("%s: option -o takes \"name=value\""
 				      "argument", __func__);
 				retval = 1;
 				goto bailout;
 			}
 			free(tmpstr);
 			nvlist_add_string(option_list, name, value);
 			break;
 		}
 		case 's':
 			if (strcasecmp(optarg, "auto") != 0) {
 				retval = expand_number(optarg, &lun_size);
 				if (retval != 0) {
 					warn("%s: invalid -s argument",
 					    __func__);
 					retval = 1;
 					goto bailout;
 				}
 			}
 			lun_size_set = 1;
 			break;
 		case 'S':
 			serial_num = strdup(optarg);
 			break;
 		case 't':
 			device_type = strtoul(optarg, NULL, 0);
 			break;
 		default:
 			break;
 		}
 	}
 
 	if (backend_name == NULL) {
 		warnx("%s: backend name (-b) must be specified", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	bzero(&req, sizeof(req));
 
 	strlcpy(req.backend, backend_name, sizeof(req.backend));
 	req.reqtype = CTL_LUNREQ_CREATE;
 
 	if (blocksize_set != 0)
 		req.reqdata.create.blocksize_bytes = blocksize;
 
 	if (lun_size_set != 0)
 		req.reqdata.create.lun_size_bytes = lun_size;
 
 	if (lun_id_set != 0) {
 		req.reqdata.create.flags |= CTL_LUN_FLAG_ID_REQ;
 		req.reqdata.create.req_lun_id = req_lun_id;
 	}
 
 	req.reqdata.create.flags |= CTL_LUN_FLAG_DEV_TYPE;
 
 	if (device_type != -1)
 		req.reqdata.create.device_type = device_type;
 	else
 		req.reqdata.create.device_type = T_DIRECT;
 
 	if (serial_num != NULL) {
 		strlcpy(req.reqdata.create.serial_num, serial_num,
 			sizeof(req.reqdata.create.serial_num));
 		req.reqdata.create.flags |= CTL_LUN_FLAG_SERIAL_NUM;
 	}
 
 	if (device_id != NULL) {
 		strlcpy(req.reqdata.create.device_id, device_id,
 			sizeof(req.reqdata.create.device_id));
 		req.reqdata.create.flags |= CTL_LUN_FLAG_DEVID;
 	}
 
 	req.args = nvlist_pack(option_list, &req.args_len);
 	if (req.args == NULL) {
 		warn("%s: error packing nvlist", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	retval = ioctl(fd, CTL_LUN_REQ, &req);
 	free(req.args);
 	if (retval == -1) {
 		warn("%s: error issuing CTL_LUN_REQ ioctl", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	switch (req.status) {
 	case CTL_LUN_ERROR:
 		warnx("LUN creation error: %s", req.error_str);
 		retval = 1;
 		goto bailout;
 	case CTL_LUN_WARNING:
 		warnx("LUN creation warning: %s", req.error_str);
 		break;
 	case CTL_LUN_OK:
 		break;
 	default:
 		warnx("unknown LUN creation status: %d", req.status);
 		retval = 1;
 		goto bailout;
 	}
 
 	fprintf(stdout, "LUN created successfully\n");
 	fprintf(stdout, "backend:       %s\n", req.backend);
 	fprintf(stdout, "device type:   %d\n",req.reqdata.create.device_type);
 	fprintf(stdout, "LUN size:      %ju bytes\n",
 		(uintmax_t)req.reqdata.create.lun_size_bytes);
 	fprintf(stdout, "blocksize      %u bytes\n",
 		req.reqdata.create.blocksize_bytes);
 	fprintf(stdout, "LUN ID:        %d\n", req.reqdata.create.req_lun_id);
 	fprintf(stdout, "Serial Number: %s\n", req.reqdata.create.serial_num);
 	fprintf(stdout, "Device ID:     %s\n", req.reqdata.create.device_id);
 
 bailout:
 	nvlist_destroy(option_list);
 	return (retval);
 }
 
 static int
 cctl_rm_lun(int fd, int argc, char **argv, char *combinedopt)
 {
 	struct ctl_lun_req req;
 	uint32_t lun_id = 0;
 	int lun_id_set = 0;
 	char *backend_name = NULL;
 	nvlist_t *option_list;
 	int retval = 0, c;
 
 	option_list = nvlist_create(0);
 	if (option_list == NULL)
 		err(1, "%s: unable to allocate nvlist", __func__);
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'b':
 			backend_name = strdup(optarg);
 			break;
 		case 'l':
 			lun_id = strtoul(optarg, NULL, 0);
 			lun_id_set = 1;
 			break;
 		case 'o': {
 			char *tmpstr;
 			char *name, *value;
 
 			tmpstr = strdup(optarg);
 			name = strsep(&tmpstr, "=");
 			if (name == NULL) {
 				warnx("%s: option -o takes \"name=value\""
 				      "argument", __func__);
 				retval = 1;
 				goto bailout;
 			}
 			value = strsep(&tmpstr, "=");
 			if (value == NULL) {
 				warnx("%s: option -o takes \"name=value\""
 				      "argument", __func__);
 				retval = 1;
 				goto bailout;
 			}
 			free(tmpstr);
 			nvlist_add_string(option_list, name, value);
 			break;
 		}
 		default:
 			break;
 		}
 	}
 
 	if (backend_name == NULL)
 		errx(1, "%s: backend name (-b) must be specified", __func__);
 
 	if (lun_id_set == 0)
 		errx(1, "%s: LUN id (-l) must be specified", __func__);
 
 	bzero(&req, sizeof(req));
 
 	strlcpy(req.backend, backend_name, sizeof(req.backend));
 	req.reqtype = CTL_LUNREQ_RM;
 
 	req.reqdata.rm.lun_id = lun_id;
 		
 	req.args = nvlist_pack(option_list, &req.args_len);
 	if (req.args == NULL) {
 		warn("%s: error packing nvlist", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	retval = ioctl(fd, CTL_LUN_REQ, &req);
 	free(req.args);
 	if (retval == -1) {
 		warn("%s: error issuing CTL_LUN_REQ ioctl", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	switch (req.status) {
 	case CTL_LUN_ERROR:
 		warnx("LUN removal error: %s", req.error_str);
 		retval = 1;
 		goto bailout;
 	case CTL_LUN_WARNING:
 		warnx("LUN removal warning: %s", req.error_str);
 		break;
 	case CTL_LUN_OK:
 		break;
 	default:
 		warnx("unknown LUN removal status: %d", req.status);
 		retval = 1;
 		goto bailout;
 	}
 
 	printf("LUN %d removed successfully\n", lun_id);
 
 bailout:
 	nvlist_destroy(option_list);
 	return (retval);
 }
 
 static int
 cctl_modify_lun(int fd, int argc, char **argv, char *combinedopt)
 {
 	struct ctl_lun_req req;
 	uint64_t lun_size = 0;
 	uint32_t lun_id = 0;
 	int lun_id_set = 0, lun_size_set = 0;
 	char *backend_name = NULL;
 	nvlist_t *option_list;
 	int retval = 0, c;
 
 	option_list = nvlist_create(0);
 	if (option_list == NULL)
 		err(1, "%s: unable to allocate nvlist", __func__);
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'b':
 			backend_name = strdup(optarg);
 			break;
 		case 'l':
 			lun_id = strtoul(optarg, NULL, 0);
 			lun_id_set = 1;
 			break;
 		case 'o': {
 			char *tmpstr;
 			char *name, *value;
 
 			tmpstr = strdup(optarg);
 			name = strsep(&tmpstr, "=");
 			if (name == NULL) {
 				warnx("%s: option -o takes \"name=value\""
 				      "argument", __func__);
 				retval = 1;
 				goto bailout;
 			}
 			value = strsep(&tmpstr, "=");
 			if (value == NULL) {
 				warnx("%s: option -o takes \"name=value\""
 				      "argument", __func__);
 				retval = 1;
 				goto bailout;
 			}
 			free(tmpstr);
 			nvlist_add_string(option_list, name, value);
 			break;
 		}
 		case 's':
 			if (strcasecmp(optarg, "auto") != 0) {
 				retval = expand_number(optarg, &lun_size);
 				if (retval != 0) {
 					warn("%s: invalid -s argument",
 					    __func__);
 					retval = 1;
 					goto bailout;
 				}
 			}
 			lun_size_set = 1;
 			break;
 		default:
 			break;
 		}
 	}
 
 	if (backend_name == NULL)
 		errx(1, "%s: backend name (-b) must be specified", __func__);
 
 	if (lun_id_set == 0)
 		errx(1, "%s: LUN id (-l) must be specified", __func__);
 
 	if (lun_size_set == 0 && nvlist_empty(option_list))
 		errx(1, "%s: size (-s) or options (-o) must be specified",
 		    __func__);
 
 	bzero(&req, sizeof(req));
 
 	strlcpy(req.backend, backend_name, sizeof(req.backend));
 	req.reqtype = CTL_LUNREQ_MODIFY;
 
 	req.reqdata.modify.lun_id = lun_id;
 	req.reqdata.modify.lun_size_bytes = lun_size;
 
 	req.args = nvlist_pack(option_list, &req.args_len);
 	if (req.args == NULL) {
 		warn("%s: error packing nvlist", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	retval = ioctl(fd, CTL_LUN_REQ, &req);
 	free(req.args);
 	if (retval == -1) {
 		warn("%s: error issuing CTL_LUN_REQ ioctl", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	switch (req.status) {
 	case CTL_LUN_ERROR:
 		warnx("LUN modification error: %s", req.error_str);
 		retval = 1;
 		goto bailout;
 	case CTL_LUN_WARNING:
 		warnx("LUN modification warning: %s", req.error_str);
 		break;
 	case CTL_LUN_OK:
 		break;
 	default:
 		warnx("unknown LUN modification status: %d", req.status);
 		retval = 1;
 		goto bailout;
 	}
 
 	printf("LUN %d modified successfully\n", lun_id);
 
 bailout:
 	nvlist_destroy(option_list);
 	return (retval);
 }
 
 struct cctl_islist_conn {
 	int connection_id;
 	char *initiator;
 	char *initiator_addr;
 	char *initiator_alias;
 	char *target;
 	char *target_alias;
 	char *header_digest;
 	char *data_digest;
 	char *max_recv_data_segment_length;
 	char *max_send_data_segment_length;
 	char *max_burst_length;
 	char *first_burst_length;
 	char *offload;
 	int immediate_data;
 	int iser;
 	STAILQ_ENTRY(cctl_islist_conn) links;
 };
 
 struct cctl_islist_data {
 	int num_conns;
 	STAILQ_HEAD(,cctl_islist_conn) conn_list;
 	struct cctl_islist_conn *cur_conn;
 	int level;
 	struct sbuf *cur_sb[32];
 };
 
 static void
 cctl_islist_start_element(void *user_data, const char *name, const char **attr)
 {
 	int i;
 	struct cctl_islist_data *islist;
 	struct cctl_islist_conn *cur_conn;
 
 	islist = (struct cctl_islist_data *)user_data;
 	cur_conn = islist->cur_conn;
 	islist->level++;
 	if ((u_int)islist->level >= nitems(islist->cur_sb))
 		errx(1, "%s: too many nesting levels, %zd max", __func__,
 		     nitems(islist->cur_sb));
 
 	islist->cur_sb[islist->level] = sbuf_new_auto();
 	if (islist->cur_sb[islist->level] == NULL)
 		err(1, "%s: Unable to allocate sbuf", __func__);
 
 	if (strcmp(name, "connection") == 0) {
 		if (cur_conn != NULL)
 			errx(1, "%s: improper connection element nesting",
 			    __func__);
 
 		cur_conn = calloc(1, sizeof(*cur_conn));
 		if (cur_conn == NULL)
 			err(1, "%s: cannot allocate %zd bytes", __func__,
 			    sizeof(*cur_conn));
 
 		islist->num_conns++;
 		islist->cur_conn = cur_conn;
 
 		STAILQ_INSERT_TAIL(&islist->conn_list, cur_conn, links);
 
 		for (i = 0; attr[i] != NULL; i += 2) {
 			if (strcmp(attr[i], "id") == 0) {
 				cur_conn->connection_id =
 				    strtoull(attr[i+1], NULL, 0);
 			} else {
 				errx(1,
 				    "%s: invalid connection attribute %s = %s",
 				     __func__, attr[i], attr[i+1]);
 			}
 		}
 	}
 }
 
 static void
 cctl_islist_end_element(void *user_data, const char *name)
 {
 	struct cctl_islist_data *islist;
 	struct cctl_islist_conn *cur_conn;
 	char *str;
 
 	islist = (struct cctl_islist_data *)user_data;
 	cur_conn = islist->cur_conn;
 
 	if ((cur_conn == NULL)
 	 && (strcmp(name, "ctlislist") != 0))
 		errx(1, "%s: cur_conn == NULL! (name = %s)", __func__, name);
 
 	if (islist->cur_sb[islist->level] == NULL)
 		errx(1, "%s: no valid sbuf at level %d (name %s)", __func__,
 		     islist->level, name);
 
 	sbuf_finish(islist->cur_sb[islist->level]);
 	str = strdup(sbuf_data(islist->cur_sb[islist->level]));
 	if (str == NULL)
 		err(1, "%s can't allocate %zd bytes for string", __func__,
 		    sbuf_len(islist->cur_sb[islist->level]));
 
 	sbuf_delete(islist->cur_sb[islist->level]);
 	islist->cur_sb[islist->level] = NULL;
 	islist->level--;
 
 	if (strcmp(name, "initiator") == 0) {
 		cur_conn->initiator = str;
 		str = NULL;
 	} else if (strcmp(name, "initiator_addr") == 0) {
 		cur_conn->initiator_addr = str;
 		str = NULL;
 	} else if (strcmp(name, "initiator_alias") == 0) {
 		cur_conn->initiator_alias = str;
 		str = NULL;
 	} else if (strcmp(name, "target") == 0) {
 		cur_conn->target = str;
 		str = NULL;
 	} else if (strcmp(name, "target_alias") == 0) {
 		cur_conn->target_alias = str;
 		str = NULL;
 	} else if (strcmp(name, "target_portal_group_tag") == 0) {
 	} else if (strcmp(name, "header_digest") == 0) {
 		cur_conn->header_digest = str;
 		str = NULL;
 	} else if (strcmp(name, "data_digest") == 0) {
 		cur_conn->data_digest = str;
 		str = NULL;
 	} else if (strcmp(name, "max_recv_data_segment_length") == 0) {
 		cur_conn->max_recv_data_segment_length = str;
 		str = NULL;
 	} else if (strcmp(name, "max_send_data_segment_length") == 0) {
 		cur_conn->max_send_data_segment_length = str;
 		str = NULL;
 	} else if (strcmp(name, "max_burst_length") == 0) {
 		cur_conn->max_burst_length = str;
 		str = NULL;
 	} else if (strcmp(name, "first_burst_length") == 0) {
 		cur_conn->first_burst_length = str;
 		str = NULL;
 	} else if (strcmp(name, "offload") == 0) {
 		cur_conn->offload = str;
 		str = NULL;
 	} else if (strcmp(name, "immediate_data") == 0) {
 		cur_conn->immediate_data = atoi(str);
 	} else if (strcmp(name, "iser") == 0) {
 		cur_conn->iser = atoi(str);
 	} else if (strcmp(name, "connection") == 0) {
 		islist->cur_conn = NULL;
 	} else if (strcmp(name, "ctlislist") == 0) {
 		/* Nothing. */
 	} else {
 		/*
 		 * Unknown element; ignore it for forward compatibility.
 		 */
 	}
 
 	free(str);
 }
 
 static void
 cctl_islist_char_handler(void *user_data, const XML_Char *str, int len)
 {
 	struct cctl_islist_data *islist;
 
 	islist = (struct cctl_islist_data *)user_data;
 
 	sbuf_bcat(islist->cur_sb[islist->level], str, len);
 }
 
 static int
 cctl_islist(int fd, int argc, char **argv, char *combinedopt)
 {
 	struct ctl_iscsi req;
 	struct cctl_islist_data islist;
 	struct cctl_islist_conn *conn;
 	XML_Parser parser;
 	char *conn_str;
 	int conn_len;
 	int dump_xml = 0;
 	int c, retval, verbose = 0;
 
 	retval = 0;
 	conn_len = 4096;
 
 	bzero(&islist, sizeof(islist));
 	STAILQ_INIT(&islist.conn_list);
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'v':
 			verbose = 1;
 			break;
 		case 'x':
 			dump_xml = 1;
 			break;
 		default:
 			break;
 		}
 	}
 
 retry:
 	conn_str = malloc(conn_len);
 
 	bzero(&req, sizeof(req));
 	req.type = CTL_ISCSI_LIST;
 	req.data.list.alloc_len = conn_len;
 	req.data.list.conn_xml = conn_str;
 
 	if (ioctl(fd, CTL_ISCSI, &req) == -1) {
 		warn("%s: error issuing CTL_ISCSI ioctl", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	if (req.status == CTL_ISCSI_ERROR) {
 		warnx("%s: error returned from CTL_ISCSI ioctl:\n%s",
 		      __func__, req.error_str);
 	} else if (req.status == CTL_ISCSI_LIST_NEED_MORE_SPACE) {
 		conn_len = conn_len << 1;
 		goto retry;
 	}
 
 	if (dump_xml != 0) {
 		printf("%s", conn_str);
 		goto bailout;
 	}
 
 	parser = XML_ParserCreate(NULL);
 	if (parser == NULL) {
 		warn("%s: Unable to create XML parser", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	XML_SetUserData(parser, &islist);
 	XML_SetElementHandler(parser, cctl_islist_start_element,
 	    cctl_islist_end_element);
 	XML_SetCharacterDataHandler(parser, cctl_islist_char_handler);
 
 	retval = XML_Parse(parser, conn_str, strlen(conn_str), 1);
 	if (retval != 1) {
 		warnx("%s: Unable to parse XML: Error %d", __func__,
 		    XML_GetErrorCode(parser));
 		XML_ParserFree(parser);
 		retval = 1;
 		goto bailout;
 	}
 	retval = 0;
 	XML_ParserFree(parser);
 
 	if (verbose != 0) {
 		STAILQ_FOREACH(conn, &islist.conn_list, links) {
 			printf("%-25s %d\n", "Session ID:", conn->connection_id);
 			printf("%-25s %s\n", "Initiator name:", conn->initiator);
 			printf("%-25s %s\n", "Initiator portal:", conn->initiator_addr);
 			printf("%-25s %s\n", "Initiator alias:", conn->initiator_alias);
 			printf("%-25s %s\n", "Target name:", conn->target);
 			printf("%-25s %s\n", "Target alias:", conn->target_alias);
 			printf("%-25s %s\n", "Header digest:", conn->header_digest);
 			printf("%-25s %s\n", "Data digest:", conn->data_digest);
 			printf("%-25s %s\n", "MaxRecvDataSegmentLength:", conn->max_recv_data_segment_length);
 			printf("%-25s %s\n", "MaxSendDataSegmentLength:", conn->max_send_data_segment_length);
 			printf("%-25s %s\n", "MaxBurstLen:", conn->max_burst_length);
 			printf("%-25s %s\n", "FirstBurstLen:", conn->first_burst_length);
 			printf("%-25s %s\n", "ImmediateData:", conn->immediate_data ? "Yes" : "No");
 			printf("%-25s %s\n", "iSER (RDMA):", conn->iser ? "Yes" : "No");
 			printf("%-25s %s\n", "Offload driver:", conn->offload);
 			printf("\n");
 		}
 	} else {
 		printf("%4s %-16s %-36s %-36s\n", "ID", "Portal", "Initiator name",
 		    "Target name");
 		STAILQ_FOREACH(conn, &islist.conn_list, links) {
 			printf("%4u %-16s %-36s %-36s\n",
 			    conn->connection_id, conn->initiator_addr, conn->initiator,
 			    conn->target);
 		}
 	}
 bailout:
 	free(conn_str);
 
 	return (retval);
 }
 
 static int
 cctl_islogout(int fd, int argc, char **argv, char *combinedopt)
 {
 	struct ctl_iscsi req;
 	int retval = 0, c;
 	int all = 0, connection_id = -1, nargs = 0;
 	char *initiator_name = NULL, *initiator_addr = NULL;
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'a':
 			all = 1;
 			nargs++;
 			break;
 		case 'c':
 			connection_id = strtoul(optarg, NULL, 0);
 			nargs++;
 			break;
 		case 'i':
 			initiator_name = strdup(optarg);
 			if (initiator_name == NULL)
 				err(1, "%s: strdup", __func__);
 			nargs++;
 			break;
 		case 'p':
 			initiator_addr = strdup(optarg);
 			if (initiator_addr == NULL)
 				err(1, "%s: strdup", __func__);
 			nargs++;
 			break;
 		default:
 			break;
 		}
 	}
 
 	if (nargs == 0)
 		errx(1, "%s: either -a, -c, -i, or -p must be specified",
 		    __func__);
 	if (nargs > 1)
 		errx(1, "%s: only one of -a, -c, -i, or -p may be specified",
 		    __func__);
 
 	bzero(&req, sizeof(req));
 	req.type = CTL_ISCSI_LOGOUT;
 	req.data.logout.connection_id = connection_id;
 	if (initiator_addr != NULL)
 		strlcpy(req.data.logout.initiator_addr,
 		    initiator_addr, sizeof(req.data.logout.initiator_addr));
 	if (initiator_name != NULL)
 		strlcpy(req.data.logout.initiator_name,
 		    initiator_name, sizeof(req.data.logout.initiator_name));
 	if (all != 0)
 		req.data.logout.all = 1;
 
 	if (ioctl(fd, CTL_ISCSI, &req) == -1) {
 		warn("%s: error issuing CTL_ISCSI ioctl", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	if (req.status != CTL_ISCSI_OK) {
 		warnx("%s: error returned from CTL iSCSI logout request:\n%s",
 		      __func__, req.error_str);
 		retval = 1;
 		goto bailout;
 	}
 
 	printf("iSCSI logout requests submitted\n");
 
 bailout:
 	return (retval);
 }
 
 static int
 cctl_isterminate(int fd, int argc, char **argv, char *combinedopt)
 {
 	struct ctl_iscsi req;
 	int retval = 0, c;
 	int all = 0, connection_id = -1, nargs = 0;
 	char *initiator_name = NULL, *initiator_addr = NULL;
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'a':
 			all = 1;
 			nargs++;
 			break;
 		case 'c':
 			connection_id = strtoul(optarg, NULL, 0);
 			nargs++;
 			break;
 		case 'i':
 			initiator_name = strdup(optarg);
 			if (initiator_name == NULL)
 				err(1, "%s: strdup", __func__);
 			nargs++;
 			break;
 		case 'p':
 			initiator_addr = strdup(optarg);
 			if (initiator_addr == NULL)
 				err(1, "%s: strdup", __func__);
 			nargs++;
 			break;
 		default:
 			break;
 		}
 	}
 
 	if (nargs == 0)
 		errx(1, "%s: either -a, -c, -i, or -p must be specified",
 		    __func__);
 	if (nargs > 1)
 		errx(1, "%s: only one of -a, -c, -i, or -p may be specified",
 		    __func__);
 
 	bzero(&req, sizeof(req));
 	req.type = CTL_ISCSI_TERMINATE;
 	req.data.terminate.connection_id = connection_id;
 	if (initiator_addr != NULL)
 		strlcpy(req.data.terminate.initiator_addr,
 		    initiator_addr, sizeof(req.data.terminate.initiator_addr));
 	if (initiator_name != NULL)
 		strlcpy(req.data.terminate.initiator_name,
 		    initiator_name, sizeof(req.data.terminate.initiator_name));
 	if (all != 0)
 		req.data.terminate.all = 1;
 
 	if (ioctl(fd, CTL_ISCSI, &req) == -1) {
 		warn("%s: error issuing CTL_ISCSI ioctl", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	if (req.status != CTL_ISCSI_OK) {
 		warnx("%s: error returned from CTL iSCSI connection "
 		    "termination request:\n%s", __func__, req.error_str);
 		retval = 1;
 		goto bailout;
 	}
 
 	printf("iSCSI connections terminated\n");
 
 bailout:
 	return (retval);
 }
 
 /*
  * Name/value pair used for per-LUN attributes.
  */
 struct cctl_lun_nv {
 	char *name;
 	char *value;
 	STAILQ_ENTRY(cctl_lun_nv) links;
 };
 
 /*
  * Backend LUN information.
  */
 struct cctl_lun {
 	uint64_t lun_id;
 	char *backend_type;
 	uint64_t size_blocks;
 	uint32_t blocksize;
 	char *serial_number;
 	char *device_id;
 	STAILQ_HEAD(,cctl_lun_nv) attr_list;
 	STAILQ_ENTRY(cctl_lun) links;
 };
 
 struct cctl_devlist_data {
 	int num_luns;
 	STAILQ_HEAD(,cctl_lun) lun_list;
 	struct cctl_lun *cur_lun;
 	int level;
 	struct sbuf *cur_sb[32];
 };
 
 static void
 cctl_start_element(void *user_data, const char *name, const char **attr)
 {
 	int i;
 	struct cctl_devlist_data *devlist;
 	struct cctl_lun *cur_lun;
 
 	devlist = (struct cctl_devlist_data *)user_data;
 	cur_lun = devlist->cur_lun;
 	devlist->level++;
 	if ((u_int)devlist->level >= nitems(devlist->cur_sb))
 		errx(1, "%s: too many nesting levels, %zd max", __func__,
 		     nitems(devlist->cur_sb));
 
 	devlist->cur_sb[devlist->level] = sbuf_new_auto();
 	if (devlist->cur_sb[devlist->level] == NULL)
 		err(1, "%s: Unable to allocate sbuf", __func__);
 
 	if (strcmp(name, "lun") == 0) {
 		if (cur_lun != NULL)
 			errx(1, "%s: improper lun element nesting", __func__);
 
 		cur_lun = calloc(1, sizeof(*cur_lun));
 		if (cur_lun == NULL)
 			err(1, "%s: cannot allocate %zd bytes", __func__,
 			    sizeof(*cur_lun));
 
 		devlist->num_luns++;
 		devlist->cur_lun = cur_lun;
 
 		STAILQ_INIT(&cur_lun->attr_list);
 		STAILQ_INSERT_TAIL(&devlist->lun_list, cur_lun, links);
 
 		for (i = 0; attr[i] != NULL; i += 2) {
 			if (strcmp(attr[i], "id") == 0) {
 				cur_lun->lun_id = strtoull(attr[i+1], NULL, 0);
 			} else {
 				errx(1, "%s: invalid LUN attribute %s = %s",
 				     __func__, attr[i], attr[i+1]);
 			}
 		}
 	}
 }
 
 static void
 cctl_end_element(void *user_data, const char *name)
 {
 	struct cctl_devlist_data *devlist;
 	struct cctl_lun *cur_lun;
 	char *str;
 
 	devlist = (struct cctl_devlist_data *)user_data;
 	cur_lun = devlist->cur_lun;
 
 	if ((cur_lun == NULL)
 	 && (strcmp(name, "ctllunlist") != 0))
 		errx(1, "%s: cur_lun == NULL! (name = %s)", __func__, name);
 
 	if (devlist->cur_sb[devlist->level] == NULL)
 		errx(1, "%s: no valid sbuf at level %d (name %s)", __func__,
 		     devlist->level, name);
 
 	if (sbuf_finish(devlist->cur_sb[devlist->level]) != 0)
 		err(1, "%s: sbuf_finish", __func__);
 	str = strdup(sbuf_data(devlist->cur_sb[devlist->level]));
 	if (str == NULL)
 		err(1, "%s can't allocate %zd bytes for string", __func__,
 		    sbuf_len(devlist->cur_sb[devlist->level]));
 
 	if (strlen(str) == 0) {
 		free(str);
 		str = NULL;
 	}
 
 	sbuf_delete(devlist->cur_sb[devlist->level]);
 	devlist->cur_sb[devlist->level] = NULL;
 	devlist->level--;
 
 	if (strcmp(name, "backend_type") == 0) {
 		cur_lun->backend_type = str;
 		str = NULL;
 	} else if (strcmp(name, "size") == 0) {
 		cur_lun->size_blocks = strtoull(str, NULL, 0);
 	} else if (strcmp(name, "blocksize") == 0) {
 		cur_lun->blocksize = strtoul(str, NULL, 0);
 	} else if (strcmp(name, "serial_number") == 0) {
 		cur_lun->serial_number = str;
 		str = NULL;
 	} else if (strcmp(name, "device_id") == 0) {
 		cur_lun->device_id = str;
 		str = NULL;
 	} else if (strcmp(name, "lun") == 0) {
 		devlist->cur_lun = NULL;
 	} else if (strcmp(name, "ctllunlist") == 0) {
 		/* Nothing. */
 	} else {
 		struct cctl_lun_nv *nv;
 
 		nv = calloc(1, sizeof(*nv));
 		if (nv == NULL)
 			err(1, "%s: can't allocate %zd bytes for nv pair",
 			    __func__, sizeof(*nv));
 
 		nv->name = strdup(name);
 		if (nv->name == NULL)
 			err(1, "%s: can't allocated %zd bytes for string",
 			    __func__, strlen(name));
 
 		nv->value = str;
 		str = NULL;
 		STAILQ_INSERT_TAIL(&cur_lun->attr_list, nv, links);
 	}
 
 	free(str);
 }
 
 static void
 cctl_char_handler(void *user_data, const XML_Char *str, int len)
 {
 	struct cctl_devlist_data *devlist;
 
 	devlist = (struct cctl_devlist_data *)user_data;
 
 	sbuf_bcat(devlist->cur_sb[devlist->level], str, len);
 }
 
 static int
 cctl_devlist(int fd, int argc, char **argv, char *combinedopt)
 {
 	struct ctl_lun_list list;
 	struct cctl_devlist_data devlist;
 	struct cctl_lun *lun;
 	XML_Parser parser;
 	char *lun_str;
 	int lun_len;
 	int dump_xml = 0;
 	int retval, c;
 	char *backend = NULL;
 	int verbose = 0;
 
 	retval = 0;
 	lun_len = 4096;
 
 	bzero(&devlist, sizeof(devlist));
 	STAILQ_INIT(&devlist.lun_list);
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'b':
 			backend = strdup(optarg);
 			break;
 		case 'v':
 			verbose++;
 			break;
 		case 'x':
 			dump_xml = 1;
 			break;
 		default:
 			break;
 		}
 	}
 
 retry:
 	lun_str = malloc(lun_len);
 
 	bzero(&list, sizeof(list));
 	list.alloc_len = lun_len;
 	list.status = CTL_LUN_LIST_NONE;
 	list.lun_xml = lun_str;
 
 	if (ioctl(fd, CTL_LUN_LIST, &list) == -1) {
 		warn("%s: error issuing CTL_LUN_LIST ioctl", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	if (list.status == CTL_LUN_LIST_ERROR) {
 		warnx("%s: error returned from CTL_LUN_LIST ioctl:\n%s",
 		      __func__, list.error_str);
 	} else if (list.status == CTL_LUN_LIST_NEED_MORE_SPACE) {
 		lun_len = lun_len << 1;
 		goto retry;
 	}
 
 	if (dump_xml != 0) {
 		printf("%s", lun_str);
 		goto bailout;
 	}
 
 	parser = XML_ParserCreate(NULL);
 	if (parser == NULL) {
 		warn("%s: Unable to create XML parser", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	XML_SetUserData(parser, &devlist);
 	XML_SetElementHandler(parser, cctl_start_element, cctl_end_element);
 	XML_SetCharacterDataHandler(parser, cctl_char_handler);
 
 	retval = XML_Parse(parser, lun_str, strlen(lun_str), 1);
 	if (retval != 1) {
 		warnx("%s: Unable to parse XML: Error %d", __func__,
 		    XML_GetErrorCode(parser));
 		XML_ParserFree(parser);
 		retval = 1;
 		goto bailout;
 	}
 	retval = 0;
 	XML_ParserFree(parser);
 
 	printf("LUN Backend  %18s %4s %-16s %-16s\n", "Size (Blocks)", "BS",
 	       "Serial Number", "Device ID");
 	STAILQ_FOREACH(lun, &devlist.lun_list, links) {
 		struct cctl_lun_nv *nv;
 
 		if ((backend != NULL)
 		 && (strcmp(lun->backend_type, backend) != 0))
 			continue;
 
 		printf("%3ju %-8s %18ju %4u %-16s %-16s\n",
 		       (uintmax_t)lun->lun_id,
 		       lun->backend_type, (uintmax_t)lun->size_blocks,
 		       lun->blocksize, lun->serial_number, lun->device_id);
 
 		if (verbose == 0)
 			continue;
 
 		STAILQ_FOREACH(nv, &lun->attr_list, links) {
 			printf("      %s=%s\n", nv->name, nv->value);
 		}
 	}
 bailout:
 	free(lun_str);
 
 	return (retval);
 }
 
 /*
  * Port information.
  */
 struct cctl_port {
 	uint64_t port_id;
 	char *online;
 	char *frontend_type;
 	char *name;
 	int pp, vp;
 	char *target, *port, *lun_map;
 	STAILQ_HEAD(,cctl_lun_nv) init_list;
 	STAILQ_HEAD(,cctl_lun_nv) lun_list;
 	STAILQ_HEAD(,cctl_lun_nv) attr_list;
 	STAILQ_ENTRY(cctl_port) links;
 };
 
 struct cctl_portlist_data {
 	int num_ports;
 	STAILQ_HEAD(,cctl_port) port_list;
 	struct cctl_port *cur_port;
 	int level;
 	uint64_t cur_id;
 	struct sbuf *cur_sb[32];
 };
 
 static void
 cctl_start_pelement(void *user_data, const char *name, const char **attr)
 {
 	int i;
 	struct cctl_portlist_data *portlist;
 	struct cctl_port *cur_port;
 
 	portlist = (struct cctl_portlist_data *)user_data;
 	cur_port = portlist->cur_port;
 	portlist->level++;
 	if ((u_int)portlist->level >= nitems(portlist->cur_sb))
 		errx(1, "%s: too many nesting levels, %zd max", __func__,
 		     nitems(portlist->cur_sb));
 
 	portlist->cur_sb[portlist->level] = sbuf_new_auto();
 	if (portlist->cur_sb[portlist->level] == NULL)
 		err(1, "%s: Unable to allocate sbuf", __func__);
 
 	portlist->cur_id = 0;
 	for (i = 0; attr[i] != NULL; i += 2) {
 		if (strcmp(attr[i], "id") == 0) {
 			portlist->cur_id = strtoull(attr[i+1], NULL, 0);
 			break;
 		}
 	}
 
 	if (strcmp(name, "targ_port") == 0) {
 		if (cur_port != NULL)
 			errx(1, "%s: improper port element nesting", __func__);
 
 		cur_port = calloc(1, sizeof(*cur_port));
 		if (cur_port == NULL)
 			err(1, "%s: cannot allocate %zd bytes", __func__,
 			    sizeof(*cur_port));
 
 		portlist->num_ports++;
 		portlist->cur_port = cur_port;
 
 		STAILQ_INIT(&cur_port->init_list);
 		STAILQ_INIT(&cur_port->lun_list);
 		STAILQ_INIT(&cur_port->attr_list);
 		cur_port->port_id = portlist->cur_id;
 		STAILQ_INSERT_TAIL(&portlist->port_list, cur_port, links);
 	}
 }
 
 static void
 cctl_end_pelement(void *user_data, const char *name)
 {
 	struct cctl_portlist_data *portlist;
 	struct cctl_port *cur_port;
 	char *str;
 
 	portlist = (struct cctl_portlist_data *)user_data;
 	cur_port = portlist->cur_port;
 
 	if ((cur_port == NULL)
 	 && (strcmp(name, "ctlportlist") != 0))
 		errx(1, "%s: cur_port == NULL! (name = %s)", __func__, name);
 
 	if (portlist->cur_sb[portlist->level] == NULL)
 		errx(1, "%s: no valid sbuf at level %d (name %s)", __func__,
 		     portlist->level, name);
 
 	if (sbuf_finish(portlist->cur_sb[portlist->level]) != 0)
 		err(1, "%s: sbuf_finish", __func__);
 	str = strdup(sbuf_data(portlist->cur_sb[portlist->level]));
 	if (str == NULL)
 		err(1, "%s can't allocate %zd bytes for string", __func__,
 		    sbuf_len(portlist->cur_sb[portlist->level]));
 
 	if (strlen(str) == 0) {
 		free(str);
 		str = NULL;
 	}
 
 	sbuf_delete(portlist->cur_sb[portlist->level]);
 	portlist->cur_sb[portlist->level] = NULL;
 	portlist->level--;
 
 	if (strcmp(name, "frontend_type") == 0) {
 		cur_port->frontend_type = str;
 		str = NULL;
 	} else if (strcmp(name, "port_name") == 0) {
 		cur_port->name = str;
 		str = NULL;
 	} else if (strcmp(name, "online") == 0) {
 		cur_port->online = str;
 		str = NULL;
 	} else if (strcmp(name, "physical_port") == 0) {
 		cur_port->pp = strtoull(str, NULL, 0);
 	} else if (strcmp(name, "virtual_port") == 0) {
 		cur_port->vp = strtoull(str, NULL, 0);
 	} else if (strcmp(name, "target") == 0) {
 		cur_port->target = str;
 		str = NULL;
 	} else if (strcmp(name, "port") == 0) {
 		cur_port->port = str;
 		str = NULL;
 	} else if (strcmp(name, "lun_map") == 0) {
 		cur_port->lun_map = str;
 		str = NULL;
 	} else if (strcmp(name, "targ_port") == 0) {
 		portlist->cur_port = NULL;
 	} else if (strcmp(name, "ctlportlist") == 0) {
 		/* Nothing. */
 	} else {
 		struct cctl_lun_nv *nv;
 
 		nv = calloc(1, sizeof(*nv));
 		if (nv == NULL)
 			err(1, "%s: can't allocate %zd bytes for nv pair",
 			    __func__, sizeof(*nv));
 
 		if (strcmp(name, "initiator") == 0 ||
 		    strcmp(name, "lun") == 0)
 			asprintf(&nv->name, "%ju", portlist->cur_id);
 		else
 			nv->name = strdup(name);
 		if (nv->name == NULL)
 			err(1, "%s: can't allocated %zd bytes for string",
 			    __func__, strlen(name));
 
 		nv->value = str;
 		str = NULL;
 		if (strcmp(name, "initiator") == 0)
 			STAILQ_INSERT_TAIL(&cur_port->init_list, nv, links);
 		else if (strcmp(name, "lun") == 0)
 			STAILQ_INSERT_TAIL(&cur_port->lun_list, nv, links);
 		else
 			STAILQ_INSERT_TAIL(&cur_port->attr_list, nv, links);
 	}
 
 	free(str);
 }
 
 static void
 cctl_char_phandler(void *user_data, const XML_Char *str, int len)
 {
 	struct cctl_portlist_data *portlist;
 
 	portlist = (struct cctl_portlist_data *)user_data;
 
 	sbuf_bcat(portlist->cur_sb[portlist->level], str, len);
 }
 
 static int
 cctl_portlist(int fd, int argc, char **argv, char *combinedopt)
 {
 	struct ctl_lun_list list;
 	struct cctl_portlist_data portlist;
 	struct cctl_port *port;
 	XML_Parser parser;
 	char *port_str = NULL;
 	int port_len;
 	int dump_xml = 0;
 	int retval, c;
 	char *frontend = NULL;
 	uint64_t portarg = UINT64_MAX;
 	int verbose = 0, init = 0, lun = 0, quiet = 0;
 
 	retval = 0;
 	port_len = 4096;
 
 	bzero(&portlist, sizeof(portlist));
 	STAILQ_INIT(&portlist.port_list);
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'f':
 			frontend = strdup(optarg);
 			break;
 		case 'i':
 			init++;
 			break;
 		case 'l':
 			lun++;
 			break;
 		case 'p':
 			portarg = strtoll(optarg, NULL, 0);
 			break;
 		case 'q':
 			quiet++;
 			break;
 		case 'v':
 			verbose++;
 			break;
 		case 'x':
 			dump_xml = 1;
 			break;
 		default:
 			break;
 		}
 	}
 
 retry:
 	port_str = (char *)realloc(port_str, port_len);
 
 	bzero(&list, sizeof(list));
 	list.alloc_len = port_len;
 	list.status = CTL_LUN_LIST_NONE;
 	list.lun_xml = port_str;
 
 	if (ioctl(fd, CTL_PORT_LIST, &list) == -1) {
 		warn("%s: error issuing CTL_PORT_LIST ioctl", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	if (list.status == CTL_LUN_LIST_ERROR) {
 		warnx("%s: error returned from CTL_PORT_LIST ioctl:\n%s",
 		      __func__, list.error_str);
 	} else if (list.status == CTL_LUN_LIST_NEED_MORE_SPACE) {
 		port_len = port_len << 1;
 		goto retry;
 	}
 
 	if (dump_xml != 0) {
 		printf("%s", port_str);
 		goto bailout;
 	}
 
 	parser = XML_ParserCreate(NULL);
 	if (parser == NULL) {
 		warn("%s: Unable to create XML parser", __func__);
 		retval = 1;
 		goto bailout;
 	}
 
 	XML_SetUserData(parser, &portlist);
 	XML_SetElementHandler(parser, cctl_start_pelement, cctl_end_pelement);
 	XML_SetCharacterDataHandler(parser, cctl_char_phandler);
 
 	retval = XML_Parse(parser, port_str, strlen(port_str), 1);
 	if (retval != 1) {
 		warnx("%s: Unable to parse XML: Error %d", __func__,
 		    XML_GetErrorCode(parser));
 		XML_ParserFree(parser);
 		retval = 1;
 		goto bailout;
 	}
 	retval = 0;
 	XML_ParserFree(parser);
 
 	if (quiet == 0)
 		printf("Port Online Frontend Name     pp vp\n");
 	STAILQ_FOREACH(port, &portlist.port_list, links) {
 		struct cctl_lun_nv *nv;
 
 		if ((frontend != NULL)
 		 && (strcmp(port->frontend_type, frontend) != 0))
 			continue;
 
 		if ((portarg != UINT64_MAX) && (portarg != port->port_id))
 			continue;
 
 		printf("%-4ju %-6s %-8s %-8s %-2d %-2d %s\n",
 		    (uintmax_t)port->port_id, port->online,
 		    port->frontend_type, port->name, port->pp, port->vp,
 		    port->port ? port->port : "");
 
 		if (init || verbose) {
 			if (port->target)
 				printf("  Target: %s\n", port->target);
 			STAILQ_FOREACH(nv, &port->init_list, links) {
 				printf("  Initiator %s: %s\n",
 				    nv->name, nv->value);
 			}
 		}
 
 		if (lun || verbose) {
 			if (port->lun_map) {
 				STAILQ_FOREACH(nv, &port->lun_list, links)
 					printf("  LUN %s: %s\n",
 					    nv->name, nv->value);
 				if (STAILQ_EMPTY(&port->lun_list))
 					printf("  No LUNs mapped\n");
 			} else
 				printf("  All LUNs mapped\n");
 		}
 
 		if (verbose) {
 			STAILQ_FOREACH(nv, &port->attr_list, links) {
 				printf("      %s=%s\n", nv->name, nv->value);
 			}
 		}
 	}
 bailout:
 	free(port_str);
 
 	return (retval);
 }
 
 static int
 cctl_lunmap(int fd, int argc, char **argv, char *combinedopt)
 {
 	struct ctl_lun_map lm;
 	int retval = 0, c;
 
 	retval = 0;
 	lm.port = UINT32_MAX;
 	lm.plun = UINT32_MAX;
 	lm.lun = UINT32_MAX;
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'p':
 			lm.port = strtoll(optarg, NULL, 0);
 			break;
 		case 'l':
 			lm.plun = strtoll(optarg, NULL, 0);
 			break;
 		case 'L':
 			lm.lun = strtoll(optarg, NULL, 0);
 			break;
 		default:
 			break;
 		}
 	}
 
 	if (ioctl(fd, CTL_LUN_MAP, &lm) == -1) {
 		warn("%s: error issuing CTL_LUN_MAP ioctl", __func__);
 		retval = 1;
 	}
 
 	return (retval);
 }
 
+struct cctl_nvlist_conn {
+	int connection_id;
+	char *hostnqn;
+	char *subnqn;
+	int trtype;
+	STAILQ_ENTRY(cctl_nvlist_conn) links;
+};
+
+struct cctl_nvlist_data {
+	int num_conns;
+	STAILQ_HEAD(,cctl_nvlist_conn) conn_list;
+	struct cctl_nvlist_conn *cur_conn;
+	u_int level;
+	struct sbuf *cur_sb[32];
+};
+
+static void
+cctl_nvlist_start_element(void *user_data, const char *name, const char **attr)
+{
+	int i;
+	struct cctl_nvlist_data *nvlist;
+	struct cctl_nvlist_conn *cur_conn;
+
+	nvlist = (struct cctl_nvlist_data *)user_data;
+	cur_conn = nvlist->cur_conn;
+	nvlist->level++;
+	if ((u_int)nvlist->level >= nitems(nvlist->cur_sb))
+		errx(1, "%s: too many nesting levels, %zd max", __func__,
+		    nitems(nvlist->cur_sb));
+
+	nvlist->cur_sb[nvlist->level] = sbuf_new_auto();
+	if (nvlist->cur_sb[nvlist->level] == NULL)
+		err(1, "%s: Unable to allocate sbuf", __func__);
+
+	if (strcmp(name, "connection") == 0) {
+		if (cur_conn != NULL)
+			errx(1, "%s: improper connection element nesting",
+			    __func__);
+
+		cur_conn = calloc(1, sizeof(*cur_conn));
+		if (cur_conn == NULL)
+			err(1, "%s: cannot allocate %zd bytes", __func__,
+			    sizeof(*cur_conn));
+
+		nvlist->num_conns++;
+		nvlist->cur_conn = cur_conn;
+
+		STAILQ_INSERT_TAIL(&nvlist->conn_list, cur_conn, links);
+
+		for (i = 0; attr[i] != NULL; i += 2) {
+			if (strcmp(attr[i], "id") == 0) {
+				cur_conn->connection_id =
+				    strtoull(attr[i+1], NULL, 0);
+			} else {
+				errx(1,
+				    "%s: invalid connection attribute %s = %s",
+				     __func__, attr[i], attr[i+1]);
+			}
+		}
+	}
+}
+
+static void
+cctl_nvlist_end_element(void *user_data, const char *name)
+{
+	struct cctl_nvlist_data *nvlist;
+	struct cctl_nvlist_conn *cur_conn;
+	char *str;
+
+	nvlist = (struct cctl_nvlist_data *)user_data;
+	cur_conn = nvlist->cur_conn;
+
+	if ((cur_conn == NULL) && (strcmp(name, "ctlnvmflist") != 0))
+		errx(1, "%s: cur_conn == NULL! (name = %s)", __func__, name);
+
+	if (nvlist->cur_sb[nvlist->level] == NULL)
+		errx(1, "%s: no valid sbuf at level %d (name %s)", __func__,
+		     nvlist->level, name);
+
+	sbuf_finish(nvlist->cur_sb[nvlist->level]);
+	str = strdup(sbuf_data(nvlist->cur_sb[nvlist->level]));
+	if (str == NULL)
+		err(1, "%s can't allocate %zd bytes for string", __func__,
+		    sbuf_len(nvlist->cur_sb[nvlist->level]));
+
+	sbuf_delete(nvlist->cur_sb[nvlist->level]);
+	nvlist->cur_sb[nvlist->level] = NULL;
+	nvlist->level--;
+
+	if (strcmp(name, "hostnqn") == 0) {
+		cur_conn->hostnqn = str;
+		str = NULL;
+	} else if (strcmp(name, "subnqn") == 0) {
+		cur_conn->subnqn = str;
+		str = NULL;
+	} else if (strcmp(name, "trtype") == 0) {
+		cur_conn->trtype = atoi(str);
+		str = NULL;
+	} else if (strcmp(name, "connection") == 0) {
+		nvlist->cur_conn = NULL;
+	} else if (strcmp(name, "ctlnvmflist") == 0) {
+		/* Nothing. */
+	} else {
+		/*
+		 * Unknown element; ignore it for forward compatibility.
+		 */
+	}
+
+	free(str);
+}
+
+static void
+cctl_nvlist_char_handler(void *user_data, const XML_Char *str, int len)
+{
+	struct cctl_nvlist_data *nvlist;
+
+	nvlist = (struct cctl_nvlist_data *)user_data;
+
+	sbuf_bcat(nvlist->cur_sb[nvlist->level], str, len);
+}
+
+static const char *
+nvmf_transport_descr(u_int trtype)
+{
+	static char buf[16];
+
+	switch (trtype) {
+	case NVMF_TRTYPE_RDMA:
+		return ("RDMA");
+	case NVMF_TRTYPE_FC:
+		return ("Fibre Channel");
+	case NVMF_TRTYPE_TCP:
+		return ("TCP");
+	default:
+		snprintf(buf, sizeof(buf), "%#x", trtype);
+		return (buf);
+	}
+}
+
+static int
+cctl_nvlist(int fd, int argc, char **argv, char *combinedopt)
+{
+	struct ctl_nvmf req;
+	struct cctl_nvlist_data nvlist;
+	struct cctl_nvlist_conn *conn;
+	XML_Parser parser;
+	char *conn_str;
+	int conn_len;
+	int dump_xml = 0;
+	int c, retval, verbose = 0;
+
+	retval = 0;
+	conn_len = 4096;
+
+	bzero(&nvlist, sizeof(nvlist));
+	STAILQ_INIT(&nvlist.conn_list);
+
+	while ((c = getopt(argc, argv, combinedopt)) != -1) {
+		switch (c) {
+		case 'v':
+			verbose = 1;
+			break;
+		case 'x':
+			dump_xml = 1;
+			break;
+		default:
+			break;
+		}
+	}
+
+retry:
+	conn_str = malloc(conn_len);
+
+	bzero(&req, sizeof(req));
+	req.type = CTL_NVMF_LIST;
+	req.data.list.alloc_len = conn_len;
+	req.data.list.conn_xml = conn_str;
+
+	if (ioctl(fd, CTL_NVMF, &req) == -1) {
+		warn("%s: error issuing CTL_NVMF ioctl", __func__);
+		retval = 1;
+		goto bailout;
+	}
+
+	if (req.status == CTL_NVMF_ERROR) {
+		warnx("%s: error returned from CTL_NVMF ioctl:\n%s",
+		      __func__, req.error_str);
+	} else if (req.status == CTL_NVMF_LIST_NEED_MORE_SPACE) {
+		conn_len = conn_len << 1;
+		goto retry;
+	}
+
+	if (dump_xml != 0) {
+		printf("%s", conn_str);
+		goto bailout;
+	}
+
+	parser = XML_ParserCreate(NULL);
+	if (parser == NULL) {
+		warn("%s: Unable to create XML parser", __func__);
+		retval = 1;
+		goto bailout;
+	}
+
+	XML_SetUserData(parser, &nvlist);
+	XML_SetElementHandler(parser, cctl_nvlist_start_element,
+	    cctl_nvlist_end_element);
+	XML_SetCharacterDataHandler(parser, cctl_nvlist_char_handler);
+
+	retval = XML_Parse(parser, conn_str, strlen(conn_str), 1);
+	if (retval != 1) {
+		warnx("%s: Unable to parse XML: Error %d", __func__,
+		    XML_GetErrorCode(parser));
+		XML_ParserFree(parser);
+		retval = 1;
+		goto bailout;
+	}
+	retval = 0;
+	XML_ParserFree(parser);
+
+	if (verbose != 0) {
+		STAILQ_FOREACH(conn, &nvlist.conn_list, links) {
+			printf("%-25s %d\n", "Controller ID:", conn->connection_id);
+			printf("%-25s %s\n", "Host NQN:", conn->hostnqn);
+			printf("%-25s %s\n", "Subsystem NQN:", conn->subnqn);
+			printf("%-25s %s\n", "Transport:",
+			    nvmf_transport_descr(conn->trtype));
+			printf("\n");
+		}
+	} else {
+		printf("%4s %-16s %-36s %-36s\n", "ID", "Transport", "HostNQN",
+		    "SubNQN");
+		STAILQ_FOREACH(conn, &nvlist.conn_list, links) {
+			printf("%4u %-16s %-36s %-36s\n",
+			    conn->connection_id,
+			    nvmf_transport_descr(conn->trtype),
+			    conn->hostnqn, conn->subnqn);
+		}
+	}
+bailout:
+	free(conn_str);
+
+	return (retval);
+}
+
 void
 usage(int error)
 {
 	fprintf(error ? stderr : stdout,
 "Usage:\n"
 "Primary commands:\n"
 "         ctladm tur         [dev_id][general options]\n"
 "         ctladm inquiry     [dev_id][general options]\n"
 "         ctladm devid       [dev_id][general options]\n"
 "         ctladm reqsense    [dev_id][general options]\n"
 "         ctladm reportluns  [dev_id][general options]\n"
 "         ctladm read        [dev_id][general options] <-l lba> <-d len>\n"
 "                            <-f file|-> <-b blocksize> [-c cdbsize][-N]\n"
 "         ctladm write       [dev_id][general options] <-l lba> <-d len>\n"
 "                            <-f file|-> <-b blocksize> [-c cdbsize][-N]\n"
 "         ctladm readcap     [dev_id][general options] [-c cdbsize]\n"
 "         ctladm modesense   [dev_id][general options] <-m page|-l> [-P pc]\n"
 "                            [-d] [-S subpage] [-c cdbsize]\n"
 "         ctladm prin        [dev_id][general options] <-a action>\n"
 "         ctladm prout       [dev_id][general options] <-a action>\n"
 "                            <-r restype] [-k key] [-s sa_key]\n"
 "         ctladm rtpg        [dev_id][general options]\n"
 "         ctladm start       [dev_id][general options] [-i] [-o]\n"
 "         ctladm stop        [dev_id][general options] [-i] [-o]\n"
 "         ctladm synccache   [dev_id][general options] [-l lba]\n"
 "                            [-b blockcount] [-r] [-i] [-c cdbsize]\n"
 "         ctladm create      <-b backend> [-B blocksize] [-d device_id]\n"
 "                            [-l lun_id] [-o name=value] [-s size_bytes]\n"
 "                            [-S serial_num] [-t dev_type]\n"
 "         ctladm remove      <-b backend> <-l lun_id> [-o name=value]\n"
 "         ctladm modify      <-b backend> <-l lun_id> <-s size_bytes>\n"
 "         ctladm devlist     [-b backend] [-v] [-x]\n"
 "         ctladm lunlist\n"
 "         ctladm lunmap      -p targ_port [-l pLUN] [-L cLUN]\n"
 "         ctladm delay       [dev_id] <-l datamove|done> [-T oneshot|cont]\n"
 "                            [-t secs]\n"
 "         ctladm inject      [dev_id] <-i action> <-p pattern> [-r lba,len]\n"
 "                            [-s len fmt [args]] [-c] [-d delete_id]\n"
 "         ctladm port        <-o <on|off> | [-w wwnn][-W wwpn]>\n"
 "                            [-p targ_port] [-t port_type]\n"
 "                            <-c> [-d driver] [-O name=value]\n"
 "                            <-r> <-p targ_port>\n"
 "         ctladm portlist    [-f frontend] [-i] [-p targ_port] [-q] [-v] [-x]\n"
 "         ctladm islist      [-v | -x]\n"
 "         ctladm islogout    <-a | -c connection-id | -i name | -p portal>\n"
 "         ctladm isterminate <-a | -c connection-id | -i name | -p portal>\n"
+"         ctladm nvlist      [-v | -x]\n"
 "         ctladm dumpooa\n"
 "         ctladm dumpstructs\n"
 "         ctladm help\n"
 "General Options:\n"
 "-I intiator_id           : defaults to 7, used to change the initiator id\n"
 "-C retries               : specify the number of times to retry this command\n"
 "-D devicename            : specify the device to operate on\n"
 "                         : (default is %s)\n"
 "read/write options:\n"
 "-l lba                   : logical block address\n"
 "-d len                   : read/write length, in blocks\n"
 "-f file|-                : write/read data to/from file or stdout/stdin\n"
 "-b blocksize             : block size, in bytes\n"
 "-c cdbsize               : specify minimum cdb size: 6, 10, 12 or 16\n"
 "-N                       : do not copy data to/from userland\n"
 "readcapacity options:\n"
 "-c cdbsize               : specify minimum cdb size: 10 or 16\n"
 "modesense options:\n"
 "-m page                  : specify the mode page to view\n"
 "-l                       : request a list of supported pages\n"
 "-P pc                    : specify the page control value: 0-3 (current,\n"
 "                           changeable, default, saved, respectively)\n"
 "-d                       : disable block descriptors for mode sense\n"
 "-S subpage               : specify a subpage\n"
 "-c cdbsize               : specify minimum cdb size: 6 or 10\n"
 "persistent reserve in options:\n"
 "-a action                : specify the action value: 0-2 (read key, read\n"
 "                           reservation, read capabilities, respectively)\n"
 "persistent reserve out options:\n"
 "-a action                : specify the action value: 0-5 (register, reserve,\n"
 "                           release, clear, preempt, register and ignore)\n"
 "-k key                   : key value\n"
 "-s sa_key                : service action value\n"
 "-r restype               : specify the reservation type: 0-5(wr ex, ex ac,\n"
 "                           wr ex ro, ex ac ro, wr ex ar, ex ac ar)\n"
 "start/stop options:\n"
 "-i                       : set the immediate bit (CTL does not support this)\n"
 "-o                       : set the on/offline bit\n"
 "synccache options:\n"
 "-l lba                   : set the starting LBA\n"
 "-b blockcount            : set the length to sync in blocks\n"
 "-r                       : set the relative addressing bit\n"
 "-i                       : set the immediate bit\n"
 "-c cdbsize               : specify minimum cdb size: 10 or 16\n"
 "create options:\n"
 "-b backend               : backend name (\"block\", \"ramdisk\", etc.)\n"
 "-B blocksize             : LUN blocksize in bytes (some backends)\n"
 "-d device_id             : SCSI VPD page 0x83 ID\n"
 "-l lun_id                : requested LUN number\n"
 "-o name=value            : backend-specific options, multiple allowed\n"
 "-s size_bytes            : LUN size in bytes (some backends)\n"
 "-S serial_num            : SCSI VPD page 0x80 serial number\n"
 "-t dev_type              : SCSI device type (0=disk, 3=processor)\n"
 "remove options:\n"
 "-b backend               : backend name (\"block\", \"ramdisk\", etc.)\n"
 "-l lun_id                : LUN number to delete\n"
 "-o name=value            : backend-specific options, multiple allowed\n"
 "devlist options:\n"
 "-b backend               : list devices from specified backend only\n"
 "-v                       : be verbose, show backend attributes\n"
 "-x                       : dump raw XML\n"
 "delay options:\n"
 "-l datamove|done         : delay command at datamove or done phase\n"
 "-T oneshot               : delay one command, then resume normal completion\n"
 "-T cont                  : delay all commands\n"
 "-t secs                  : number of seconds to delay\n"
 "inject options:\n"
 "-i error_action          : action to perform\n"
 "-p pattern               : command pattern to look for\n"
 "-r lba,len               : LBA range for pattern\n"
 "-s len fmt [args]        : sense data for custom sense action\n"
 "-c                       : continuous operation\n"
 "-d delete_id             : error id to delete\n"
 "port options:\n"
 "-c                       : create new ioctl or iscsi frontend port\n"
 "-d                       : specify ioctl or iscsi frontend type\n"
 "-l                       : list frontend ports\n"
 "-o on|off                : turn frontend ports on or off\n"
 "-O pp|vp                 : create new frontend port using pp and/or vp\n"
 "-w wwnn                  : set WWNN for one frontend\n"
 "-W wwpn                  : set WWPN for one frontend\n"
 "-t port_type             : specify fc, scsi, ioctl, internal frontend type\n"
 "-p targ_port             : specify target port number\n"
 "-r                       : remove frontend port\n" 
 "-q                       : omit header in list output\n"
 "-x                       : output port list in XML format\n"
 "portlist options:\n"
 "-f frontend              : specify frontend type\n"
 "-i                       : report target and initiators addresses\n"
 "-l                       : report LUN mapping\n"
 "-p targ_port             : specify target port number\n"
 "-q                       : omit header in list output\n"
 "-v                       : verbose output (report all port options)\n"
 "-x                       : output port list in XML format\n"
 "lunmap options:\n"
 "-p targ_port             : specify target port number\n"
 "-l pLUN                  : specify port-visible LUN\n"
 "-L cLUN                  : specify CTL LUN\n",
 CTL_DEFAULT_DEV);
 }
 
 int
 main(int argc, char **argv)
 {
 	int c;
 	ctladm_cmdfunction command;
 	ctladm_cmdargs cmdargs;
 	ctladm_optret optreturn;
 	char *device;
 	const char *mainopt = "C:D:I:";
 	const char *subopt = NULL;
 	char combinedopt[256];
 	int lun;
 	int optstart = 2;
 	int retval, fd;
 	int retries;
 	int initid;
 	int saved_errno;
 
 	retval = 0;
 	cmdargs = CTLADM_ARG_NONE;
 	command = CTLADM_CMD_HELP;
 	device = NULL;
 	fd = -1;
 	retries = 0;
 	lun = 0;
 	initid = 7;
 
 	if (argc < 2) {
 		usage(1);
 		retval = 1;
 		goto bailout;
 	}
 
 	/*
 	 * Get the base option.
 	 */
 	optreturn = getoption(option_table,argv[1], &command, &cmdargs,&subopt);
 
 	if (optreturn == CC_OR_AMBIGUOUS) {
 		warnx("ambiguous option %s", argv[1]);
 		usage(0);
 		exit(1);
 	} else if (optreturn == CC_OR_NOT_FOUND) {
 		warnx("option %s not found", argv[1]);
 		usage(0);
 		exit(1);
 	}
 
 	if (cmdargs & CTLADM_ARG_NEED_TL) {
 		if ((argc < 3) || (!isdigit(argv[2][0]))) {
 			warnx("option %s requires a lun argument",
 			      argv[1]);
 			usage(0);
 			exit(1);
 		}
 		lun = strtol(argv[2], NULL, 0);
 
 		cmdargs |= CTLADM_ARG_TARG_LUN;
 		optstart++;
 	}
 
 	/*
 	 * Ahh, getopt(3) is a pain.
 	 *
 	 * This is a gross hack.  There really aren't many other good
 	 * options (excuse the pun) for parsing options in a situation like
 	 * this.  getopt is kinda braindead, so you end up having to run
 	 * through the options twice, and give each invocation of getopt
 	 * the option string for the other invocation.
 	 *
 	 * You would think that you could just have two groups of options.
 	 * The first group would get parsed by the first invocation of
 	 * getopt, and the second group would get parsed by the second
 	 * invocation of getopt.  It doesn't quite work out that way.  When
 	 * the first invocation of getopt finishes, it leaves optind pointing
 	 * to the argument _after_ the first argument in the second group.
 	 * So when the second invocation of getopt comes around, it doesn't
 	 * recognize the first argument it gets and then bails out.
 	 *
 	 * A nice alternative would be to have a flag for getopt that says
 	 * "just keep parsing arguments even when you encounter an unknown
 	 * argument", but there isn't one.  So there's no real clean way to
 	 * easily parse two sets of arguments without having one invocation
 	 * of getopt know about the other.
 	 *
 	 * Without this hack, the first invocation of getopt would work as
 	 * long as the generic arguments are first, but the second invocation
 	 * (in the subfunction) would fail in one of two ways.  In the case
 	 * where you don't set optreset, it would fail because optind may be
 	 * pointing to the argument after the one it should be pointing at.
 	 * In the case where you do set optreset, and reset optind, it would
 	 * fail because getopt would run into the first set of options, which
 	 * it doesn't understand.
 	 *
 	 * All of this would "sort of" work if you could somehow figure out
 	 * whether optind had been incremented one option too far.  The
 	 * mechanics of that, however, are more daunting than just giving
 	 * both invocations all of the expect options for either invocation.
 	 *
 	 * Needless to say, I wouldn't mind if someone invented a better
 	 * (non-GPL!) command line parsing interface than getopt.  I
 	 * wouldn't mind if someone added more knobs to getopt to make it
 	 * work better.  Who knows, I may talk myself into doing it someday,
 	 * if the standards weenies let me.  As it is, it just leads to
 	 * hackery like this and causes people to avoid it in some cases.
 	 *
 	 * KDM, September 8th, 1998
 	 */
 	if (subopt != NULL)
 		sprintf(combinedopt, "%s%s", mainopt, subopt);
 	else
 		sprintf(combinedopt, "%s", mainopt);
 
 	/*
 	 * Start getopt processing at argv[2/3], since we've already
 	 * accepted argv[1..2] as the command name, and as a possible
 	 * device name.
 	 */
 	optind = optstart;
 
 	/*
 	 * Now we run through the argument list looking for generic
 	 * options, and ignoring options that possibly belong to
 	 * subfunctions.
 	 */
 	while ((c = getopt(argc, argv, combinedopt))!= -1){
 		switch (c) {
 		case 'C':
 			cmdargs |= CTLADM_ARG_RETRIES;
 			retries = strtol(optarg, NULL, 0);
 			break;
 		case 'D':
 			device = strdup(optarg);
 			cmdargs |= CTLADM_ARG_DEVICE;
 			break;
 		case 'I':
 			cmdargs |= CTLADM_ARG_INITIATOR;
 			initid = strtol(optarg, NULL, 0);
 			break;
 		default:
 			break;
 		}
 	}
 
 	if ((cmdargs & CTLADM_ARG_INITIATOR) == 0)
 		initid = 7;
 
 	optind = optstart;
 	optreset = 1;
 
 	/*
 	 * Default to opening the CTL device for now.
 	 */
 	if (((cmdargs & CTLADM_ARG_DEVICE) == 0)
 	 && (command != CTLADM_CMD_HELP)) {
 		device = strdup(CTL_DEFAULT_DEV);
 		cmdargs |= CTLADM_ARG_DEVICE;
 	}
 
 	if ((cmdargs & CTLADM_ARG_DEVICE)
 	 && (command != CTLADM_CMD_HELP)) {
 		fd = open(device, O_RDWR);
 		if (fd == -1 && errno == ENOENT) {
 			saved_errno = errno;
 			retval = kldload("ctl");
 			if (retval != -1)
 				fd = open(device, O_RDWR);
 			else
 				errno = saved_errno;
 		}
 		if (fd == -1) {
 			fprintf(stderr, "%s: error opening %s: %s\n",
 				argv[0], device, strerror(errno));
 			retval = 1;
 			goto bailout;
 		}
 #ifdef	WANT_ISCSI
 		switch (command) {
 		case CTLADM_CMD_ISLIST:
 		case CTLADM_CMD_ISLOGOUT:
 		case CTLADM_CMD_ISTERMINATE:
 			if (modfind("cfiscsi") == -1 &&
 			    kldload("cfiscsi") == -1)
 				warn("couldn't load cfiscsi");
 			break;
 		default:
 			break;
 		}
 #endif
 	} else if ((command != CTLADM_CMD_HELP)
 		&& ((cmdargs & CTLADM_ARG_DEVICE) == 0)) {
 		fprintf(stderr, "%s: you must specify a device with the "
 			"--device argument for this command\n", argv[0]);
 		command = CTLADM_CMD_HELP;
 		retval = 1;
 	}
 
 	switch (command) {
 	case CTLADM_CMD_TUR:
 		retval = cctl_tur(fd, lun, initid, retries);
 		break;
 	case CTLADM_CMD_INQUIRY:
 		retval = cctl_inquiry(fd, lun, initid, retries);
 		break;
 	case CTLADM_CMD_REQ_SENSE:
 		retval = cctl_req_sense(fd, lun, initid, retries);
 		break;
 	case CTLADM_CMD_REPORT_LUNS:
 		retval = cctl_report_luns(fd, lun, initid, retries);
 		break;
 	case CTLADM_CMD_CREATE:
 		retval = cctl_create_lun(fd, argc, argv, combinedopt);
 		break;
 	case CTLADM_CMD_RM:
 		retval = cctl_rm_lun(fd, argc, argv, combinedopt);
 		break;
 	case CTLADM_CMD_DEVLIST:
 		retval = cctl_devlist(fd, argc, argv, combinedopt);
 		break;
 	case CTLADM_CMD_READ:
 	case CTLADM_CMD_WRITE:
 		retval = cctl_read_write(fd, lun, initid, retries,
 					 argc, argv, combinedopt, command);
 		break;
 	case CTLADM_CMD_PORT:
 		retval = cctl_port(fd, argc, argv, combinedopt);
 		break;
 	case CTLADM_CMD_PORTLIST:
 		retval = cctl_portlist(fd, argc, argv, combinedopt);
 		break;
 	case CTLADM_CMD_LUNMAP:
 		retval = cctl_lunmap(fd, argc, argv, combinedopt);
 		break;
 	case CTLADM_CMD_READCAPACITY:
 		retval = cctl_read_capacity(fd, lun, initid, retries,
 					    argc, argv, combinedopt);
 		break;
 	case CTLADM_CMD_MODESENSE:
 		retval = cctl_mode_sense(fd, lun, initid, retries,
 					 argc, argv, combinedopt);
 		break;
 	case CTLADM_CMD_START:
 	case CTLADM_CMD_STOP:
 		retval = cctl_start_stop(fd, lun, initid, retries,
 					 (command == CTLADM_CMD_START) ? 1 : 0,
 					 argc, argv, combinedopt);
 		break;
 	case CTLADM_CMD_SYNC_CACHE:
 		retval = cctl_sync_cache(fd, lun, initid, retries,
 					 argc, argv, combinedopt);
 		break;
 	case CTLADM_CMD_LUNLIST:
 		retval = cctl_lunlist(fd);
 		break;
 	case CTLADM_CMD_DELAY:
 		retval = cctl_delay(fd, lun, argc, argv, combinedopt);
 		break;
 	case CTLADM_CMD_ERR_INJECT:
 		retval = cctl_error_inject(fd, lun, argc, argv,
 					   combinedopt);
 		break;
 	case CTLADM_CMD_DUMPOOA:
 		retval = cctl_dump_ooa(fd, argc, argv);
 		break;
 	case CTLADM_CMD_DUMPSTRUCTS:
 		retval = cctl_dump_structs(fd, cmdargs);
 		break;
 	case CTLADM_CMD_PRES_IN:
 		retval = cctl_persistent_reserve_in(fd, lun, initid,
 		                                    argc, argv, combinedopt,
 						    retries);
 		break;
 	case CTLADM_CMD_PRES_OUT:
 		retval = cctl_persistent_reserve_out(fd, lun, initid,
 						     argc, argv, combinedopt,
 						     retries);
 		break;
 	case CTLADM_CMD_INQ_VPD_DEVID:
 	        retval = cctl_inquiry_vpd_devid(fd, lun, initid);
 		break;
 	case CTLADM_CMD_RTPG:
 	        retval = cctl_report_target_port_group(fd, lun, initid);
 		break;
 	case CTLADM_CMD_MODIFY:
 	        retval = cctl_modify_lun(fd, argc, argv, combinedopt);
 		break;
 	case CTLADM_CMD_ISLIST:
 	        retval = cctl_islist(fd, argc, argv, combinedopt);
 		break;
 	case CTLADM_CMD_ISLOGOUT:
 	        retval = cctl_islogout(fd, argc, argv, combinedopt);
 		break;
 	case CTLADM_CMD_ISTERMINATE:
 	        retval = cctl_isterminate(fd, argc, argv, combinedopt);
 		break;
+	case CTLADM_CMD_NVLIST:
+	        retval = cctl_nvlist(fd, argc, argv, combinedopt);
+		break;
 	case CTLADM_CMD_HELP:
 	default:
 		usage(retval);
 		break;
 	}
 bailout:
 
 	if (fd != -1)
 		close(fd);
 
 	exit (retval);
 }
 
 /*
  * vim: ts=8
  */